| Index: trunk/WikiWord/WikiWordIntegrator/src/test/java/de/brightbyte/wikiword/integrator/LoadForeignPropertiesTest-tableImport.csv |
| — | — | @@ -0,0 +1,3 @@ |
| | 2 | +concept_name,name,firstName,phone |
| | 3 | +FooBar,Foo,Bar,12345 |
| | 4 | +XyzzyQuux,Xyzzy,Quux,67890 |
| \ No newline at end of file |
| Index: trunk/WikiWord/WikiWordIntegrator/src/test/java/de/brightbyte/wikiword/integrator/LoadForeignPropertiesTest-tableImport.properties |
| — | — | @@ -0,0 +1,5 @@ |
| | 2 | +file="LoadForeignPropertiesTest-tableImport.csv" |
| | 3 | +authority="acme" |
| | 4 | +concept-id-field="concept_name" |
| | 5 | +fields=(concept_name,name,firstName,phone) |
| | 6 | + |
| Index: trunk/WikiWord/WikiWordIntegrator/src/test/java/de/brightbyte/wikiword/integrator/LoadForeignPropertiesTest.xml |
| — | — | @@ -0,0 +1,8 @@ |
| | 2 | +<?xml version='1.0' encoding='UTF-8'?> |
| | 3 | +<dataset> |
| | 4 | + <tripleImport concept_name="FooBar" name="Foo" firstName="Bar" phone="12345"/> |
| | 5 | + <tripleImport concept_name="XyzzyQuux" name="Xyzzy" firstName="Quux" phone="67890"/> |
| | 6 | + <tableImport concept_name="FooBar" name="Foo" firstName="Bar" phone="12345"/> |
| | 7 | + <tableImport concept_name="XyzzyQuux" name="Xyzzy" firstName="Quux" phone="67890"/> |
| | 8 | +</dataset> |
| | 9 | + |
| Index: trunk/WikiWord/WikiWordIntegrator/src/test/java/de/brightbyte/wikiword/integrator/test-tweaks.properties |
| — | — | @@ -0,0 +1,100 @@ |
| | 2 | +## System config |
| | 3 | +console.encoding = "UTF-8" |
| | 4 | + |
| | 5 | +## language handling |
| | 6 | + |
| | 7 | +# treat "commons" as a language code |
| | 8 | +languages.commonsAsLanguage = false |
| | 9 | + |
| | 10 | +# treat "simple" as a language code |
| | 11 | +languages.simpleAsLanguage = true |
| | 12 | + |
| | 13 | +## RDF Export Config |
| | 14 | +# datase URI qualifier - should uniquely identify the entity creating |
| | 15 | +# the datasets (that is, YOU). This enusres unique dataset URIs for |
| | 16 | +# datasets created by different people. |
| | 17 | +# The default is "*" which means "unknown, don't use this publically". |
| | 18 | +rdf.dataset.qualifier = "*" |
| | 19 | + |
| | 20 | +## Import Driver |
| | 21 | +# Run import in a thread separate from the one reading and parsing the dump. |
| | 22 | +# Disabling this by the queue size to 0 will slightly reduce overhead on single-core systems; |
| | 23 | +# Using a queue will not have as big an impact if unzippers (bunzip/gunzip) are used |
| | 24 | +#dumpdriver.pageImportQueue = 8 |
| | 25 | +dumpdriver.pageImportQueue = 0 |
| | 26 | + |
| | 27 | +# external unzippers - may boost performance, especially |
| | 28 | +# on multi-core systems. The name of the file to |
| | 29 | +# unzip will be appended to the command given here. Spaces |
| | 30 | +# before the last / are taken to be part of the path, spaces |
| | 31 | +# after the last / separate parameters. |
| | 32 | +input.externalBunzip = null |
| | 33 | +input.externalGunzip = null |
| | 34 | +#input.externalBunzip = "/bin/bunzip2 -c" |
| | 35 | +#input.externalGunzip = "/bin/gunzip -c" |
| | 36 | + |
| | 37 | +### Importer Output and Persistance ############ |
| | 38 | +importer.progressInterval = 1000 |
| | 39 | +importer.safepointInterval = 30000 |
| | 40 | +#importer.safepointInterval = 1000 |
| | 41 | + |
| | 42 | +### Database Performance ####################### |
| | 43 | +#dbstore.backgroundFlushQueue = 4 |
| | 44 | +dbstore.backgroundFlushQueue = 0 |
| | 45 | +dbstore.useEntityBuffer = false |
| | 46 | +dbstore.useRelationBuffer = false |
| | 47 | +#dbstore.useEntityBuffer = true |
| | 48 | +#dbstore.useRelationBuffer = true |
| | 49 | +#dbstore.insertionBufferFactor = 16 |
| | 50 | +dbstore.insertionBufferFactor = 64 |
| | 51 | +dbstore.engine = "MyISAM" |
| | 52 | +#dbstore.engine = "InnoDB" |
| | 53 | + |
| | 54 | +#sql mode - see http://dev.mysql.com/doc/refman/5.1/en/server-sql-mode.html |
| | 55 | +dbstore.sqlMode = "STRICT_ALL_TABLES" |
| | 56 | +#dbstore.sqlMode = "STRICT_TRANS_TABLES" |
| | 57 | + |
| | 58 | +#NOTE: MySQL does not support 4-byte utf-8 codes. So turn everything into binary... |
| | 59 | +dbstore.useBinaryText = true; |
| | 60 | + |
| | 61 | +#maximum size of sql statement, bytes! MySQL's default: a bit below 16M bytes (!) |
| | 62 | +#if not specified, mysql is asked for the current value. |
| | 63 | +#dbstore.maxStatementSize = 16776192; |
| | 64 | + |
| | 65 | +#chunk size to use when chunking large updates by id |
| | 66 | +#default is 100000, set to 0 to disable all chunking |
| | 67 | +dbstore.queryChunkSize = 100000 |
| | 68 | +dbstore.updateChunkSize = 100000 |
| | 69 | + |
| | 70 | +### Property Cache Fields ########################### |
| | 71 | +#dbstore.cacheReferenceSeparator = '\u001E' |
| | 72 | +#dbstore.cacheReferenceFieldSeparator = '\u001F' |
| | 73 | +dbstore.listBlobSize = 65025 |
| | 74 | + |
| | 75 | +### ID manager ###################################### |
| | 76 | +# NOTE: when using this, allow for 116 bytes plus the average size of names per ID entry. |
| | 77 | +# So if you have anaverage name length of 12 and expect1million entries, |
| | 78 | +# allow for about 1.3 gigabyte RAM to be used for ID caching. |
| | 79 | +dbstore.idManager=false |
| | 80 | +#dbstore.auxFileDir defaults to system temp dir |
| | 81 | +#dbstore.auxFileDir="/tmp" |
| | 82 | +dbstore.idManager.bufferSize=16384 |
| | 83 | + |
| | 84 | +### CycleFinder ##################################### |
| | 85 | +dbstore.CycleFinder.levelWarningThreshold=32 |
| | 86 | +dbstore.CycleFinder.degreeWarningThreshold=1024 |
| | 87 | +dbstore.CycleFinder.maxDepth=1024 |
| | 88 | + |
| | 89 | +### Database Debug Output ###################### |
| | 90 | +#see java.util.logging.Level for codes to use with dbstore.logLevel |
| | 91 | +dbstore.logLevel = 720 |
| | 92 | +dbstore.explainSQLThreashold = 0 |
| | 93 | +#dbstore.explainSQLThreashold = 1000000 |
| | 94 | +dbstore.slowSQLThreashold = 0 |
| | 95 | +#dbstore.slowSQLThreashold = 10 |
| | 96 | +#dbstore.slowSQLThreashold = 60 |
| | 97 | +dbstore.traceSQL = false |
| | 98 | + |
| | 99 | +### Custom special purpose packages ################# |
| | 100 | +wikiword.ConfigPackages=["de.brightbyte.wikiword.bibliography","de.brightbyte.wikiword.bibliography.wikis"] |
| | 101 | + |
| Index: trunk/WikiWord/WikiWordIntegrator/src/test/java/de/brightbyte/wikiword/integrator/LoadForeignPropertiesTest-tripleImport.csv |
| — | — | @@ -0,0 +1,7 @@ |
| | 2 | +concept_name,property,value |
| | 3 | +FooBar,name,Foo |
| | 4 | +FooBar,firstName,Bar |
| | 5 | +FooBar,phone,12345 |
| | 6 | +XyzzyQuux,name,Xyzzy |
| | 7 | +XyzzyQuux,firstName,Quux |
| | 8 | +XyzzyQuux,phone,67890 |
| \ No newline at end of file |
| Index: trunk/WikiWord/WikiWordIntegrator/src/test/java/de/brightbyte/wikiword/integrator/LoadForeignPropertiesTest-tripleImport.properties |
| — | — | @@ -0,0 +1,6 @@ |
| | 2 | +file="LoadForeignPropertiesTest-tripleImport.csv" |
| | 3 | +authority="acme" |
| | 4 | +concept-id-field="concept_name" |
| | 5 | +property-name-field="property" |
| | 6 | +property-value-field="value" |
| | 7 | + |
| Index: trunk/WikiWord/WikiWordIntegrator/src/test/java/de/brightbyte/wikiword/integrator/IntegratorAppTestBase.java |
| — | — | @@ -14,36 +14,38 @@ |
| 15 | 15 | } |
| 16 | 16 | |
| 17 | 17 | public TweakSet loadTweakSet() throws IOException { |
| 18 | | - URL url = requireAuxilliaryFileURL(getBaseName()+".tweaks.properties"); |
| | 18 | + URL url = requireAuxilliaryFileURL("test-tweaks.properties"); |
| 19 | 19 | TweakSet tweaks = new TweakSet(); |
| 20 | 20 | tweaks.loadTweaks(url); |
| 21 | 21 | return tweaks; |
| 22 | 22 | } |
| 23 | 23 | |
| 24 | 24 | public FeatureSetSourceDescriptor loadSourceDescriptor(String testName) throws IOException { |
| 25 | | - URL url = requireAuxilliaryFileURL(getBaseName()+"."+testName+".properties"); |
| | 25 | + URL url = requireAuxilliaryFileURL(getBaseName()+"-"+testName+".properties"); |
| 26 | 26 | FeatureSetSourceDescriptor descriptor = new FeatureSetSourceDescriptor(); |
| 27 | 27 | descriptor.loadTweaks(url); |
| 28 | 28 | return descriptor; |
| 29 | 29 | } |
| 30 | | - |
| | 30 | + |
| 31 | 31 | protected abstract T createApp(); |
| 32 | 32 | |
| 33 | | - protected T prepareApp(FeatureSetSourceDescriptor sourceDescriptor) throws IOException { |
| | 33 | + protected T prepareApp(FeatureSetSourceDescriptor sourceDescriptor, String targetTable) throws IOException { |
| 34 | 34 | TweakSet tweaks = loadTweakSet(); |
| 35 | 35 | T app = createApp(); |
| 36 | 36 | |
| 37 | | - app.testInit(testDataSource, DatasetIdentifier.forName("TEST", "xx"), tweaks, sourceDescriptor); |
| | 37 | + app.testInit(testDataSource, DatasetIdentifier.forName("TEST", "xx"), tweaks, sourceDescriptor, targetTable); |
| 38 | 38 | return app; |
| 39 | 39 | } |
| 40 | 40 | |
| 41 | 41 | protected void runApp(String testName) throws Exception { |
| 42 | 42 | FeatureSetSourceDescriptor source = loadSourceDescriptor(testName); |
| 43 | | - runApp(source); |
| | 43 | + launchApp(source, testName); |
| | 44 | + |
| | 45 | + assertTableContent(testName, "SELECT * FROM "+testName); //FIXME: sort order |
| 44 | 46 | } |
| 45 | 47 | |
| 46 | | - protected void runApp(FeatureSetSourceDescriptor sourceDescriptor) throws Exception { |
| 47 | | - T app = prepareApp(sourceDescriptor); |
| | 48 | + protected void launchApp(FeatureSetSourceDescriptor sourceDescriptor, String targetTable) throws Exception { |
| | 49 | + T app = prepareApp(sourceDescriptor, targetTable); |
| 48 | 50 | app.testLaunch(); |
| 49 | 51 | } |
| 50 | 52 | |