Skip to content

Apache Lucene open-source search software

Migrate OpenNLP 'ant train-test-models' to Gradle (#14198)

Michael Froh30d18dfbe1e6
diff --git a/lucene/analysis/opennlp/build.gradle b/lucene/analysis/opennlp/build.gradle
index 02449c9f819..da3a960ab08 100644
--- a/lucene/analysis/opennlp/build.gradle
+++ b/lucene/analysis/opennlp/build.gradle
@@ -26,3 +26,32 @@ dependencies {
moduleTestImplementation project(':lucene:test-framework')
}
+
+ext {
+ testModelDataDir = file('src/tools/test-model-data')
+ testsUserDir = file('src/test-files')
+ testModelDir = file("${testsUserDir}/org/apache/lucene/analysis/opennlp")
+}
+
+tasks.register('trainTestModels') {
+ description = 'Train all small test models for unit tests'
+ doLast {
+ mkdir testModelDir
+ trainModel('SentenceDetectorTrainer', 'en', 'sentences.txt', 'en-test-sent.bin')
+ trainModel('TokenizerTrainer', 'en', 'tokenizer.txt', 'en-test-tokenizer.bin')
+ trainModel('POSTaggerTrainer', 'en', 'pos.txt', 'en-test-pos-maxent.bin')
+ trainModel('ChunkerTrainerME', 'en', 'chunks.txt', 'en-test-chunker.bin')
+ trainModel('TokenNameFinderTrainer', 'en', 'ner.txt', 'en-test-ner.bin', ['-params', 'ner_TrainerParams.txt'])
+ trainModel('LemmatizerTrainerME', 'en', 'lemmas.txt', 'en-test-lemmatizer.bin')
+ }
+}
+
+def trainModel(String command, String lang, String data, String model, List extraArgs = []) {
+ javaexec {
+ classpath = sourceSets.main.compileClasspath
+ mainClass = 'opennlp.tools.cmdline.CLI'
+ workingDir = testModelDataDir
+ args = [command, '-lang', lang, '-data', data, '-model', "${testModelDir}/${model}"] + extraArgs
+ }
+}
+
diff --git a/lucene/analysis/opennlp/src/test-files/org/apache/lucene/analysis/opennlp/en-test-chunker.bin b/lucene/analysis/opennlp/src/test-files/org/apache/lucene/analysis/opennlp/en-test-chunker.bin
index 4bc4a7a77de..d299b73df39 100644
Binary files a/lucene/analysis/opennlp/src/test-files/org/apache/lucene/analysis/opennlp/en-test-chunker.bin and b/lucene/analysis/opennlp/src/test-files/org/apache/lucene/analysis/opennlp/en-test-chunker.bin differ
diff --git a/lucene/analysis/opennlp/src/test-files/org/apache/lucene/analysis/opennlp/en-test-lemmatizer.bin b/lucene/analysis/opennlp/src/test-files/org/apache/lucene/analysis/opennlp/en-test-lemmatizer.bin
index 140b39c9942..651f2e86bdb 100644
Binary files a/lucene/analysis/opennlp/src/test-files/org/apache/lucene/analysis/opennlp/en-test-lemmatizer.bin and b/lucene/analysis/opennlp/src/test-files/org/apache/lucene/analysis/opennlp/en-test-lemmatizer.bin differ
diff --git a/lucene/analysis/opennlp/src/test-files/org/apache/lucene/analysis/opennlp/en-test-ner.bin b/lucene/analysis/opennlp/src/test-files/org/apache/lucene/analysis/opennlp/en-test-ner.bin
index 42fe17d0c30..7bfa6df700c 100644
Binary files a/lucene/analysis/opennlp/src/test-files/org/apache/lucene/analysis/opennlp/en-test-ner.bin and b/lucene/analysis/opennlp/src/test-files/org/apache/lucene/analysis/opennlp/en-test-ner.bin differ
diff --git a/lucene/analysis/opennlp/src/test-files/org/apache/lucene/analysis/opennlp/en-test-pos-maxent.bin b/lucene/analysis/opennlp/src/test-files/org/apache/lucene/analysis/opennlp/en-test-pos-maxent.bin
index 8c452e66ce5..ff96a71317c 100644
Binary files a/lucene/analysis/opennlp/src/test-files/org/apache/lucene/analysis/opennlp/en-test-pos-maxent.bin and b/lucene/analysis/opennlp/src/test-files/org/apache/lucene/analysis/opennlp/en-test-pos-maxent.bin differ
diff --git a/lucene/analysis/opennlp/src/test-files/org/apache/lucene/analysis/opennlp/en-test-sent.bin b/lucene/analysis/opennlp/src/test-files/org/apache/lucene/analysis/opennlp/en-test-sent.bin
index ae297091aac..2d8d374f949 100644
Binary files a/lucene/analysis/opennlp/src/test-files/org/apache/lucene/analysis/opennlp/en-test-sent.bin and b/lucene/analysis/opennlp/src/test-files/org/apache/lucene/analysis/opennlp/en-test-sent.bin differ
diff --git a/lucene/analysis/opennlp/src/test-files/org/apache/lucene/analysis/opennlp/en-test-tokenizer.bin b/lucene/analysis/opennlp/src/test-files/org/apache/lucene/analysis/opennlp/en-test-tokenizer.bin
index df084d1da34..124675b8449 100644
Binary files a/lucene/analysis/opennlp/src/test-files/org/apache/lucene/analysis/opennlp/en-test-tokenizer.bin and b/lucene/analysis/opennlp/src/test-files/org/apache/lucene/analysis/opennlp/en-test-tokenizer.bin differ
diff --git a/lucene/analysis/opennlp/src/test/org/apache/lucene/analysis/opennlp/TestOpenNLPChunkerFilterFactory.java b/lucene/analysis/opennlp/src/test/org/apache/lucene/analysis/opennlp/TestOpenNLPChunkerFilterFactory.java
index dc21abe1a7d..708f7bafc17 100644
--- a/lucene/analysis/opennlp/src/test/org/apache/lucene/analysis/opennlp/TestOpenNLPChunkerFilterFactory.java
+++ b/lucene/analysis/opennlp/src/test/org/apache/lucene/analysis/opennlp/TestOpenNLPChunkerFilterFactory.java
@@ -58,7 +58,7 @@ public class TestOpenNLPChunkerFilterFactory extends BaseTokenStreamTestCase {
8, 15, 17, 21, 23, 29, 30, 39, 46, 48, 49, 51, 57, 58
};
private static final String[] SENTENCES_chunks = {
- "B-NP", "I-NP", "I-NP", "I-NP", "I-NP", "I-NP", "O", "B-NP", "I-NP", "I-NP", "O", "B-NP",
+ "B-NP", "I-NP", "I-NP", "B-VP", "B-NP", "I-NP", "O", "B-NP", "I-NP", "I-NP", "O", "B-NP",
"I-NP", "O"
};