Apache Lucene open-source search software
LUCENE-8183: Added the abbility to get noSubMatches and noOverlapping Matches (#12437)
Martin Demberger55f2f9958bd8
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 2eed5e676cb..8fcd8f97af5 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -132,6 +132,9 @@ New Features* GITHUB#12383: Introduced LeafCollector#finish, a hook that runs aftercollection has finished running on a leaf. (Adrien Grand)+* LUCENE-8183, GITHUB#9231: Added the abbility to get noSubMatches and noOverlappingMatches in+ HyphenationCompoundWordFilter (Martin Demberger, original from Rupert Westenthaler)+Improvements---------------------* GITHUB#12374: Add CachingLeafSlicesSupplier to compute the LeafSlices for concurrent segment search (Sorabh Hamirwasia)diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.javaindex 20378288052..254d6c61b48 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilter.java@@ -34,6 +34,9 @@ import org.xml.sax.InputSource;*/public class HyphenationCompoundWordTokenFilter extends CompoundWordTokenFilterBase {private final HyphenationTree hyphenator;+ private final boolean noSubMatches;+ private final boolean noOverlappingMatches;+ private final boolean calcSubMatches;/*** Creates a new {@link HyphenationCompoundWordTokenFilter} instance.@@ -51,6 +54,8 @@ public class HyphenationCompoundWordTokenFilter extends CompoundWordTokenFilterBDEFAULT_MIN_WORD_SIZE,DEFAULT_MIN_SUBWORD_SIZE,DEFAULT_MAX_SUBWORD_SIZE,+ false,+ false,false);}@@ -73,9 +78,47 @@ public class HyphenationCompoundWordTokenFilter extends CompoundWordTokenFilterBint minSubwordSize,int maxSubwordSize,boolean onlyLongestMatch) {+ this(+ input,+ hyphenator,+ dictionary,+ minWordSize,+ minSubwordSize,+ maxSubwordSize,+ onlyLongestMatch,+ false,+ false);+ }++ /**+ * Creates a new {@link HyphenationCompoundWordTokenFilter} instance.+ *+ * @param input the {@link org.apache.lucene.analysis.TokenStream} to process+ * @param hyphenator the hyphenation pattern tree to use for hyphenation+ * @param dictionary the word dictionary to match against.+ * @param minWordSize only words longer than this get processed+ * @param minSubwordSize only subwords longer than this get to the output stream+ * @param maxSubwordSize only subwords shorter than this get to the output stream+ * @param onlyLongestMatch Add only the longest matching subword to the stream+ * @param noSubMatches Excludes subwords that are enclosed by an other token+ * @param noOverlappingMatches Excludes subwords that overlap with an other subword+ */+ public HyphenationCompoundWordTokenFilter(+ TokenStream input,+ HyphenationTree hyphenator,+ CharArraySet dictionary,+ int minWordSize,+ int minSubwordSize,+ int maxSubwordSize,+ boolean onlyLongestMatch,+ boolean noSubMatches,+ boolean noOverlappingMatches) {super(input, dictionary, minWordSize, minSubwordSize, maxSubwordSize, onlyLongestMatch);this.hyphenator = Objects.requireNonNull(hyphenator, "hyphenator");+ this.noSubMatches = noSubMatches;+ this.noOverlappingMatches = noOverlappingMatches;+ this.calcSubMatches = !onlyLongestMatch && !noSubMatches && !noOverlappingMatches;}/**@@ -140,26 +183,44 @@ public class HyphenationCompoundWordTokenFilter extends CompoundWordTokenFilterB@Overrideprotected void decompose() {+ // if the token is in the dictionary and we are not interested in subMatches+ // we can skip decomposing this token (see testNoSubAndTokenInDictionary unit test)+ // NOTE:+ // we check against token and the token that is one character+ // shorter to avoid problems with genitive 's characters and other binding characters+ if (dictionary != null+ && !this.calcSubMatches+ && (dictionary.contains(termAtt.buffer(), 0, termAtt.length())+ || termAtt.length() > 1+ && dictionary.contains(termAtt.buffer(), 0, termAtt.length() - 1))) {+ return; // the whole token is in the dictionary - do not decompose+ }+// get the hyphenation pointsHyphenation hyphens = hyphenator.hyphenate(termAtt.buffer(), 0, termAtt.length(), 1, 1);// No hyphen points found -> exitif (hyphens == null) {return;}+ int maxSubwordSize = Math.min(this.maxSubwordSize, termAtt.length() - 1);++ int consumed = -1; // hyp of the longest token added (for noSub)final int[] hyp = hyphens.getHyphenationPoints();for (int i = 0; i < hyp.length; ++i) {- int remaining = hyp.length - i;+ if (noOverlappingMatches) { // if we do not want overlapping subwords+ i = Math.max(i, consumed); // skip over consumed hyp+ }int start = hyp[i];- CompoundToken longestMatchToken = null;- for (int j = 1; j < remaining; j++) {- int partLength = hyp[i + j] - start;+ int until = noSubMatches ? Math.max(consumed, i) : i;+ for (int j = hyp.length - 1; j > until; j--) {+ int partLength = hyp[j] - start;// if the part is longer than maxSubwordSize we// are done with this round- if (partLength > this.maxSubwordSize) {- break;+ if (partLength > maxSubwordSize) {+ continue;}// we only put subwords to the token stream@@ -167,42 +228,26 @@ public class HyphenationCompoundWordTokenFilter extends CompoundWordTokenFilterBif (partLength < this.minSubwordSize) {// BOGUS/BROKEN/FUNKY/WACKO: somehow we have negative 'parts' according to the// calculation above, and we rely upon minSubwordSize being >=0 to filter them out...- continue;+ break;}// check the dictionaryif (dictionary == null || dictionary.contains(termAtt.buffer(), start, partLength)) {- if (this.onlyLongestMatch) {- if (longestMatchToken != null) {- if (longestMatchToken.txt.length() < partLength) {- longestMatchToken = new CompoundToken(start, partLength);- }- } else {- longestMatchToken = new CompoundToken(start, partLength);- }- } else {- tokens.add(new CompoundToken(start, partLength));+ tokens.add(new CompoundToken(start, partLength));+ consumed = j; // mark the current hyp as consumed+ if (!calcSubMatches) {+ break; // do not search for shorter matches}} else if (dictionary.contains(termAtt.buffer(), start, partLength - 1)) {// check the dictionary again with a word that is one character- // shorter- // to avoid problems with genitive 's characters and other binding- // characters- if (this.onlyLongestMatch) {- if (longestMatchToken != null) {- if (longestMatchToken.txt.length() < partLength - 1) {- longestMatchToken = new CompoundToken(start, partLength - 1);- }- } else {- longestMatchToken = new CompoundToken(start, partLength - 1);- }- } else {- tokens.add(new CompoundToken(start, partLength - 1));+ // shorter to avoid problems with genitive 's characters and+ // other binding characters+ tokens.add(new CompoundToken(start, partLength - 1));+ consumed = j; // mark the current hyp as consumed+ if (!calcSubMatches) {+ break; // do not search for shorter matches}- }- }- if (this.onlyLongestMatch && longestMatchToken != null) {- tokens.add(longestMatchToken);+ } // else dictionary is present but does not contain the part}}}diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilterFactory.javaindex a1f8809c27d..8a40de23fb8 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilterFactory.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/compound/HyphenationCompoundWordTokenFilterFactory.java@@ -77,6 +77,8 @@ public class HyphenationCompoundWordTokenFilterFactory extends TokenFilterFactorprivate final int minSubwordSize;private final int maxSubwordSize;private final boolean onlyLongestMatch;+ private final boolean noSubMatches;+ private final boolean noOverlappingMatches;/** Creates a new HyphenationCompoundWordTokenFilterFactory */public HyphenationCompoundWordTokenFilterFactory(Map<String, String> args) {@@ -90,6 +92,8 @@ public class HyphenationCompoundWordTokenFilterFactory extends TokenFilterFactormaxSubwordSize =getInt(args, "maxSubwordSize", CompoundWordTokenFilterBase.DEFAULT_MAX_SUBWORD_SIZE);onlyLongestMatch = getBoolean(args, "onlyLongestMatch", false);+ noSubMatches = getBoolean(args, "noSubMatches", false);+ noOverlappingMatches = getBoolean(args, "noOverlappingMatches", false);if (!args.isEmpty()) {throw new IllegalArgumentException("Unknown parameters: " + args);}@@ -127,6 +131,8 @@ public class HyphenationCompoundWordTokenFilterFactory extends TokenFilterFactorminWordSize,minSubwordSize,maxSubwordSize,- onlyLongestMatch);+ onlyLongestMatch,+ noSubMatches,+ noOverlappingMatches);}}diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.javaindex 67a36f94223..446f25d8d1d 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestCompoundWordTokenFilter.java@@ -104,7 +104,7 @@ public class TestCompoundWordTokenFilter extends BaseTokenStreamTestCase {// min=2, max=4assertTokenStreamContents(- tf, new String[] {"basketballkurv", "ba", "sket", "bal", "ball", "kurv"});+ tf, new String[] {"basketballkurv", "ba", "sket", "ball", "bal", "kurv"});tf =new HyphenationCompoundWordTokenFilter(@@ -131,14 +131,14 @@ public class TestCompoundWordTokenFilter extends BaseTokenStreamTestCase {tf,new String[] {"basketballkurv",- "basket",- "basketbal","basketball",- "sket",- "sketbal",+ "basketbal",+ "basket","sketball",- "ball",+ "sketbal",+ "sket","ballkurv",+ "ball","lkurv","kurv"});@@ -367,8 +367,165 @@ public class TestCompoundWordTokenFilter extends BaseTokenStreamTestCase {HyphenationCompoundWordTokenFilter tf =new HyphenationCompoundWordTokenFilter(whitespaceMockTokenizer("Rindfleisch"), hyphenator);- // TODO Rindfleisch returned twice is another issue of the HyphenationCompoundTokenFilter- assertTokenStreamContents(tf, new String[] {"Rindfleisch", "Rind", "Rindfleisch", "fleisch"});+ assertTokenStreamContents(tf, new String[] {"Rindfleisch", "Rind", "fleisch"});+ }++ public void testNoSubAndNoOverlap() throws Exception { // LUCENE-8183+ String input = "fußballpumpe";+ InputSource is =+ new InputSource(getClass().getResource("hyphenation-LUCENE-8183.xml").toExternalForm());+ HyphenationTree hyphenator = HyphenationCompoundWordTokenFilter.getHyphenationTree(is);+ CharArraySet dictionary = makeDictionary("fuß", "fußball", "ballpumpe", "ball", "pumpe");++ // test the default configuration+ HyphenationCompoundWordTokenFilter tf1 =+ new HyphenationCompoundWordTokenFilter(+ whitespaceMockTokenizer(input), hyphenator, dictionary);+ assertTokenStreamContents(+ tf1, new String[] {"fußballpumpe", "fußball", "fuß", "ballpumpe", "ball", "pumpe"});++ // test with onlyLongestMatch+ HyphenationCompoundWordTokenFilter tf2 =+ new HyphenationCompoundWordTokenFilter(+ whitespaceMockTokenizer(input),+ hyphenator,+ dictionary,+ CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE,+ CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE,+ CompoundWordTokenFilterBase.DEFAULT_MAX_SUBWORD_SIZE,+ true);+ assertTokenStreamContents(tf2, new String[] {"fußballpumpe", "fußball", "ballpumpe", "pumpe"});++ // test with noSub enabled and noOverlap disabled+ HyphenationCompoundWordTokenFilter tf3 =+ new HyphenationCompoundWordTokenFilter(+ whitespaceMockTokenizer(input),+ hyphenator,+ dictionary,+ CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE,+ CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE,+ CompoundWordTokenFilterBase.DEFAULT_MAX_SUBWORD_SIZE,+ true,+ true,+ false);+ assertTokenStreamContents(tf3, new String[] {"fußballpumpe", "fußball", "ballpumpe"});+ // assert that the onlyLongestMatch state does not matter if noSub is active+ HyphenationCompoundWordTokenFilter tf3b =+ new HyphenationCompoundWordTokenFilter(+ whitespaceMockTokenizer(input),+ hyphenator,+ dictionary,+ CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE,+ CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE,+ CompoundWordTokenFilterBase.DEFAULT_MAX_SUBWORD_SIZE,+ false,+ true,+ false);+ assertTokenStreamContents(tf3b, new String[] {"fußballpumpe", "fußball", "ballpumpe"});++ // test with noOverlap enabled+ HyphenationCompoundWordTokenFilter tf4 =+ new HyphenationCompoundWordTokenFilter(+ whitespaceMockTokenizer(input),+ hyphenator,+ dictionary,+ CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE,+ CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE,+ CompoundWordTokenFilterBase.DEFAULT_MAX_SUBWORD_SIZE,+ true,+ true,+ true);+ // NOTE: 'fußball' consumes 'ball' as possible start so 'ballpumpe' is not considered and+ // 'pumpe' is added+ assertTokenStreamContents(tf4, new String[] {"fußballpumpe", "fußball", "pumpe"});++ // assert that the noSub and onlyLongestMatch states do not matter+ HyphenationCompoundWordTokenFilter tf4b =+ new HyphenationCompoundWordTokenFilter(+ whitespaceMockTokenizer(input),+ hyphenator,+ dictionary,+ CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE,+ CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE,+ CompoundWordTokenFilterBase.DEFAULT_MAX_SUBWORD_SIZE,+ false,+ false,+ true);+ assertTokenStreamContents(tf4b, new String[] {"fußballpumpe", "fußball", "pumpe"});++ HyphenationCompoundWordTokenFilter tf4c =+ new HyphenationCompoundWordTokenFilter(+ whitespaceMockTokenizer(input),+ hyphenator,+ dictionary,+ CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE,+ CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE,+ CompoundWordTokenFilterBase.DEFAULT_MAX_SUBWORD_SIZE,+ true,+ false,+ true);+ assertTokenStreamContents(tf4c, new String[] {"fußballpumpe", "fußball", "pumpe"});+ }++ public void testNoSubAndTokenInDictionary() throws Exception { // LUCENE-8183+ // test that no subwords are added if the token is part of the dictionary and+ // onlyLongestMatch or noSub is present+ String input = "fußball";+ InputSource is =+ new InputSource(getClass().getResource("hyphenation-LUCENE-8183.xml").toExternalForm());+ HyphenationTree hyphenator = HyphenationCompoundWordTokenFilter.getHyphenationTree(is);+ CharArraySet dictionary = makeDictionary("fußball", "fuß", "ball");++ // test the default configuration as baseline+ HyphenationCompoundWordTokenFilter tf5 =+ new HyphenationCompoundWordTokenFilter(+ whitespaceMockTokenizer(input), hyphenator, dictionary);+ assertTokenStreamContents(tf5, new String[] {"fußball", "fuß", "ball"});++ // when onlyLongestMatch is enabled fußball matches dictionary. So even so+ // fußball is not added as token it MUST prevent shorter matches to be added+ HyphenationCompoundWordTokenFilter tf6 =+ new HyphenationCompoundWordTokenFilter(+ whitespaceMockTokenizer(input),+ hyphenator,+ dictionary,+ CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE,+ CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE,+ CompoundWordTokenFilterBase.DEFAULT_MAX_SUBWORD_SIZE,+ true,+ false,+ false);+ assertTokenStreamContents(tf6, new String[] {"fußball"});++ // when noSub is enabled fuß and ball MUST NOT be added as subwords as fußball is in the+ // dictionary+ HyphenationCompoundWordTokenFilter tf7 =+ new HyphenationCompoundWordTokenFilter(+ whitespaceMockTokenizer(input),+ hyphenator,+ dictionary,+ CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE,+ CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE,+ CompoundWordTokenFilterBase.DEFAULT_MAX_SUBWORD_SIZE,+ false,+ true,+ false);+ assertTokenStreamContents(tf7, new String[] {"fußball"});++ // when noOverlap is enabled fuß and ball MUST NOT be added as subwords as fußball is in the+ // dictionary+ HyphenationCompoundWordTokenFilter tf8 =+ new HyphenationCompoundWordTokenFilter(+ whitespaceMockTokenizer(input),+ hyphenator,+ dictionary,+ CompoundWordTokenFilterBase.DEFAULT_MIN_WORD_SIZE,+ CompoundWordTokenFilterBase.DEFAULT_MIN_SUBWORD_SIZE,+ CompoundWordTokenFilterBase.DEFAULT_MAX_SUBWORD_SIZE,+ false,+ false,+ true);+ assertTokenStreamContents(tf8, new String[] {"fußball"});}public static interface MockRetainAttribute extends Attribute {diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestHyphenationCompoundWordTokenFilterFactory.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestHyphenationCompoundWordTokenFilterFactory.javaindex d13e518faed..12bed128d79 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestHyphenationCompoundWordTokenFilterFactory.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/TestHyphenationCompoundWordTokenFilterFactory.java@@ -47,6 +47,33 @@ public class TestHyphenationCompoundWordTokenFilterFactory extends BaseTokenStrenew int[] {1, 1, 1, 1, 1, 1, 1, 1, 0, 0});}+ /**+ * just tests that the two no configuration options are correctly processed tests for the+ * functionality are part of {@link TestCompoundWordTokenFilter}+ */+ public void testLucene8183() throws Exception {+ Reader reader = new StringReader("basketballkurv");+ TokenStream stream = new MockTokenizer(MockTokenizer.WHITESPACE, false);+ ((Tokenizer) stream).setReader(reader);+ stream =+ tokenFilterFactory(+ "HyphenationCompoundWord",+ "hyphenator",+ "da_UTF8.xml",+ "dictionary",+ "compoundDictionary_lucene8183.txt",+ "onlyLongestMatch",+ "false",+ "noSubMatches",+ "true",+ "noOverlappingMatches",+ "false")+ .create(stream);++ assertTokenStreamContents(+ stream, new String[] {"basketballkurv", "basketball", "kurv"}, new int[] {1, 0, 0});+ }+/*** Ensure the factory works with no dictionary: using hyphenation grammar only. Also change the* min/max subword sizes from the default. When using no dictionary, it's generally necessary to@@ -68,7 +95,7 @@ public class TestHyphenationCompoundWordTokenFilterFactory extends BaseTokenStre.create(stream);assertTokenStreamContents(- stream, new String[] {"basketballkurv", "ba", "sket", "bal", "ball", "kurv"});+ stream, new String[] {"basketballkurv", "ba", "sket", "ball", "bal", "kurv"});}/** Test that bogus arguments result in exception */diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/compoundDictionary_lucene8183.txt b/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/compoundDictionary_lucene8183.txtnew file mode 100644index 00000000000..1e62f896ae3--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/compoundDictionary_lucene8183.txt@@ -0,0 +1,20 @@+# Licensed to the Apache Software Foundation (ASF) under one or more+# contributor license agreements. See the NOTICE file distributed with+# this work for additional information regarding copyright ownership.+# The ASF licenses this file to You under the Apache License, Version 2.0+# (the "License"); you may not use this file except in compliance with+# the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+# A set of words for testing LUCENE-8183+basketball+basket+ball+kurvdiff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/hyphenation-LUCENE-8183.xml b/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/hyphenation-LUCENE-8183.xmlnew file mode 100644index 00000000000..21072cc6c58--- /dev/null+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/compound/hyphenation-LUCENE-8183.xml@@ -0,0 +1,59 @@+<?xml version="1.0" encoding="utf-8"?>+<!DOCTYPE hyphenation-info SYSTEM "hyphenation.dtd">+<!--+ Licensed under the Apache License, Version 2.0 (the "License");+ you may not use this file except in compliance with the License.+ You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.+-->+<hyphenation-info>++<hyphen-char value="-"/>+<hyphen-min before="2" after="2"/>++<classes>+aA+bB+cC+dD+eE+fF+gG+hH+iI+jJ+kK+lL+mM+nN+oO+pP+rR+sS+tT+uU+vV+wW+xX+yY+zZ+ß+</classes>+ <patterns>+fuß1ball1pum2pe+fuß1ball+ </patterns>+</hyphenation-info>+<!-- fuß1+ball2+pum2+pe1+-->\ No newline at end of file