Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

LUCENE-9574 Add DropIfFlaggedFilterFactory (#1979)

Gus Heckab5671d367eb
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index 4e4da5d2e36..fa924b2c6f5 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -190,6 +190,13 @@ Other
* LUCENE-9544: add regenerate gradle script for nori dictionary (Namgyu Kim)
+======================= Lucene 8.8.0 =======================
+
+New Features
+---------------------
+
+* LUCENE-9574 A token filter to drop tokens that match all specified flags.
+
======================= Lucene 8.7.0 =======================
API Changes
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/DropIfFlaggedFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/DropIfFlaggedFilter.java
new file mode 100644
index 00000000000..940c23fb161
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/DropIfFlaggedFilter.java
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.analysis.miscellaneous;
+
+import org.apache.lucene.analysis.FilteringTokenFilter;
+import org.apache.lucene.analysis.TokenStream;
+import org.apache.lucene.analysis.tokenattributes.FlagsAttribute;
+
+/**
+ * Allows Tokens with a given combination of flags to be dropped. If all flags specified are present
+ * the token is dropped, otherwise it is retained.
+ *
+ * @see DropIfFlaggedFilterFactory
+ * @since 8.8.0
+ */
+public final class DropIfFlaggedFilter extends FilteringTokenFilter {
+ private final FlagsAttribute flagsAtt = addAttribute(FlagsAttribute.class);
+ private final int dropFlags;
+
+ /**
+ * Construct a token stream filtering the given input.
+ *
+ * @param input the source stream
+ * @param dropFlags a combination of flags that indicates that the token should be dropped.
+ */
+ public DropIfFlaggedFilter(TokenStream input, int dropFlags) {
+ super(input);
+ this.dropFlags = dropFlags;
+ }
+
+ @Override
+ protected boolean accept() {
+ return (flagsAtt.getFlags() & dropFlags) != dropFlags;
+ }
+
+}
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/DropIfFlaggedFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/DropIfFlaggedFilterFactory.java
new file mode 100644
index 00000000000..3147912ce67
--- /dev/null
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/miscellaneous/DropIfFlaggedFilterFactory.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.analysis.miscellaneous;
+
+import java.util.Map;
+
+import org.apache.lucene.analysis.TokenFilterFactory;
+import org.apache.lucene.analysis.TokenStream;
+
+/**
+ * Provides a filter that will drop tokens matching a set of flags. This might be used if you had
+ * both custom filters that identify tokens to be removed, but need to run before other filters that
+ * want to see the token that will eventually be dropped. Alternately you might have separate flag setting
+ * filters and then remove tokens that match a particular combination of those filters.<br>
+ * <br>
+ * In Solr this might be configured such as
+ * <pre class="prettyprint">
+ * &lt;analyzer type="index"&gt;
+ * &lt;tokenizer class="solr.WhitespaceTokenizerFactory"/&gt;
+ * &lt;-- other filters --&gt;
+ * &lt;filter class="solr.DropIfFlaggedFilterFactory" dropFlags="9"/&gt;
+ * &lt;/analyzer&gt;
+ * </pre>
+ * The above would drop any token that had the first and fourth bit set.
+ *
+ * @since 8.8.0
+ * @lucene.spi {@value #NAME}
+ */
+public final class DropIfFlaggedFilterFactory extends TokenFilterFactory {
+ /**
+ * SPI name
+ */
+ public static final String NAME = "dropIfFlagged";
+
+ private final int dropFlags;
+
+ /**
+ * Initialize this factory via a set of key-value pairs.
+ */
+ public DropIfFlaggedFilterFactory(Map<String, String> args) {
+ super(args);
+ dropFlags = getInt(args,"dropFlags", 2);
+
+ }
+
+ /** Default ctor for compatibility with SPI */
+ public DropIfFlaggedFilterFactory() {
+ throw defaultCtorException();
+ }
+
+
+ @Override
+ public TokenStream create(TokenStream input) {
+ return new DropIfFlaggedFilter(input, dropFlags);
+ }
+}
diff --git a/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.TokenFilterFactory b/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.TokenFilterFactory
index 1cc77c2dae1..be82bf2967b 100644
--- a/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.TokenFilterFactory
+++ b/lucene/analysis/common/src/resources/META-INF/services/org.apache.lucene.analysis.TokenFilterFactory
@@ -70,6 +70,7 @@ org.apache.lucene.analysis.miscellaneous.CodepointCountFilterFactory
org.apache.lucene.analysis.miscellaneous.ConcatenateGraphFilterFactory
org.apache.lucene.analysis.miscellaneous.DateRecognizerFilterFactory
org.apache.lucene.analysis.miscellaneous.DelimitedTermFrequencyTokenFilterFactory
+org.apache.lucene.analysis.miscellaneous.DropIfFlaggedFilterFactory
org.apache.lucene.analysis.miscellaneous.FingerprintFilterFactory
org.apache.lucene.analysis.miscellaneous.FixBrokenOffsetsFilterFactory
org.apache.lucene.analysis.miscellaneous.HyphenatedWordsFilterFactory
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestDropIfFlaggedFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestDropIfFlaggedFilter.java
new file mode 100644
index 00000000000..be80ec670e1
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestDropIfFlaggedFilter.java
@@ -0,0 +1,73 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.lucene.analysis.miscellaneous;
+
+import org.apache.lucene.analysis.BaseTokenStreamTestCase;
+import org.apache.lucene.analysis.CannedTokenStream;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenStream;
+
+/**
+ * Test that this filter removes tokens that match a particular set of flags.
+ */
+public class TestDropIfFlaggedFilter extends BaseTokenStreamTestCase {
+
+ /**
+ * Test the straight forward cases. When all flags match the token should be dropped
+ */
+ public void testDropped() throws Exception {
+
+ Token token = new Token("foo", 0, 2);
+ Token token2 = new Token("bar", 4, 6);
+ Token token3 = new Token("baz", 8, 10);
+ Token token4 = new Token("bam", 12, 14);
+
+ token.setFlags(0); // 000 no flags match
+ token2.setFlags(1);// 001 one flag matches
+ token3.setFlags(2);// 010 no flags match
+ token4.setFlags(7);// 111 both flags match (drop)
+
+ TokenStream ts = new CannedTokenStream(token, token2, token3, token4);
+ ts = new DropIfFlaggedFilter(ts, 5); // 101
+
+ assertTokenStreamContents(ts, new String[]{
+ "foo", "bar", "baz"}, new int[]{0, 4, 8}, new int[]{2, 6, 10}, new int[]{1, 1, 1});
+ }
+
+ /**
+ * Test where the first and last token are dropped.
+ */
+ public void testDroppedFirst() throws Exception {
+
+ Token token = new Token("foo", 0, 2);
+ Token token2 = new Token("bar", 4, 6);
+ Token token3 = new Token("baz", 8, 10);
+ Token token4 = new Token("bam", 12, 14);
+
+ token.setFlags(4); // 100 flag matches (drop)
+ token2.setFlags(1);// 001 no flags match
+ token3.setFlags(2);// 010 no flags match
+ token4.setFlags(7);// 111 flag matches (drop)
+
+ TokenStream ts = new CannedTokenStream(token, token2, token3, token4);
+ ts = new DropIfFlaggedFilter(ts, 4) ;
+
+ assertTokenStreamContents(ts, new String[]{
+ "bar", "baz"}, new int[]{ 4, 8}, new int[]{6, 10}, new int[]{2, 1});
+ }
+
+}
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestDropIfFlaggedFilterFactory.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestDropIfFlaggedFilterFactory.java
new file mode 100644
index 00000000000..2f5de391749
--- /dev/null
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/miscellaneous/TestDropIfFlaggedFilterFactory.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.lucene.analysis.miscellaneous;
+
+import org.apache.lucene.analysis.BaseTokenStreamFactoryTestCase;
+import org.apache.lucene.analysis.CannedTokenStream;
+import org.apache.lucene.analysis.Token;
+import org.apache.lucene.analysis.TokenFilterFactory;
+import org.apache.lucene.analysis.TokenStream;
+
+/**
+ * This test just ensures the factory works, detailed tests in {@link TestDropIfFlaggedFilter}
+ */
+public class TestDropIfFlaggedFilterFactory extends BaseTokenStreamFactoryTestCase {
+
+ private static final Token[] TOKENS = { token("foo",1,0,2), token("bar",3, 4,6) };
+
+ public void testFactory() throws Exception {
+ TokenStream stream = new CannedTokenStream(TOKENS);
+ TokenFilterFactory tokenFilterFactory = tokenFilterFactory("dropIfFlagged", "flags", "2");
+ stream = tokenFilterFactory.create(stream);
+ assertTokenStreamContents(stream, new String[] { "foo" }, null, null, new String[] { "word",}, new int[] { 1 });
+ }
+
+ private static Token token(String term, int flags, int soff, int eoff) {
+ Token token = new Token();
+ token.setEmpty();
+ token.append(term);
+ token.setFlags(flags);
+ token.setOffset(soff,eoff);
+ return token;
+ }
+}