Skip to content

Apache Lucene open-source search software

Pattern replace keywords (#16489)

Nathan Gass972293ce92e2
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceFilter.java
index 49ac2454729..144c8ad03ea 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceFilter.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceFilter.java
@@ -22,6 +22,7 @@ import java.util.regex.Pattern;
import org.apache.lucene.analysis.TokenFilter;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
+import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;
/**
* A TokenFilter which applies a Pattern to each token in the stream, replacing match occurrences
@@ -35,7 +36,9 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
public final class PatternReplaceFilter extends TokenFilter {
private final String replacement;
private final boolean all;
+ private final boolean ignoreKeywords;
private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);
+ private final KeywordAttribute keywordAtt = addAttribute(KeywordAttribute.class);
private final Matcher m;
/**
@@ -50,9 +53,27 @@ public final class PatternReplaceFilter extends TokenFilter {
* @see Matcher#quoteReplacement
*/
public PatternReplaceFilter(TokenStream in, Pattern p, String replacement, boolean all) {
+ this(in, p, replacement, all, false);
+ }
+
+ /**
+ * Constructs an instance to replace either the first, or all occurrences
+ *
+ * @param in the TokenStream to process
+ * @param p the patterm to apply to each Token
+ * @param replacement the "replacement string" to substitute, if null a blank string will be used.
+ * Note that this is not the literal string that will be used, '$' and '\' have special
+ * meaning.
+ * @param all if true, all matches will be replaced otherwise just the first match.
+ * @param ignoreKeywords if true, tokens with KeywordAttribute set to true will not be processed
+ * @see Matcher#quoteReplacement
+ */
+ public PatternReplaceFilter(
+ TokenStream in, Pattern p, String replacement, boolean all, boolean ignoreKeywords) {
super(in);
this.replacement = (null == replacement) ? "" : replacement;
this.all = all;
+ this.ignoreKeywords = ignoreKeywords;
this.m = p.matcher(termAtt);
}
@@ -60,6 +81,10 @@ public final class PatternReplaceFilter extends TokenFilter {
public boolean incrementToken() throws IOException {
if (!input.incrementToken()) return false;
+ if (ignoreKeywords && keywordAtt.isKeyword()) {
+ return true;
+ }
+
m.reset();
if (m.find()) {
// replaceAll/replaceFirst will reset() this previous find.
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceFilterFactory.java
index 3672209af75..1c6e4ab476e 100644
--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceFilterFactory.java
+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceFilterFactory.java
@@ -46,6 +46,7 @@ public class PatternReplaceFilterFactory extends TokenFilterFactory {
final Pattern pattern;
final String replacement;
final boolean replaceAll;
+ final boolean ignoreKeywords;
/** Creates a new PatternReplaceFilterFactory */
public PatternReplaceFilterFactory(Map<String, String> args) {
@@ -53,6 +54,7 @@ public class PatternReplaceFilterFactory extends TokenFilterFactory {
pattern = getPattern(args, "pattern");
replacement = get(args, "replacement");
replaceAll = "all".equals(get(args, "replace", Arrays.asList("all", "first"), "all"));
+ ignoreKeywords = getBoolean(args, "ignoreKeywords", false);
if (!args.isEmpty()) {
throw new IllegalArgumentException("Unknown parameters: " + args);
}
@@ -65,6 +67,6 @@ public class PatternReplaceFilterFactory extends TokenFilterFactory {
@Override
public PatternReplaceFilter create(TokenStream input) {
- return new PatternReplaceFilter(input, pattern, replacement, replaceAll);
+ return new PatternReplaceFilter(input, pattern, replacement, replaceAll, ignoreKeywords);
}
}
diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java
index c45b1388900..45e18e40a53 100644
--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java
+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java
@@ -22,6 +22,8 @@ import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.Tokenizer;
import org.apache.lucene.analysis.core.KeywordTokenizer;
+import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;
+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;
import org.apache.lucene.tests.analysis.BaseTokenStreamTestCase;
import org.apache.lucene.tests.analysis.MockTokenizer;
@@ -108,4 +110,37 @@ public class TestPatternReplaceFilter extends BaseTokenStreamTestCase {
checkOneTerm(a, "", "");
a.close();
}
+
+ public void testKeywordFilter() throws Exception {
+ assertAnalyzesTo(
+ keywordTestAnalyzer(true), "banana kappa alpha", new String[] {"bXnXnX", "kappa", "XlphX"});
+
+ assertAnalyzesTo(
+ keywordTestAnalyzer(false),
+ "banana kappa alpha",
+ new String[] {"bXnXnX", "kXppX", "XlphX"});
+ }
+
+ private Analyzer keywordTestAnalyzer(boolean ignoreKeywords) throws Exception {
+ return new Analyzer() {
+ @Override
+ protected TokenStreamComponents createComponents(String fieldName) {
+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);
+ KeywordMarkerFilter keywordFilter =
+ new KeywordMarkerFilter(tokenizer) {
+ private final CharTermAttribute term = addAttribute(CharTermAttribute.class);
+
+ @Override
+ public boolean isKeyword() {
+ // Mark terms starting with 'k' as keywords
+ return term.toString().startsWith("k");
+ }
+ };
+ return new TokenStreamComponents(
+ tokenizer,
+ new PatternReplaceFilter(
+ keywordFilter, Pattern.compile("a"), "X", true, ignoreKeywords));
+ }
+ };
+ }
}