Apache Lucene open-source search software
Pattern replace keywords (#16489)
Nathan Gass972293ce92e2
diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceFilter.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceFilter.javaindex 49ac2454729..144c8ad03ea 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceFilter.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceFilter.java@@ -22,6 +22,7 @@ import java.util.regex.Pattern;import org.apache.lucene.analysis.TokenFilter;import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;+import org.apache.lucene.analysis.tokenattributes.KeywordAttribute;/*** A TokenFilter which applies a Pattern to each token in the stream, replacing match occurrences@@ -35,7 +36,9 @@ import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;public final class PatternReplaceFilter extends TokenFilter {private final String replacement;private final boolean all;+ private final boolean ignoreKeywords;private final CharTermAttribute termAtt = addAttribute(CharTermAttribute.class);+ private final KeywordAttribute keywordAtt = addAttribute(KeywordAttribute.class);private final Matcher m;/**@@ -50,9 +53,27 @@ public final class PatternReplaceFilter extends TokenFilter {* @see Matcher#quoteReplacement*/public PatternReplaceFilter(TokenStream in, Pattern p, String replacement, boolean all) {+ this(in, p, replacement, all, false);+ }++ /**+ * Constructs an instance to replace either the first, or all occurrences+ *+ * @param in the TokenStream to process+ * @param p the patterm to apply to each Token+ * @param replacement the "replacement string" to substitute, if null a blank string will be used.+ * Note that this is not the literal string that will be used, '$' and '\' have special+ * meaning.+ * @param all if true, all matches will be replaced otherwise just the first match.+ * @param ignoreKeywords if true, tokens with KeywordAttribute set to true will not be processed+ * @see Matcher#quoteReplacement+ */+ public PatternReplaceFilter(+ TokenStream in, Pattern p, String replacement, boolean all, boolean ignoreKeywords) {super(in);this.replacement = (null == replacement) ? "" : replacement;this.all = all;+ this.ignoreKeywords = ignoreKeywords;this.m = p.matcher(termAtt);}@@ -60,6 +81,10 @@ public final class PatternReplaceFilter extends TokenFilter {public boolean incrementToken() throws IOException {if (!input.incrementToken()) return false;+ if (ignoreKeywords && keywordAtt.isKeyword()) {+ return true;+ }+m.reset();if (m.find()) {// replaceAll/replaceFirst will reset() this previous find.diff --git a/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceFilterFactory.java b/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceFilterFactory.javaindex 3672209af75..1c6e4ab476e 100644--- a/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceFilterFactory.java+++ b/lucene/analysis/common/src/java/org/apache/lucene/analysis/pattern/PatternReplaceFilterFactory.java@@ -46,6 +46,7 @@ public class PatternReplaceFilterFactory extends TokenFilterFactory {final Pattern pattern;final String replacement;final boolean replaceAll;+ final boolean ignoreKeywords;/** Creates a new PatternReplaceFilterFactory */public PatternReplaceFilterFactory(Map<String, String> args) {@@ -53,6 +54,7 @@ public class PatternReplaceFilterFactory extends TokenFilterFactory {pattern = getPattern(args, "pattern");replacement = get(args, "replacement");replaceAll = "all".equals(get(args, "replace", Arrays.asList("all", "first"), "all"));+ ignoreKeywords = getBoolean(args, "ignoreKeywords", false);if (!args.isEmpty()) {throw new IllegalArgumentException("Unknown parameters: " + args);}@@ -65,6 +67,6 @@ public class PatternReplaceFilterFactory extends TokenFilterFactory {@Overridepublic PatternReplaceFilter create(TokenStream input) {- return new PatternReplaceFilter(input, pattern, replacement, replaceAll);+ return new PatternReplaceFilter(input, pattern, replacement, replaceAll, ignoreKeywords);}}diff --git a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.javaindex c45b1388900..45e18e40a53 100644--- a/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java+++ b/lucene/analysis/common/src/test/org/apache/lucene/analysis/pattern/TestPatternReplaceFilter.java@@ -22,6 +22,8 @@ import org.apache.lucene.analysis.Analyzer;import org.apache.lucene.analysis.TokenStream;import org.apache.lucene.analysis.Tokenizer;import org.apache.lucene.analysis.core.KeywordTokenizer;+import org.apache.lucene.analysis.miscellaneous.KeywordMarkerFilter;+import org.apache.lucene.analysis.tokenattributes.CharTermAttribute;import org.apache.lucene.tests.analysis.BaseTokenStreamTestCase;import org.apache.lucene.tests.analysis.MockTokenizer;@@ -108,4 +110,37 @@ public class TestPatternReplaceFilter extends BaseTokenStreamTestCase {checkOneTerm(a, "", "");a.close();}++ public void testKeywordFilter() throws Exception {+ assertAnalyzesTo(+ keywordTestAnalyzer(true), "banana kappa alpha", new String[] {"bXnXnX", "kappa", "XlphX"});++ assertAnalyzesTo(+ keywordTestAnalyzer(false),+ "banana kappa alpha",+ new String[] {"bXnXnX", "kXppX", "XlphX"});+ }++ private Analyzer keywordTestAnalyzer(boolean ignoreKeywords) throws Exception {+ return new Analyzer() {+ @Override+ protected TokenStreamComponents createComponents(String fieldName) {+ Tokenizer tokenizer = new MockTokenizer(MockTokenizer.WHITESPACE, false);+ KeywordMarkerFilter keywordFilter =+ new KeywordMarkerFilter(tokenizer) {+ private final CharTermAttribute term = addAttribute(CharTermAttribute.class);++ @Override+ public boolean isKeyword() {+ // Mark terms starting with 'k' as keywords+ return term.toString().startsWith("k");+ }+ };+ return new TokenStreamComponents(+ tokenizer,+ new PatternReplaceFilter(+ keywordFilter, Pattern.compile("a"), "X", true, ignoreKeywords));+ }+ };+ }}