Apache Solr open-source search software
ensure original token is not set to type:SYNONYM in SGF (#4960)
Renato Haeberli51a78ca464bf
diff --git a/changelog/unreleased/preserve-original-token-type-managed-synonym-graph-filter.yml b/changelog/unreleased/preserve-original-token-type-managed-synonym-graph-filter.ymlnew file mode 100644index 00000000000..08725631593--- /dev/null+++ b/changelog/unreleased/preserve-original-token-type-managed-synonym-graph-filter.yml@@ -0,0 +1,9 @@+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc+title: ManagedSynonymGraphFilterFactory no longer tags the original token as type `SYNONYM` for equivalence-group mappings (e.g. `[terma]->[terma, termb, termc]`), using Lucene's `includeOrig` flag so only the injected synonyms get that type.+type: fixed+authors:+ - name: Renato Haeberli+ nick: renatoh+links:+ - name: PR#4960+ url: https://github.com/apache/solr/pull/4960diff --git a/solr/core/src/java/org/apache/solr/rest/schema/analysis/ManagedSynonymGraphFilterFactory.java b/solr/core/src/java/org/apache/solr/rest/schema/analysis/ManagedSynonymGraphFilterFactory.javaindex 6b9e6d6bedd..47998ae9607 100644--- a/solr/core/src/java/org/apache/solr/rest/schema/analysis/ManagedSynonymGraphFilterFactory.java+++ b/solr/core/src/java/org/apache/solr/rest/schema/analysis/ManagedSynonymGraphFilterFactory.java@@ -365,16 +365,24 @@ public class ManagedSynonymGraphFilterFactory extends BaseManagedTokenFilterFactboolean ignoreCase = synonymManager.getIgnoreCase();for (CasePreservedSynonymMappings cpsm : synonymManager.synonymMappings.values()) {for (Map.Entry<String, Set<String>> entry : cpsm.mappings.entrySet()) {+ // Use includeOrig=true so the original token keeps its upstream type and only the+ // injected synonyms are tagged as SYNONYM+ boolean mapsToItself = entry.getValue().contains(entry.getKey());+ CharsRef casedTerm =+ analyze(+ synonymManager.applyCaseSetting(ignoreCase, entry.getKey()),+ new CharsRefBuilder());for (String mapping : entry.getValue()) {- // apply the case setting to match the behavior of the SynonymMap builder- CharsRef casedTerm =- analyze(- synonymManager.applyCaseSetting(ignoreCase, entry.getKey()),- new CharsRefBuilder());CharsRef casedMapping =analyze(synonymManager.applyCaseSetting(ignoreCase, mapping), new CharsRefBuilder());- add(casedTerm, casedMapping, false);+ if (mapsToItself) {+ if (!casedTerm.equals(casedMapping)) {+ add(casedTerm, casedMapping, true);+ }+ } else {+ add(casedTerm, casedMapping, false);+ }}}}diff --git a/solr/core/src/test/org/apache/solr/rest/schema/analysis/TestManagedSynonymGraphFilterFactory.java b/solr/core/src/test/org/apache/solr/rest/schema/analysis/TestManagedSynonymGraphFilterFactory.javaindex dc58da73485..e42ddf06a9a 100644--- a/solr/core/src/test/org/apache/solr/rest/schema/analysis/TestManagedSynonymGraphFilterFactory.java+++ b/solr/core/src/test/org/apache/solr/rest/schema/analysis/TestManagedSynonymGraphFilterFactory.java@@ -247,6 +247,32 @@ public class TestManagedSynonymGraphFilterFactory extends RestTestBase {assertJDelete(endpoint + "/cheerful/joyful", "/error/code==404");}+ @Test+ public void testUpstreamTokenTypeIsPreserved() throws Exception {+ String endpoint = "/schema/analysis/synonyms/englishgraph";++ List<String> equivGroup = Arrays.asList("funny", "entertaining", "whimsical");+ assertJPut(endpoint, toJSONString(equivGroup), "/responseHeader/status==0");++ assertJQ(endpoint + "/funny", "/funny==['entertaining','funny','whimsical']");++ restTestHarness.reload();++ String analysisUrl = "/analysis/field?analysis.fieldtype=managed_graph_en&analysis.query=funny";+ String synFilter = "//arr[contains(@name,'SynonymGraphFilter')]";+ assertQ(+ analysisUrl,+ "count("+ + synFilter+ + "/lst[str[@name='text'][.='funny']]/str[@name='type'][.='<ALPHANUM>']) = 1",+ "count("+ + synFilter+ + "/lst[str[@name='text'][.='entertaining']]/str[@name='type'][.='SYNONYM']) = 1",+ "count("+ + synFilter+ + "/lst[str[@name='text'][.='whimsical']]/str[@name='type'][.='SYNONYM']) = 1");+ }+/** Can we add and remove stopwords with umlauts */@Testpublic void testCanHandleDecodingAndEncodingForSynonyms() throws Exception {