Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

ensure original token is not set to type:SYNONYM in SGF (#4960)

Renato Haeberli51a78ca464bf
diff --git a/changelog/unreleased/preserve-original-token-type-managed-synonym-graph-filter.yml b/changelog/unreleased/preserve-original-token-type-managed-synonym-graph-filter.yml
new file mode 100644
index 00000000000..08725631593
--- /dev/null
+++ b/changelog/unreleased/preserve-original-token-type-managed-synonym-graph-filter.yml
@@ -0,0 +1,9 @@
+# See https://github.com/apache/solr/blob/main/dev-docs/changelog.adoc
+title: ManagedSynonymGraphFilterFactory no longer tags the original token as type `SYNONYM` for equivalence-group mappings (e.g. `[terma]->[terma, termb, termc]`), using Lucene's `includeOrig` flag so only the injected synonyms get that type.
+type: fixed
+authors:
+ - name: Renato Haeberli
+ nick: renatoh
+links:
+ - name: PR#4960
+ url: https://github.com/apache/solr/pull/4960
diff --git a/solr/core/src/java/org/apache/solr/rest/schema/analysis/ManagedSynonymGraphFilterFactory.java b/solr/core/src/java/org/apache/solr/rest/schema/analysis/ManagedSynonymGraphFilterFactory.java
index 6b9e6d6bedd..47998ae9607 100644
--- a/solr/core/src/java/org/apache/solr/rest/schema/analysis/ManagedSynonymGraphFilterFactory.java
+++ b/solr/core/src/java/org/apache/solr/rest/schema/analysis/ManagedSynonymGraphFilterFactory.java
@@ -365,16 +365,24 @@ public class ManagedSynonymGraphFilterFactory extends BaseManagedTokenFilterFact
boolean ignoreCase = synonymManager.getIgnoreCase();
for (CasePreservedSynonymMappings cpsm : synonymManager.synonymMappings.values()) {
for (Map.Entry<String, Set<String>> entry : cpsm.mappings.entrySet()) {
+ // Use includeOrig=true so the original token keeps its upstream type and only the
+ // injected synonyms are tagged as SYNONYM
+ boolean mapsToItself = entry.getValue().contains(entry.getKey());
+ CharsRef casedTerm =
+ analyze(
+ synonymManager.applyCaseSetting(ignoreCase, entry.getKey()),
+ new CharsRefBuilder());
for (String mapping : entry.getValue()) {
- // apply the case setting to match the behavior of the SynonymMap builder
- CharsRef casedTerm =
- analyze(
- synonymManager.applyCaseSetting(ignoreCase, entry.getKey()),
- new CharsRefBuilder());
CharsRef casedMapping =
analyze(
synonymManager.applyCaseSetting(ignoreCase, mapping), new CharsRefBuilder());
- add(casedTerm, casedMapping, false);
+ if (mapsToItself) {
+ if (!casedTerm.equals(casedMapping)) {
+ add(casedTerm, casedMapping, true);
+ }
+ } else {
+ add(casedTerm, casedMapping, false);
+ }
}
}
}
diff --git a/solr/core/src/test/org/apache/solr/rest/schema/analysis/TestManagedSynonymGraphFilterFactory.java b/solr/core/src/test/org/apache/solr/rest/schema/analysis/TestManagedSynonymGraphFilterFactory.java
index dc58da73485..e42ddf06a9a 100644
--- a/solr/core/src/test/org/apache/solr/rest/schema/analysis/TestManagedSynonymGraphFilterFactory.java
+++ b/solr/core/src/test/org/apache/solr/rest/schema/analysis/TestManagedSynonymGraphFilterFactory.java
@@ -247,6 +247,32 @@ public class TestManagedSynonymGraphFilterFactory extends RestTestBase {
assertJDelete(endpoint + "/cheerful/joyful", "/error/code==404");
}
+ @Test
+ public void testUpstreamTokenTypeIsPreserved() throws Exception {
+ String endpoint = "/schema/analysis/synonyms/englishgraph";
+
+ List<String> equivGroup = Arrays.asList("funny", "entertaining", "whimsical");
+ assertJPut(endpoint, toJSONString(equivGroup), "/responseHeader/status==0");
+
+ assertJQ(endpoint + "/funny", "/funny==['entertaining','funny','whimsical']");
+
+ restTestHarness.reload();
+
+ String analysisUrl = "/analysis/field?analysis.fieldtype=managed_graph_en&analysis.query=funny";
+ String synFilter = "//arr[contains(@name,'SynonymGraphFilter')]";
+ assertQ(
+ analysisUrl,
+ "count("
+ + synFilter
+ + "/lst[str[@name='text'][.='funny']]/str[@name='type'][.='<ALPHANUM>']) = 1",
+ "count("
+ + synFilter
+ + "/lst[str[@name='text'][.='entertaining']]/str[@name='type'][.='SYNONYM']) = 1",
+ "count("
+ + synFilter
+ + "/lst[str[@name='text'][.='whimsical']]/str[@name='type'][.='SYNONYM']) = 1");
+ }
+
/** Can we add and remove stopwords with umlauts */
@Test
public void testCanHandleDecodingAndEncodingForSynonyms() throws Exception {