Apache Solr open-source search software
SOLR-17575: Fixed broken backwards compatibility with the legacy "langid.whitelist" config in Solr Langid (#2886)
Alexander Zagniotovcebdb2d790a6
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 840de727b00..e32f1f1ac4f 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -227,6 +227,8 @@ Bug Fixes* SOLR-16976: Remove log4j-jul jar and use slf4j bridge for JUL to prevent exception from being logged when remote JMXis enabled (Shawn Heisey, Stephen Zhou, Eric Pugh, Christine Poerschke, David Smiley)+* SOLR-17575: Fixed broken backwards compatibility with the legacy "langid.whitelist" config in Solr Langid. (Jan Høydahl, Alexander Zagniotov)+Dependency Upgrades---------------------(No changes)diff --git a/solr/modules/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java b/solr/modules/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.javaindex 21921440cae..f4f1b9cc83c 100644--- a/solr/modules/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java+++ b/solr/modules/langid/src/java/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessor.java@@ -111,8 +111,8 @@ public abstract class LanguageIdentifierUpdateProcessor extends UpdateRequestProoverwrite = params.getBool(OVERWRITE, false);langAllowlist = new HashSet<>();threshold = params.getDouble(THRESHOLD, DOCID_THRESHOLD_DEFAULT);- String legacyAllowList = params.get(LANG_WHITELIST, "");- if (legacyAllowList.length() > 0) {+ final String legacyAllowList = params.get(LANG_WHITELIST, "").trim();+ if (!legacyAllowList.isEmpty()) {// nowarn compile time string concatenationlog.warn(LANG_WHITELIST@@ -120,11 +120,10 @@ public abstract class LanguageIdentifierUpdateProcessor extends UpdateRequestPro+ LANG_ALLOWLIST+ " instead."); // nowarn}- if (params.get(LANG_ALLOWLIST, legacyAllowList).length() > 0) {- for (String lang : params.get(LANG_ALLOWLIST, "").split(",")) {- langAllowlist.add(lang);- }- }+ Arrays.stream(params.get(LANG_ALLOWLIST, legacyAllowList).split(","))+ .map(String::trim)+ .filter(lang -> !lang.isEmpty())+ .forEach(langAllowlist::add);// Mapping params (field centric)enableMapping = params.getBool(MAP_ENABLE, false);diff --git a/solr/modules/langid/src/test/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java b/solr/modules/langid/src/test/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.javaindex 4d8d398a25c..15e62d11a50 100644--- a/solr/modules/langid/src/test/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java+++ b/solr/modules/langid/src/test/org/apache/solr/update/processor/LanguageIdentifierUpdateProcessorFactoryTestCase.java@@ -18,6 +18,7 @@ package org.apache.solr.update.processor;import java.util.ArrayList;import java.util.List;+import java.util.Set;import org.apache.solr.SolrTestCaseJ4;import org.apache.solr.common.SolrInputDocument;import org.apache.solr.common.params.ModifiableSolrParams;@@ -464,6 +465,31 @@ public abstract class LanguageIdentifierUpdateProcessorFactoryTestCase extends SassertTrue(mappedIndividual.containsKey("text2_ru"));}+ @Test+ public void testAllowlist() throws Exception {+ ModifiableSolrParams parameters = new ModifiableSolrParams();+ parameters.add("langid.fl", "name,subject");+ parameters.add("langid.langField", "language_s");+ parameters.add("langid.allowlist", "no,en ,, ,sv, sv");+ liProcessor = createLangIdProcessor(parameters);++ // Make sure that empty language codes have been filtered out and others trimmed.+ assertEquals(Set.of("no", "en", "sv"), liProcessor.langAllowlist);+ }++ @Test+ public void testAllowlistBackwardsCompatabilityWithLegacyAllowlist() throws Exception {+ // The "legacy allowlist" is "langid.whitelist"+ ModifiableSolrParams parameters = new ModifiableSolrParams();+ parameters.add("langid.fl", "name,subject");+ parameters.add("langid.langField", "language_s");+ parameters.add("langid.whitelist", "no,en ,, ,sv, sv");+ liProcessor = createLangIdProcessor(parameters);++ // Make sure that empty language codes have been filtered out and others trimmed.+ assertEquals(Set.of("no", "en", "sv"), liProcessor.langAllowlist);+ }+// Various utility methodsprivate SolrInputDocument englishDoc() {