Apache Lucene open-source search software
Fix SmartChinese to only deserialize data from classpath with a native array filter applied and never cache dictionaries from custom locations (#15237)
Uwe Schindler5b846cd231b1
diff --git a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/TestsAndRandomizationPlugin.java b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/TestsAndRandomizationPlugin.javaindex 9b434b4af59..a48ae6c6143 100644--- a/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/TestsAndRandomizationPlugin.java+++ b/build-tools/build-infra/src/main/java/org/apache/lucene/gradle/plugins/java/TestsAndRandomizationPlugin.java@@ -440,6 +440,16 @@ public class TestsAndRandomizationPlugin extends LuceneGradlePlugin {task.systemProperty("java.awt.headless", "true");task.systemProperty("jdk.map.althashing.threshold", "0");+ // disallow any Java serialization without a filter+ if (project.getPath().endsWith(".tests")) {+ // LUCENE-10301: for now, do not use the serialization filter for modular tests+ // (test framework is not available).+ } else if (project.getPath().startsWith(":lucene")) {+ task.systemProperty(+ "jdk.serialFilterFactory",+ "org.apache.lucene.tests.util.TestObjectInputFilterFactory");+ }+if (!Os.isFamily(Os.FAMILY_WINDOWS)) {task.systemProperty("java.security.egd", "file:/dev/./urandom");}diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex 2e17ef96051..fc2935ef731 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -187,7 +187,10 @@ Bug FixesOther----------------------(No changes)+* GITHUB#15237: Fix SmartChinese to only deserialize dictionary data from classpath with a native array+ filter applied and never cache dictionaries from custom locations. Also install a global deserialization+ filter factory that disallows any unprotected deserialization while running tests.+ (Uwe Schindler, Isaac David)======================= Lucene 10.3.0 =======================diff --git a/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/AbstractDictionary.java b/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/AbstractDictionary.javaindex 0350161ed16..c5ff36ec89b 100644--- a/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/AbstractDictionary.java+++ b/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/AbstractDictionary.java@@ -16,6 +16,9 @@*/package org.apache.lucene.analysis.cn.smart.hhmm;+import java.io.ObjectInputFilter.FilterInfo;+import java.io.ObjectInputFilter.Status;+import java.io.ObjectInputStream;import java.io.UnsupportedEncodingException;/**@@ -195,4 +198,16 @@ abstract class AbstractDictionary {return hash;}++ /** Safety filter for {@link ObjectInputStream} to only allow arrays of primitive types. */+ final Status filterObjectInputStream(FilterInfo info) {+ var cl = info.serialClass();+ if (cl == null) {+ return Status.UNDECIDED;+ }+ while (cl.isArray()) {+ cl = cl.getComponentType();+ }+ return cl.isPrimitive() ? Status.ALLOWED : Status.REJECTED;+ }}diff --git a/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BigramDictionary.java b/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BigramDictionary.javaindex 885cc144037..fa8ce3b6505 100644--- a/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BigramDictionary.java+++ b/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BigramDictionary.java@@ -77,20 +77,12 @@ class BigramDictionary extends AbstractDictionary {return singleInstance;}- private boolean loadFromObj(Path serialObj) {- try {- loadFromInputStream(Files.newInputStream(serialObj));- return true;- } catch (Exception e) {- throw new RuntimeException(e);- }- }-@SuppressForbidden(reason = "TODO: fix code to serialize its own dictionary vs. a binary blob in the codebase")private void loadFromInputStream(InputStream serialObjectInputStream)throws IOException, ClassNotFoundException {try (ObjectInputStream input = new ObjectInputStream(serialObjectInputStream)) {+ input.setObjectInputFilter(this::filterObjectInputStream);bigramHashTable = (long[]) input.readObject();frequencyTable = (int[]) input.readObject();// log.info("load bigram dict from serialization.");@@ -114,26 +106,17 @@ class BigramDictionary extends AbstractDictionary {private void load(String dictRoot) throws IOException {String bigramDictPath = dictRoot + "/bigramdict.dct";-- Path serialObj = Paths.get(dictRoot + "/bigramdict.mem");-- if (Files.exists(serialObj) && loadFromObj(serialObj)) {-- } else {- try {- bigramHashTable = new long[PRIME_BIGRAM_LENGTH];- frequencyTable = new int[PRIME_BIGRAM_LENGTH];- for (int i = 0; i < PRIME_BIGRAM_LENGTH; i++) {- // it is possible for a value to hash to 0, but the probability is extremely low- bigramHashTable[i] = 0;- frequencyTable[i] = 0;- }- loadFromFile(bigramDictPath);- } catch (IOException e) {- throw new RuntimeException(e.getMessage());- }- saveToObj(serialObj);+ bigramHashTable = new long[PRIME_BIGRAM_LENGTH];+ frequencyTable = new int[PRIME_BIGRAM_LENGTH];+ for (int i = 0; i < PRIME_BIGRAM_LENGTH; i++) {+ // it is possible for a value to hash to 0, but the probability is extremely low+ bigramHashTable[i] = 0;+ frequencyTable[i] = 0;}+ loadFromFile(bigramDictPath);++ /* Enable the following line to regenerate the serialized file: */+ // saveToObj(Paths.get(dictRoot + "/bigramdict.mem"));}/**diff --git a/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/WordDictionary.java b/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/WordDictionary.javaindex 37e721b4ee7..ad78ca6f11e 100644--- a/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/WordDictionary.java+++ b/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/WordDictionary.java@@ -80,8 +80,20 @@ class WordDictionary extends AbstractDictionary {try {singleInstance.load();} catch (IOException _) {- String wordDictRoot = AnalyzerProfile.ANALYSIS_DATA_DIR;- singleInstance.load(wordDictRoot);+ try {+ singleInstance.load();+ } catch (IOException e) {+ String dictRoot = AnalyzerProfile.ANALYSIS_DATA_DIR;+ try {+ singleInstance.load(dictRoot);+ } catch (IOException ioe) {+ RuntimeException ex = new RuntimeException(ioe);+ ex.addSuppressed(e);+ throw ex;+ }+ } catch (ClassNotFoundException e) {+ throw new RuntimeException(e);+ }} catch (ClassNotFoundException e) {throw new RuntimeException(e);}@@ -90,39 +102,29 @@ class WordDictionary extends AbstractDictionary {}/**- * Attempt to load dictionary from provided directory, first trying coredict.mem, failing back on- * coredict.dct+ * Attempt to load dictionary from provided path to coredict.dct** @param dctFileRoot path to dictionary directory*/- public void load(String dctFileRoot) {+ public void load(String dctFileRoot) throws IOException {String dctFilePath = dctFileRoot + "/coredict.dct";- Path serialObj = Paths.get(dctFileRoot + "/coredict.mem");-- if (Files.exists(serialObj) && loadFromObj(serialObj)) {-- } else {- try {- wordIndexTable = new short[PRIME_INDEX_LENGTH];- charIndexTable = new char[PRIME_INDEX_LENGTH];- for (int i = 0; i < PRIME_INDEX_LENGTH; i++) {- charIndexTable[i] = 0;- wordIndexTable[i] = -1;- }- wordItem_charArrayTable = new char[GB2312_CHAR_NUM][][];- wordItem_frequencyTable = new int[GB2312_CHAR_NUM][];- // int total =- loadMainDataFromFile(dctFilePath);- expandDelimiterData();- mergeSameWords();- sortEachItems();- // log.info("load dictionary: " + dctFilePath + " total:" + total);- } catch (IOException e) {- throw new RuntimeException(e.getMessage());- }-- saveToObj(serialObj);+ wordIndexTable = new short[PRIME_INDEX_LENGTH];+ charIndexTable = new char[PRIME_INDEX_LENGTH];+ for (int i = 0; i < PRIME_INDEX_LENGTH; i++) {+ charIndexTable[i] = 0;+ wordIndexTable[i] = -1;}+ wordItem_charArrayTable = new char[GB2312_CHAR_NUM][][];+ wordItem_frequencyTable = new int[GB2312_CHAR_NUM][];+ // int total =+ loadMainDataFromFile(dctFilePath);+ expandDelimiterData();+ mergeSameWords();+ sortEachItems();+ // log.info("load dictionary: " + dctFilePath + " total:" + total);++ /* Enable the following line to regenerate the serialized file: */+ // saveToObj(Paths.get(dctFileRoot + "/coredict.mem"));}/**@@ -135,20 +137,12 @@ class WordDictionary extends AbstractDictionary {loadFromObjectInputStream(input);}- private boolean loadFromObj(Path serialObj) {- try {- loadFromObjectInputStream(Files.newInputStream(serialObj));- return true;- } catch (Exception e) {- throw new RuntimeException(e);- }- }-@SuppressForbidden(reason = "TODO: fix code to serialize its own dictionary vs. a binary blob in the codebase")private void loadFromObjectInputStream(InputStream serialObjectInputStream)throws IOException, ClassNotFoundException {try (ObjectInputStream input = new ObjectInputStream(serialObjectInputStream)) {+ input.setObjectInputFilter(this::filterObjectInputStream);wordIndexTable = (short[]) input.readObject();charIndexTable = (char[]) input.readObject();wordItem_charArrayTable = (char[][][]) input.readObject();@@ -159,15 +153,13 @@ class WordDictionary extends AbstractDictionary {@SuppressForbidden(reason = "TODO: fix code to serialize its own dictionary vs. a binary blob in the codebase")- private void saveToObj(Path serialObj) {+ private void saveToObj(Path serialObj) throws IOException {try (ObjectOutputStream output = new ObjectOutputStream(Files.newOutputStream(serialObj))) {output.writeObject(wordIndexTable);output.writeObject(charIndexTable);output.writeObject(wordItem_charArrayTable);output.writeObject(wordItem_frequencyTable);// log.info("serialize core dict.");- } catch (Exception _) {- // log.warn(e.getMessage());}}diff --git a/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestObjectInputFilterFactory.java b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestObjectInputFilterFactory.javanew file mode 100644index 00000000000..0483735cef4--- /dev/null+++ b/lucene/test-framework/src/java/org/apache/lucene/tests/util/TestObjectInputFilterFactory.java@@ -0,0 +1,55 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.tests.util;++import java.io.ObjectInputFilter;+import java.io.ObjectInputFilter.Status;+import java.lang.StackWalker.StackFrame;+import java.util.function.BinaryOperator;++/**+ * Make sure we do not allow Java serialization without custom filters anywhere while running tests.+ * This factory returns a "deny-all" deserialization filter if none was installed by the+ * deserializer. To allow Gradle's test runner to serialize its config, all calls from Gradle are+ * allowed.+ */+public final class TestObjectInputFilterFactory implements BinaryOperator<ObjectInputFilter> {++ /** An {@link ObjectInputFilter} that rejects any deserialization. */+ public static final ObjectInputFilter DENY_ALL_FILTER = _ -> Status.REJECTED;++ @Override+ public ObjectInputFilter apply(ObjectInputFilter curr, ObjectInputFilter next) {+ // if Gradle's deserializer is on the stack, return next filter (which is obviously null):+ if (StackWalker.getInstance()+ .walk(s -> s.anyMatch(TestObjectInputFilterFactory::isGradleSerializerStackFrame))) {+ return next;+ }+ // if no filter was configured, return our filter that rejects all deserialization:+ if (curr == null && next == null) {+ return DENY_ALL_FILTER;+ }+ // if code installs its own filter return it:+ return next;+ }++ private static boolean isGradleSerializerStackFrame(StackFrame f) {+ final String methodName = f.getMethodName(), className = f.getClassName();+ return ("deserializeWorker".equals(methodName)+ && "org.gradle.process.internal.worker.messaging.WorkerConfigSerializer".equals(className));+ }+}diff --git a/lucene/test-framework/src/test/org/apache/lucene/tests/util/TestTestObjectInputFilterFactory.java b/lucene/test-framework/src/test/org/apache/lucene/tests/util/TestTestObjectInputFilterFactory.javanew file mode 100644index 00000000000..2d9da257bdb--- /dev/null+++ b/lucene/test-framework/src/test/org/apache/lucene/tests/util/TestTestObjectInputFilterFactory.java@@ -0,0 +1,79 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.lucene.tests.util;++import static java.io.ObjectInputFilter.Config.createFilter;+import static java.io.ObjectInputFilter.Config.getSerialFilterFactory;++import java.io.ByteArrayInputStream;+import java.io.ByteArrayOutputStream;+import java.io.IOException;+import java.io.InputStream;+import java.io.InvalidClassException;+import java.io.ObjectInputStream;+import java.io.ObjectOutputStream;+import java.util.HashMap;+import org.apache.lucene.util.SuppressForbidden;+import org.junit.BeforeClass;++@SuppressForbidden(reason = "tests the forbidden code")+public final class TestTestObjectInputFilterFactory extends LuceneTestCase {++ @BeforeClass+ public static void beforeClass() {+ assumeTrue(+ "Only works when TestObjectInputFilterFactory is installed in JVM",+ getSerialFilterFactory() instanceof TestObjectInputFilterFactory);+ }++ private InputStream openStream() throws IOException {+ var out = new ByteArrayOutputStream();+ try (var oout = new ObjectOutputStream(out)) {+ oout.writeObject(new HashMap<>());+ }+ return new ByteArrayInputStream(out.toByteArray());+ }++ public void testDeserializationWithoutFilter() throws Exception {+ try (var oin = new ObjectInputStream(openStream())) {+ // no filter set!+ var ex = assertThrows(InvalidClassException.class, () -> oin.readObject());+ assertTrue(ex.getMessage().contains("REJECTED"));+ }+ }++ public void testDeserializationWithAllowFilter() throws Exception {+ try (var oin = new ObjectInputStream(openStream())) {+ oin.setObjectInputFilter(createFilter("java.util.HashMap;!*"));+ oin.readObject();+ }+ }++ public void testDeserializationWithDenyFilter() throws Exception {+ try (var oin = new ObjectInputStream(openStream())) {+ oin.setObjectInputFilter(createFilter("!*"));+ var ex = assertThrows(InvalidClassException.class, () -> oin.readObject());+ assertTrue(ex.getMessage().contains("REJECTED"));+ }+ }++ public void testBadCodeJustSettingNullFilter() throws Exception {+ try (var oin = new ObjectInputStream(openStream())) {+ assertThrows(IllegalStateException.class, () -> oin.setObjectInputFilter(null));+ }+ }+}