Apache Lucene open-source search software
Remove the deflater hack introduced because of JDK-8252739 (#977)
Uwe Schindler831ae2465b70
diff --git a/gradle/validation/forbidden-apis/defaults.all.txt b/gradle/validation/forbidden-apis/defaults.all.txtindex 0265a7f2a89..3f8cbfc4fab 100644--- a/gradle/validation/forbidden-apis/defaults.all.txt+++ b/gradle/validation/forbidden-apis/defaults.all.txt@@ -54,7 +54,3 @@ java.lang.Double#<init>(**)@defaultMessage Java deserialization is unsafe when the data is untrusted. The java developer is powerless: no checks or casts help, exploitation can happen in places such as clinit or finalize!java.io.ObjectInputStreamjava.io.ObjectOutputStream--@defaultMessage Don't set a dictionary on a Deflater using a method that takes an offset or ByteBuffer (JDK-8252739)-java.util.zip.Deflater#setDictionary(byte[],int,int)-java.util.zip.Deflater#setDictionary(java.nio.ByteBuffer)diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex c28539cbdd6..aead9611725 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -67,6 +67,8 @@ Other* LUCENE-10493: Factor out Viterbi algorithm in Kuromoji and Nori to analysis-common. (Tomoko Uchida)+* GITHUB#977, LUCENE-9500: Remove the deflater hack introduced because of JDK-8252739 (Uwe Schindler)+======================== Lucene 9.3.0 =======================API Changesdiff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/BugfixDeflater_JDK8252739.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/BugfixDeflater_JDK8252739.javadeleted file mode 100644index 8a018735f6a..00000000000--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/BugfixDeflater_JDK8252739.java+++ /dev/null@@ -1,113 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.backward_codecs.lucene87;--import java.util.Arrays;-import java.util.zip.DataFormatException;-import java.util.zip.Deflater;-import java.util.zip.Inflater;-import org.apache.lucene.util.BytesRefBuilder;-import org.apache.lucene.util.SuppressForbidden;--/**- * This class is a workaround for JDK bug <a- * href="https://bugs.openjdk.java.net/browse/JDK-8252739">JDK-8252739</a>.- *- * @lucene.internal- */-@FunctionalInterface-interface BugfixDeflater_JDK8252739 {-- public static final boolean IS_BUGGY_JDK = detectBuggyJDK();-- /**- * Creates a bugfix for {@link Deflater} instances, which works around JDK-8252739.- *- * <p>Use this whenever you intend to call {@link Deflater#setDictionary(byte[], int, int)} on a- * {@code Deflater}.- */- @SuppressForbidden(reason = "Works around bug, so it must call forbidden method")- public static BugfixDeflater_JDK8252739 createBugfix(Deflater deflater) {- if (IS_BUGGY_JDK) {- final BytesRefBuilder dictBytesScratch = new BytesRefBuilder();- return (dictBytes, off, len) -> {- if (off > 0) {- dictBytesScratch.grow(len);- System.arraycopy(dictBytes, off, dictBytesScratch.bytes(), 0, len);- deflater.setDictionary(dictBytesScratch.bytes(), 0, len);- } else {- deflater.setDictionary(dictBytes, off, len);- }- };- } else {- return deflater::setDictionary;- }- }-- /** Call this method as a workaround */- void setDictionary(byte[] dictBytes, int off, int len);-- @SuppressForbidden(reason = "Detector for the bug, so it must call buggy method")- private static boolean detectBuggyJDK() {- final byte[] testData = new byte[] {1, 2, 3, 4, 5, 6, 7, 8};- final byte[] compressed = new byte[32]; // way enough space- final Deflater deflater = new Deflater(6, true);- int compressedSize;- try {- deflater.reset();- deflater.setDictionary(testData, 4, 4);- deflater.setInput(testData);- deflater.finish();- compressedSize = deflater.deflate(compressed, 0, compressed.length, Deflater.FULL_FLUSH);- } finally {- deflater.end();- }-- // in nowrap mode we need extra 0-byte as padding, add explicit:- compressed[compressedSize] = 0;- compressedSize++;-- final Inflater inflater = new Inflater(true);- final byte[] restored = new byte[testData.length];- try {- inflater.reset();- inflater.setDictionary(testData, 4, 4);- inflater.setInput(compressed, 0, compressedSize);- final int restoredLength = inflater.inflate(restored);- if (restoredLength != testData.length) {- return true;- }- } catch (- @SuppressWarnings("unused")- DataFormatException e) {- return true;- } catch (- @SuppressWarnings("unused")- RuntimeException e) {- return true;- } finally {- inflater.end();- }-- if (Arrays.equals(testData, restored) == false) {- return true;- }-- // all fine- return false;- }-}diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/DeflateWithPresetDictCompressionMode.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/DeflateWithPresetDictCompressionMode.javaindex 09f539ef0b5..e0e277e5e5d 100644--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/DeflateWithPresetDictCompressionMode.java+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/DeflateWithPresetDictCompressionMode.java@@ -161,13 +161,11 @@ public final class DeflateWithPresetDictCompressionMode extends CompressionModeprivate static class DeflateWithPresetDictCompressor extends Compressor {final Deflater compressor;- final BugfixDeflater_JDK8252739 deflaterBugfix;byte[] compressed;boolean closed;DeflateWithPresetDictCompressor(int level) {compressor = new Deflater(level, true);- deflaterBugfix = BugfixDeflater_JDK8252739.createBugfix(compressor);compressed = new byte[64];}@@ -214,7 +212,7 @@ public final class DeflateWithPresetDictCompressionMode extends CompressionMode// And then sub blocksfor (int start = off + dictLength; start < end; start += blockLength) {compressor.reset();- deflaterBugfix.setDictionary(bytes, off, dictLength);+ compressor.setDictionary(bytes, off, dictLength);doCompress(bytes, start, Math.min(blockLength, off + len - start), out);}}diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/TestLucene87StoredFieldsFormatHighCompression.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/TestLucene87StoredFieldsFormatHighCompression.javaindex baaedbaed8c..e844c4cd26d 100644--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/TestLucene87StoredFieldsFormatHighCompression.java+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/TestLucene87StoredFieldsFormatHighCompression.java@@ -80,8 +80,4 @@ public class TestLucene87StoredFieldsFormatHighCompression extends BaseStoredFienew Lucene87StoredFieldsFormat(null);});}-- public void testShowJDKBugStatus() {- System.err.println("JDK is buggy (JDK-8252739): " + BugfixDeflater_JDK8252739.IS_BUGGY_JDK);- }}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/BugfixDeflater_JDK8252739.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/BugfixDeflater_JDK8252739.javadeleted file mode 100644index 865ffdbd4f6..00000000000--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/BugfixDeflater_JDK8252739.java+++ /dev/null@@ -1,113 +0,0 @@-/*- * Licensed to the Apache Software Foundation (ASF) under one or more- * contributor license agreements. See the NOTICE file distributed with- * this work for additional information regarding copyright ownership.- * The ASF licenses this file to You under the Apache License, Version 2.0- * (the "License"); you may not use this file except in compliance with- * the License. You may obtain a copy of the License at- *- * http://www.apache.org/licenses/LICENSE-2.0- *- * Unless required by applicable law or agreed to in writing, software- * distributed under the License is distributed on an "AS IS" BASIS,- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.- * See the License for the specific language governing permissions and- * limitations under the License.- */-package org.apache.lucene.codecs.lucene90;--import java.util.Arrays;-import java.util.zip.DataFormatException;-import java.util.zip.Deflater;-import java.util.zip.Inflater;-import org.apache.lucene.util.BytesRefBuilder;-import org.apache.lucene.util.SuppressForbidden;--/**- * This class is a workaround for JDK bug <a- * href="https://bugs.openjdk.java.net/browse/JDK-8252739">JDK-8252739</a>.- *- * @lucene.internal- */-@FunctionalInterface-interface BugfixDeflater_JDK8252739 {-- public static final boolean IS_BUGGY_JDK = detectBuggyJDK();-- /**- * Creates a bugfix for {@link Deflater} instances, which works around JDK-8252739.- *- * <p>Use this whenever you intend to call {@link Deflater#setDictionary(byte[], int, int)} on a- * {@code Deflater}.- */- @SuppressForbidden(reason = "Works around bug, so it must call forbidden method")- public static BugfixDeflater_JDK8252739 createBugfix(Deflater deflater) {- if (IS_BUGGY_JDK) {- final BytesRefBuilder dictBytesScratch = new BytesRefBuilder();- return (dictBytes, off, len) -> {- if (off > 0) {- dictBytesScratch.grow(len);- System.arraycopy(dictBytes, off, dictBytesScratch.bytes(), 0, len);- deflater.setDictionary(dictBytesScratch.bytes(), 0, len);- } else {- deflater.setDictionary(dictBytes, off, len);- }- };- } else {- return deflater::setDictionary;- }- }-- /** Call this method as a workaround */- void setDictionary(byte[] dictBytes, int off, int len);-- @SuppressForbidden(reason = "Detector for the bug, so it must call buggy method")- private static boolean detectBuggyJDK() {- final byte[] testData = new byte[] {1, 2, 3, 4, 5, 6, 7, 8};- final byte[] compressed = new byte[32]; // way enough space- final Deflater deflater = new Deflater(6, true);- int compressedSize;- try {- deflater.reset();- deflater.setDictionary(testData, 4, 4);- deflater.setInput(testData);- deflater.finish();- compressedSize = deflater.deflate(compressed, 0, compressed.length, Deflater.FULL_FLUSH);- } finally {- deflater.end();- }-- // in nowrap mode we need extra 0-byte as padding, add explicit:- compressed[compressedSize] = 0;- compressedSize++;-- final Inflater inflater = new Inflater(true);- final byte[] restored = new byte[testData.length];- try {- inflater.reset();- inflater.setDictionary(testData, 4, 4);- inflater.setInput(compressed, 0, compressedSize);- final int restoredLength = inflater.inflate(restored);- if (restoredLength != testData.length) {- return true;- }- } catch (- @SuppressWarnings("unused")- DataFormatException e) {- return true;- } catch (- @SuppressWarnings("unused")- RuntimeException e) {- return true;- } finally {- inflater.end();- }-- if (Arrays.equals(testData, restored) == false) {- return true;- }-- // all fine- return false;- }-}diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/DeflateWithPresetDictCompressionMode.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/DeflateWithPresetDictCompressionMode.javaindex 19558ef0908..26ac3868213 100644--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/DeflateWithPresetDictCompressionMode.java+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/DeflateWithPresetDictCompressionMode.java@@ -161,13 +161,11 @@ public final class DeflateWithPresetDictCompressionMode extends CompressionModeprivate static class DeflateWithPresetDictCompressor extends Compressor {final Deflater compressor;- final BugfixDeflater_JDK8252739 deflaterBugfix;byte[] compressed;boolean closed;DeflateWithPresetDictCompressor(int level) {compressor = new Deflater(level, true);- deflaterBugfix = BugfixDeflater_JDK8252739.createBugfix(compressor);compressed = new byte[64];}@@ -214,7 +212,7 @@ public final class DeflateWithPresetDictCompressionMode extends CompressionMode// And then sub blocksfor (int start = off + dictLength; start < end; start += blockLength) {compressor.reset();- deflaterBugfix.setDictionary(bytes, off, dictLength);+ compressor.setDictionary(bytes, off, dictLength);doCompress(bytes, start, Math.min(blockLength, off + len - start), out);}}diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90StoredFieldsFormatHighCompression.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90StoredFieldsFormatHighCompression.javaindex f36820ef1a6..941523a4202 100644--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90StoredFieldsFormatHighCompression.java+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90StoredFieldsFormatHighCompression.java@@ -79,8 +79,4 @@ public class TestLucene90StoredFieldsFormatHighCompression extends BaseStoredFienew Lucene90StoredFieldsFormat(null);});}-- public void testShowJDKBugStatus() {- System.err.println("JDK is buggy (JDK-8252739): " + BugfixDeflater_JDK8252739.IS_BUGGY_JDK);- }}