Skip to content

Apache Lucene open-source search software

Remove the deflater hack introduced because of JDK-8252739 (#977)

Uwe Schindler831ae2465b70
diff --git a/gradle/validation/forbidden-apis/defaults.all.txt b/gradle/validation/forbidden-apis/defaults.all.txt
index 0265a7f2a89..3f8cbfc4fab 100644
--- a/gradle/validation/forbidden-apis/defaults.all.txt
+++ b/gradle/validation/forbidden-apis/defaults.all.txt
@@ -54,7 +54,3 @@ java.lang.Double#<init>(**)
@defaultMessage Java deserialization is unsafe when the data is untrusted. The java developer is powerless: no checks or casts help, exploitation can happen in places such as clinit or finalize!
java.io.ObjectInputStream
java.io.ObjectOutputStream
-
-@defaultMessage Don't set a dictionary on a Deflater using a method that takes an offset or ByteBuffer (JDK-8252739)
-java.util.zip.Deflater#setDictionary(byte[],int,int)
-java.util.zip.Deflater#setDictionary(java.nio.ByteBuffer)
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index c28539cbdd6..aead9611725 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -67,6 +67,8 @@ Other
* LUCENE-10493: Factor out Viterbi algorithm in Kuromoji and Nori to analysis-common. (Tomoko Uchida)
+* GITHUB#977, LUCENE-9500: Remove the deflater hack introduced because of JDK-8252739 (Uwe Schindler)
+
======================== Lucene 9.3.0 =======================
API Changes
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/BugfixDeflater_JDK8252739.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/BugfixDeflater_JDK8252739.java
deleted file mode 100644
index 8a018735f6a..00000000000
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/BugfixDeflater_JDK8252739.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.backward_codecs.lucene87;
-
-import java.util.Arrays;
-import java.util.zip.DataFormatException;
-import java.util.zip.Deflater;
-import java.util.zip.Inflater;
-import org.apache.lucene.util.BytesRefBuilder;
-import org.apache.lucene.util.SuppressForbidden;
-
-/**
- * This class is a workaround for JDK bug <a
- * href="https://bugs.openjdk.java.net/browse/JDK-8252739">JDK-8252739</a>.
- *
- * @lucene.internal
- */
-@FunctionalInterface
-interface BugfixDeflater_JDK8252739 {
-
- public static final boolean IS_BUGGY_JDK = detectBuggyJDK();
-
- /**
- * Creates a bugfix for {@link Deflater} instances, which works around JDK-8252739.
- *
- * <p>Use this whenever you intend to call {@link Deflater#setDictionary(byte[], int, int)} on a
- * {@code Deflater}.
- */
- @SuppressForbidden(reason = "Works around bug, so it must call forbidden method")
- public static BugfixDeflater_JDK8252739 createBugfix(Deflater deflater) {
- if (IS_BUGGY_JDK) {
- final BytesRefBuilder dictBytesScratch = new BytesRefBuilder();
- return (dictBytes, off, len) -> {
- if (off > 0) {
- dictBytesScratch.grow(len);
- System.arraycopy(dictBytes, off, dictBytesScratch.bytes(), 0, len);
- deflater.setDictionary(dictBytesScratch.bytes(), 0, len);
- } else {
- deflater.setDictionary(dictBytes, off, len);
- }
- };
- } else {
- return deflater::setDictionary;
- }
- }
-
- /** Call this method as a workaround */
- void setDictionary(byte[] dictBytes, int off, int len);
-
- @SuppressForbidden(reason = "Detector for the bug, so it must call buggy method")
- private static boolean detectBuggyJDK() {
- final byte[] testData = new byte[] {1, 2, 3, 4, 5, 6, 7, 8};
- final byte[] compressed = new byte[32]; // way enough space
- final Deflater deflater = new Deflater(6, true);
- int compressedSize;
- try {
- deflater.reset();
- deflater.setDictionary(testData, 4, 4);
- deflater.setInput(testData);
- deflater.finish();
- compressedSize = deflater.deflate(compressed, 0, compressed.length, Deflater.FULL_FLUSH);
- } finally {
- deflater.end();
- }
-
- // in nowrap mode we need extra 0-byte as padding, add explicit:
- compressed[compressedSize] = 0;
- compressedSize++;
-
- final Inflater inflater = new Inflater(true);
- final byte[] restored = new byte[testData.length];
- try {
- inflater.reset();
- inflater.setDictionary(testData, 4, 4);
- inflater.setInput(compressed, 0, compressedSize);
- final int restoredLength = inflater.inflate(restored);
- if (restoredLength != testData.length) {
- return true;
- }
- } catch (
- @SuppressWarnings("unused")
- DataFormatException e) {
- return true;
- } catch (
- @SuppressWarnings("unused")
- RuntimeException e) {
- return true;
- } finally {
- inflater.end();
- }
-
- if (Arrays.equals(testData, restored) == false) {
- return true;
- }
-
- // all fine
- return false;
- }
-}
diff --git a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/DeflateWithPresetDictCompressionMode.java b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/DeflateWithPresetDictCompressionMode.java
index 09f539ef0b5..e0e277e5e5d 100644
--- a/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/DeflateWithPresetDictCompressionMode.java
+++ b/lucene/backward-codecs/src/java/org/apache/lucene/backward_codecs/lucene87/DeflateWithPresetDictCompressionMode.java
@@ -161,13 +161,11 @@ public final class DeflateWithPresetDictCompressionMode extends CompressionMode
private static class DeflateWithPresetDictCompressor extends Compressor {
final Deflater compressor;
- final BugfixDeflater_JDK8252739 deflaterBugfix;
byte[] compressed;
boolean closed;
DeflateWithPresetDictCompressor(int level) {
compressor = new Deflater(level, true);
- deflaterBugfix = BugfixDeflater_JDK8252739.createBugfix(compressor);
compressed = new byte[64];
}
@@ -214,7 +212,7 @@ public final class DeflateWithPresetDictCompressionMode extends CompressionMode
// And then sub blocks
for (int start = off + dictLength; start < end; start += blockLength) {
compressor.reset();
- deflaterBugfix.setDictionary(bytes, off, dictLength);
+ compressor.setDictionary(bytes, off, dictLength);
doCompress(bytes, start, Math.min(blockLength, off + len - start), out);
}
}
diff --git a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/TestLucene87StoredFieldsFormatHighCompression.java b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/TestLucene87StoredFieldsFormatHighCompression.java
index baaedbaed8c..e844c4cd26d 100644
--- a/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/TestLucene87StoredFieldsFormatHighCompression.java
+++ b/lucene/backward-codecs/src/test/org/apache/lucene/backward_codecs/lucene87/TestLucene87StoredFieldsFormatHighCompression.java
@@ -80,8 +80,4 @@ public class TestLucene87StoredFieldsFormatHighCompression extends BaseStoredFie
new Lucene87StoredFieldsFormat(null);
});
}
-
- public void testShowJDKBugStatus() {
- System.err.println("JDK is buggy (JDK-8252739): " + BugfixDeflater_JDK8252739.IS_BUGGY_JDK);
- }
}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/BugfixDeflater_JDK8252739.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/BugfixDeflater_JDK8252739.java
deleted file mode 100644
index 865ffdbd4f6..00000000000
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/BugfixDeflater_JDK8252739.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.lucene.codecs.lucene90;
-
-import java.util.Arrays;
-import java.util.zip.DataFormatException;
-import java.util.zip.Deflater;
-import java.util.zip.Inflater;
-import org.apache.lucene.util.BytesRefBuilder;
-import org.apache.lucene.util.SuppressForbidden;
-
-/**
- * This class is a workaround for JDK bug <a
- * href="https://bugs.openjdk.java.net/browse/JDK-8252739">JDK-8252739</a>.
- *
- * @lucene.internal
- */
-@FunctionalInterface
-interface BugfixDeflater_JDK8252739 {
-
- public static final boolean IS_BUGGY_JDK = detectBuggyJDK();
-
- /**
- * Creates a bugfix for {@link Deflater} instances, which works around JDK-8252739.
- *
- * <p>Use this whenever you intend to call {@link Deflater#setDictionary(byte[], int, int)} on a
- * {@code Deflater}.
- */
- @SuppressForbidden(reason = "Works around bug, so it must call forbidden method")
- public static BugfixDeflater_JDK8252739 createBugfix(Deflater deflater) {
- if (IS_BUGGY_JDK) {
- final BytesRefBuilder dictBytesScratch = new BytesRefBuilder();
- return (dictBytes, off, len) -> {
- if (off > 0) {
- dictBytesScratch.grow(len);
- System.arraycopy(dictBytes, off, dictBytesScratch.bytes(), 0, len);
- deflater.setDictionary(dictBytesScratch.bytes(), 0, len);
- } else {
- deflater.setDictionary(dictBytes, off, len);
- }
- };
- } else {
- return deflater::setDictionary;
- }
- }
-
- /** Call this method as a workaround */
- void setDictionary(byte[] dictBytes, int off, int len);
-
- @SuppressForbidden(reason = "Detector for the bug, so it must call buggy method")
- private static boolean detectBuggyJDK() {
- final byte[] testData = new byte[] {1, 2, 3, 4, 5, 6, 7, 8};
- final byte[] compressed = new byte[32]; // way enough space
- final Deflater deflater = new Deflater(6, true);
- int compressedSize;
- try {
- deflater.reset();
- deflater.setDictionary(testData, 4, 4);
- deflater.setInput(testData);
- deflater.finish();
- compressedSize = deflater.deflate(compressed, 0, compressed.length, Deflater.FULL_FLUSH);
- } finally {
- deflater.end();
- }
-
- // in nowrap mode we need extra 0-byte as padding, add explicit:
- compressed[compressedSize] = 0;
- compressedSize++;
-
- final Inflater inflater = new Inflater(true);
- final byte[] restored = new byte[testData.length];
- try {
- inflater.reset();
- inflater.setDictionary(testData, 4, 4);
- inflater.setInput(compressed, 0, compressedSize);
- final int restoredLength = inflater.inflate(restored);
- if (restoredLength != testData.length) {
- return true;
- }
- } catch (
- @SuppressWarnings("unused")
- DataFormatException e) {
- return true;
- } catch (
- @SuppressWarnings("unused")
- RuntimeException e) {
- return true;
- } finally {
- inflater.end();
- }
-
- if (Arrays.equals(testData, restored) == false) {
- return true;
- }
-
- // all fine
- return false;
- }
-}
diff --git a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/DeflateWithPresetDictCompressionMode.java b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/DeflateWithPresetDictCompressionMode.java
index 19558ef0908..26ac3868213 100644
--- a/lucene/core/src/java/org/apache/lucene/codecs/lucene90/DeflateWithPresetDictCompressionMode.java
+++ b/lucene/core/src/java/org/apache/lucene/codecs/lucene90/DeflateWithPresetDictCompressionMode.java
@@ -161,13 +161,11 @@ public final class DeflateWithPresetDictCompressionMode extends CompressionMode
private static class DeflateWithPresetDictCompressor extends Compressor {
final Deflater compressor;
- final BugfixDeflater_JDK8252739 deflaterBugfix;
byte[] compressed;
boolean closed;
DeflateWithPresetDictCompressor(int level) {
compressor = new Deflater(level, true);
- deflaterBugfix = BugfixDeflater_JDK8252739.createBugfix(compressor);
compressed = new byte[64];
}
@@ -214,7 +212,7 @@ public final class DeflateWithPresetDictCompressionMode extends CompressionMode
// And then sub blocks
for (int start = off + dictLength; start < end; start += blockLength) {
compressor.reset();
- deflaterBugfix.setDictionary(bytes, off, dictLength);
+ compressor.setDictionary(bytes, off, dictLength);
doCompress(bytes, start, Math.min(blockLength, off + len - start), out);
}
}
diff --git a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90StoredFieldsFormatHighCompression.java b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90StoredFieldsFormatHighCompression.java
index f36820ef1a6..941523a4202 100644
--- a/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90StoredFieldsFormatHighCompression.java
+++ b/lucene/core/src/test/org/apache/lucene/codecs/lucene90/TestLucene90StoredFieldsFormatHighCompression.java
@@ -79,8 +79,4 @@ public class TestLucene90StoredFieldsFormatHighCompression extends BaseStoredFie
new Lucene90StoredFieldsFormat(null);
});
}
-
- public void testShowJDKBugStatus() {
- System.err.println("JDK is buggy (JDK-8252739): " + BugfixDeflater_JDK8252739.IS_BUGGY_JDK);
- }
}