Skip to content

Apache Lucene open-source search software

Fix resource leak in loadMainDataFromFile (#14727)

xcx7759fde64c8e
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txt
index dd69e1a2ebf..4ef5d902082 100644
--- a/lucene/CHANGES.txt
+++ b/lucene/CHANGES.txt
@@ -148,6 +148,8 @@ Bug Fixes
* GITHUB#14161: PointInSetQuery's constructor now throws IllegalArgumentException
instead of UnsupportedOperationException when values are out of order. (Shubham Sharma)
+* GITHUB#14727: Fix several resource leaks. (Xu Chengxin)
+
Build
---------------------
* Upgrade forbiddenapis to version 3.9. (Uwe Schindler)
diff --git a/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BigramDictionary.java b/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BigramDictionary.java
index b4ef6af2d69..885cc144037 100644
--- a/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BigramDictionary.java
+++ b/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BigramDictionary.java
@@ -150,53 +150,56 @@ class BigramDictionary extends AbstractDictionary {
int[] buffer = new int[3];
byte[] intBuffer = new byte[4];
String tmpword;
- DataInputStream dctFile = new DataInputStream(Files.newInputStream(Paths.get(dctFilePath)));
-
- // GB2312 characters 0 - 6768
- for (i = GB2312_FIRST_CHAR; i < GB2312_FIRST_CHAR + CHAR_NUM_IN_FILE; i++) {
- String currentStr = getCCByGB2312Id(i);
- // if (i == 5231)
- // System.out.println(i);
-
- dctFile.read(intBuffer);
- // the dictionary was developed for C, and byte order must be converted to work with Java
- cnt = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt();
- if (cnt <= 0) {
- continue;
- }
- int j = 0;
- while (j < cnt) {
- dctFile.read(intBuffer);
- buffer[0] = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt(); // frequency
- dctFile.read(intBuffer);
- buffer[1] = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt(); // length
+ // Using try-with-resources
+ try (DataInputStream dctFile =
+ new DataInputStream(Files.newInputStream(Paths.get(dctFilePath)))) {
+
+ // GB2312 characters 0 - 6768
+ for (i = GB2312_FIRST_CHAR; i < GB2312_FIRST_CHAR + CHAR_NUM_IN_FILE; i++) {
+ String currentStr = getCCByGB2312Id(i);
+ // if (i == 5231)
+ // System.out.println(i);
+
dctFile.read(intBuffer);
- // buffer[2] = ByteBuffer.wrap(intBuffer).order(
- // ByteOrder.LITTLE_ENDIAN).getInt();// handle
-
- length = buffer[1];
- if (length > 0) {
- byte[] lchBuffer = new byte[length];
- dctFile.read(lchBuffer);
- tmpword = new String(lchBuffer, "GB2312");
- if (i != 3755 + GB2312_FIRST_CHAR) {
- tmpword = currentStr + tmpword;
- }
- char[] carray = tmpword.toCharArray();
- long hashId = hash1(carray);
- int index = getAvaliableIndex(hashId, carray);
- if (index != -1) {
- if (bigramHashTable[index] == 0) {
- bigramHashTable[index] = hashId;
- // bigramStringTable[index] = tmpword;
+ // the dictionary was developed for C, and byte order must be converted to work with Java
+ cnt = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt();
+ if (cnt <= 0) {
+ continue;
+ }
+ int j = 0;
+ while (j < cnt) {
+ dctFile.read(intBuffer);
+ buffer[0] =
+ ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt(); // frequency
+ dctFile.read(intBuffer);
+ buffer[1] = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt(); // length
+ dctFile.read(intBuffer);
+ // buffer[2] = ByteBuffer.wrap(intBuffer).order(
+ // ByteOrder.LITTLE_ENDIAN).getInt();// handle
+
+ length = buffer[1];
+ if (length > 0) {
+ byte[] lchBuffer = new byte[length];
+ dctFile.read(lchBuffer);
+ tmpword = new String(lchBuffer, "GB2312");
+ if (i != 3755 + GB2312_FIRST_CHAR) {
+ tmpword = currentStr + tmpword;
+ }
+ char[] carray = tmpword.toCharArray();
+ long hashId = hash1(carray);
+ int index = getAvaliableIndex(hashId, carray);
+ if (index != -1) {
+ if (bigramHashTable[index] == 0) {
+ bigramHashTable[index] = hashId;
+ // bigramStringTable[index] = tmpword;
+ }
+ frequencyTable[index] += buffer[0];
}
- frequencyTable[index] += buffer[0];
}
+ j++;
}
- j++;
}
}
- dctFile.close();
// log.info("load dictionary done! " + dctFilePath + " total:" + total);
}
diff --git a/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/WordDictionary.java b/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/WordDictionary.java
index 977ff7087e8..cba7e3d887e 100644
--- a/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/WordDictionary.java
+++ b/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/WordDictionary.java
@@ -189,57 +189,60 @@ class WordDictionary extends AbstractDictionary {
int[] buffer = new int[3];
byte[] intBuffer = new byte[4];
String tmpword;
- DataInputStream dctFile = new DataInputStream(Files.newInputStream(Paths.get(dctFilePath)));
-
- // GB2312 characters 0 - 6768
- for (i = GB2312_FIRST_CHAR; i < GB2312_FIRST_CHAR + CHAR_NUM_IN_FILE; i++) {
- // if (i == 5231)
- // System.out.println(i);
-
- dctFile.read(intBuffer);
- // the dictionary was developed for C, and byte order must be converted to work with Java
- cnt = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt();
- if (cnt <= 0) {
- wordItem_charArrayTable[i] = null;
- wordItem_frequencyTable[i] = null;
- continue;
- }
- wordItem_charArrayTable[i] = new char[cnt][];
- wordItem_frequencyTable[i] = new int[cnt];
- total += cnt;
- int j = 0;
- while (j < cnt) {
- // wordItemTable[i][j] = new WordItem();
- dctFile.read(intBuffer);
- buffer[0] = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt(); // frequency
- dctFile.read(intBuffer);
- buffer[1] = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt(); // length
+ // Use try-with-resources to ensure the stream is always closed
+ try (DataInputStream dctFile =
+ new DataInputStream(Files.newInputStream(Paths.get(dctFilePath)))) {
+
+ // GB2312 characters 0 - 6768
+ for (i = GB2312_FIRST_CHAR; i < GB2312_FIRST_CHAR + CHAR_NUM_IN_FILE; i++) {
+ // if (i == 5231)
+ // System.out.println(i);
+
dctFile.read(intBuffer);
- buffer[2] = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt(); // handle
-
- // wordItemTable[i][j].frequency = buffer[0];
- wordItem_frequencyTable[i][j] = buffer[0];
-
- length = buffer[1];
- if (length > 0) {
- byte[] lchBuffer = new byte[length];
- dctFile.read(lchBuffer);
- tmpword = new String(lchBuffer, "GB2312");
- // indexTable[i].wordItems[j].word = tmpword;
- // wordItemTable[i][j].charArray = tmpword.toCharArray();
- wordItem_charArrayTable[i][j] = tmpword.toCharArray();
- } else {
- // wordItemTable[i][j].charArray = null;
- wordItem_charArrayTable[i][j] = null;
+ // the dictionary was developed for C, and byte order must be converted to work with Java
+ cnt = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt();
+ if (cnt <= 0) {
+ wordItem_charArrayTable[i] = null;
+ wordItem_frequencyTable[i] = null;
+ continue;
+ }
+ wordItem_charArrayTable[i] = new char[cnt][];
+ wordItem_frequencyTable[i] = new int[cnt];
+ total += cnt;
+ int j = 0;
+ while (j < cnt) {
+ // wordItemTable[i][j] = new WordItem();
+ dctFile.read(intBuffer);
+ buffer[0] =
+ ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt(); // frequency
+ dctFile.read(intBuffer);
+ buffer[1] = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt(); // length
+ dctFile.read(intBuffer);
+ buffer[2] = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt(); // handle
+
+ // wordItemTable[i][j].frequency = buffer[0];
+ wordItem_frequencyTable[i][j] = buffer[0];
+
+ length = buffer[1];
+ if (length > 0) {
+ byte[] lchBuffer = new byte[length];
+ dctFile.read(lchBuffer);
+ tmpword = new String(lchBuffer, "GB2312");
+ // indexTable[i].wordItems[j].word = tmpword;
+ // wordItemTable[i][j].charArray = tmpword.toCharArray();
+ wordItem_charArrayTable[i][j] = tmpword.toCharArray();
+ } else {
+ // wordItemTable[i][j].charArray = null;
+ wordItem_charArrayTable[i][j] = null;
+ }
+ // System.out.println(indexTable[i].wordItems[j]);
+ j++;
}
- // System.out.println(indexTable[i].wordItems[j]);
- j++;
- }
- String str = getCCByGB2312Id(i);
- setTableIndex(str.charAt(0), i);
+ String str = getCCByGB2312Id(i);
+ setTableIndex(str.charAt(0), i);
+ }
}
- dctFile.close();
return total;
}
diff --git a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DirContentSource.java b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DirContentSource.java
index 0370bbfc029..35afd4d55e6 100644
--- a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DirContentSource.java
+++ b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DirContentSource.java
@@ -210,18 +210,23 @@ public class DirContentSource extends ContentSource {
name = f.toRealPath() + "_" + iteration;
}
- BufferedReader reader = Files.newBufferedReader(f, StandardCharsets.UTF_8);
String line = null;
- // First line is the date, 3rd is the title, rest is body
- String dateStr = reader.readLine();
- reader.readLine(); // skip an empty line
- String title = reader.readLine();
- reader.readLine(); // skip an empty line
- StringBuilder bodyBuf = new StringBuilder(1024);
- while ((line = reader.readLine()) != null) {
- bodyBuf.append(line).append(' ');
+ String title = null;
+ String dateStr = null;
+ StringBuilder bodyBuf = null;
+
+ // Use try-with-resources to ensure the reader is closed
+ try (BufferedReader reader = Files.newBufferedReader(f, StandardCharsets.UTF_8)) {
+ // First line is the date, 3rd is the title, rest is body
+ dateStr = reader.readLine();
+ reader.readLine(); // skip an empty line
+ title = reader.readLine();
+ reader.readLine(); // skip an empty line
+ bodyBuf = new StringBuilder(1024);
+ while ((line = reader.readLine()) != null) {
+ bodyBuf.append(line).append(' ');
+ }
}
- reader.close();
addBytes(Files.size(f));
Date date = parseDate(dateStr);
diff --git a/lucene/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.java b/lucene/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.java
index 05354c12da8..7a0fb8ee879 100644
--- a/lucene/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.java
+++ b/lucene/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.java
@@ -57,45 +57,45 @@ public class QueryDriver {
new SubmissionReport(
new PrintWriter(Files.newBufferedWriter(submissionFile, StandardCharsets.UTF_8)),
"lucene");
- FSDirectory dir = FSDirectory.open(Paths.get(args[3]));
String fieldSpec = args.length == 5 ? args[4] : "T"; // default to Title-only if not specified.
- IndexReader reader = DirectoryReader.open(dir);
- IndexSearcher searcher = new IndexSearcher(reader);
- int maxResults = 1000;
- String docNameField = "docname";
+ // --- Use try-with-resources for FSDirectory and IndexReader ---
+ try (FSDirectory dir = FSDirectory.open(Paths.get(args[3]));
+ IndexReader reader = DirectoryReader.open(dir)) {
+ IndexSearcher searcher = new IndexSearcher(reader);
+ int maxResults = 1000;
+ String docNameField = "docname";
- PrintWriter logger =
- new PrintWriter(new OutputStreamWriter(System.out, Charset.defaultCharset()), true);
+ PrintWriter logger =
+ new PrintWriter(new OutputStreamWriter(System.out, Charset.defaultCharset()), true);
- // use trec utilities to read trec topics into quality queries
- TrecTopicsReader qReader = new TrecTopicsReader();
- QualityQuery[] qqs =
- qReader.readQueries(Files.newBufferedReader(topicsFile, StandardCharsets.UTF_8));
+ // use trec utilities to read trec topics into quality queries
+ TrecTopicsReader qReader = new TrecTopicsReader();
+ QualityQuery[] qqs =
+ qReader.readQueries(Files.newBufferedReader(topicsFile, StandardCharsets.UTF_8));
- // prepare judge, with trec utilities that read from a QRels file
- Judge judge = new TrecJudge(Files.newBufferedReader(qrelsFile, StandardCharsets.UTF_8));
+ // prepare judge, with trec utilities that read from a QRels file
+ Judge judge = new TrecJudge(Files.newBufferedReader(qrelsFile, StandardCharsets.UTF_8));
- // validate topics & judgments match each other
- judge.validateData(qqs, logger);
+ // validate topics & judgments match each other
+ judge.validateData(qqs, logger);
- Set<String> fieldSet = new HashSet<>();
- if (fieldSpec.indexOf('T') >= 0) fieldSet.add("title");
- if (fieldSpec.indexOf('D') >= 0) fieldSet.add("description");
- if (fieldSpec.indexOf('N') >= 0) fieldSet.add("narrative");
+ Set<String> fieldSet = new HashSet<>();
+ if (fieldSpec.indexOf('T') >= 0) fieldSet.add("title");
+ if (fieldSpec.indexOf('D') >= 0) fieldSet.add("description");
+ if (fieldSpec.indexOf('N') >= 0) fieldSet.add("narrative");
- // set the parsing of quality queries into Lucene queries.
- QualityQueryParser qqParser = new SimpleQQParser(fieldSet.toArray(new String[0]), "body");
+ // set the parsing of quality queries into Lucene queries.
+ QualityQueryParser qqParser = new SimpleQQParser(fieldSet.toArray(new String[0]), "body");
- // run the benchmark
- QualityBenchmark qrun = new QualityBenchmark(qqs, qqParser, searcher, docNameField);
- qrun.setMaxResults(maxResults);
- QualityStats[] stats = qrun.execute(judge, submitLog, logger);
+ // run the benchmark
+ QualityBenchmark qrun = new QualityBenchmark(qqs, qqParser, searcher, docNameField);
+ qrun.setMaxResults(maxResults);
+ QualityStats[] stats = qrun.execute(judge, submitLog, logger);
- // print an avarage sum of the results
- QualityStats avg = QualityStats.average(stats);
- avg.log("SUMMARY", 2, logger, " ");
- reader.close();
- dir.close();
+ // print an avarage sum of the results
+ QualityStats avg = QualityStats.average(stats);
+ avg.log("SUMMARY", 2, logger, " ");
+ }
}
}