Apache Lucene open-source search software
Fix resource leak in loadMainDataFromFile (#14727)
xcx7759fde64c8e
diff --git a/lucene/CHANGES.txt b/lucene/CHANGES.txtindex dd69e1a2ebf..4ef5d902082 100644--- a/lucene/CHANGES.txt+++ b/lucene/CHANGES.txt@@ -148,6 +148,8 @@ Bug Fixes* GITHUB#14161: PointInSetQuery's constructor now throws IllegalArgumentExceptioninstead of UnsupportedOperationException when values are out of order. (Shubham Sharma)+* GITHUB#14727: Fix several resource leaks. (Xu Chengxin)+Build---------------------* Upgrade forbiddenapis to version 3.9. (Uwe Schindler)diff --git a/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BigramDictionary.java b/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BigramDictionary.javaindex b4ef6af2d69..885cc144037 100644--- a/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BigramDictionary.java+++ b/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/BigramDictionary.java@@ -150,53 +150,56 @@ class BigramDictionary extends AbstractDictionary {int[] buffer = new int[3];byte[] intBuffer = new byte[4];String tmpword;- DataInputStream dctFile = new DataInputStream(Files.newInputStream(Paths.get(dctFilePath)));-- // GB2312 characters 0 - 6768- for (i = GB2312_FIRST_CHAR; i < GB2312_FIRST_CHAR + CHAR_NUM_IN_FILE; i++) {- String currentStr = getCCByGB2312Id(i);- // if (i == 5231)- // System.out.println(i);-- dctFile.read(intBuffer);- // the dictionary was developed for C, and byte order must be converted to work with Java- cnt = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt();- if (cnt <= 0) {- continue;- }- int j = 0;- while (j < cnt) {- dctFile.read(intBuffer);- buffer[0] = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt(); // frequency- dctFile.read(intBuffer);- buffer[1] = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt(); // length+ // Using try-with-resources+ try (DataInputStream dctFile =+ new DataInputStream(Files.newInputStream(Paths.get(dctFilePath)))) {++ // GB2312 characters 0 - 6768+ for (i = GB2312_FIRST_CHAR; i < GB2312_FIRST_CHAR + CHAR_NUM_IN_FILE; i++) {+ String currentStr = getCCByGB2312Id(i);+ // if (i == 5231)+ // System.out.println(i);+dctFile.read(intBuffer);- // buffer[2] = ByteBuffer.wrap(intBuffer).order(- // ByteOrder.LITTLE_ENDIAN).getInt();// handle-- length = buffer[1];- if (length > 0) {- byte[] lchBuffer = new byte[length];- dctFile.read(lchBuffer);- tmpword = new String(lchBuffer, "GB2312");- if (i != 3755 + GB2312_FIRST_CHAR) {- tmpword = currentStr + tmpword;- }- char[] carray = tmpword.toCharArray();- long hashId = hash1(carray);- int index = getAvaliableIndex(hashId, carray);- if (index != -1) {- if (bigramHashTable[index] == 0) {- bigramHashTable[index] = hashId;- // bigramStringTable[index] = tmpword;+ // the dictionary was developed for C, and byte order must be converted to work with Java+ cnt = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt();+ if (cnt <= 0) {+ continue;+ }+ int j = 0;+ while (j < cnt) {+ dctFile.read(intBuffer);+ buffer[0] =+ ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt(); // frequency+ dctFile.read(intBuffer);+ buffer[1] = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt(); // length+ dctFile.read(intBuffer);+ // buffer[2] = ByteBuffer.wrap(intBuffer).order(+ // ByteOrder.LITTLE_ENDIAN).getInt();// handle++ length = buffer[1];+ if (length > 0) {+ byte[] lchBuffer = new byte[length];+ dctFile.read(lchBuffer);+ tmpword = new String(lchBuffer, "GB2312");+ if (i != 3755 + GB2312_FIRST_CHAR) {+ tmpword = currentStr + tmpword;+ }+ char[] carray = tmpword.toCharArray();+ long hashId = hash1(carray);+ int index = getAvaliableIndex(hashId, carray);+ if (index != -1) {+ if (bigramHashTable[index] == 0) {+ bigramHashTable[index] = hashId;+ // bigramStringTable[index] = tmpword;+ }+ frequencyTable[index] += buffer[0];}- frequencyTable[index] += buffer[0];}+ j++;}- j++;}}- dctFile.close();// log.info("load dictionary done! " + dctFilePath + " total:" + total);}diff --git a/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/WordDictionary.java b/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/WordDictionary.javaindex 977ff7087e8..cba7e3d887e 100644--- a/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/WordDictionary.java+++ b/lucene/analysis/smartcn/src/java/org/apache/lucene/analysis/cn/smart/hhmm/WordDictionary.java@@ -189,57 +189,60 @@ class WordDictionary extends AbstractDictionary {int[] buffer = new int[3];byte[] intBuffer = new byte[4];String tmpword;- DataInputStream dctFile = new DataInputStream(Files.newInputStream(Paths.get(dctFilePath)));-- // GB2312 characters 0 - 6768- for (i = GB2312_FIRST_CHAR; i < GB2312_FIRST_CHAR + CHAR_NUM_IN_FILE; i++) {- // if (i == 5231)- // System.out.println(i);-- dctFile.read(intBuffer);- // the dictionary was developed for C, and byte order must be converted to work with Java- cnt = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt();- if (cnt <= 0) {- wordItem_charArrayTable[i] = null;- wordItem_frequencyTable[i] = null;- continue;- }- wordItem_charArrayTable[i] = new char[cnt][];- wordItem_frequencyTable[i] = new int[cnt];- total += cnt;- int j = 0;- while (j < cnt) {- // wordItemTable[i][j] = new WordItem();- dctFile.read(intBuffer);- buffer[0] = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt(); // frequency- dctFile.read(intBuffer);- buffer[1] = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt(); // length+ // Use try-with-resources to ensure the stream is always closed+ try (DataInputStream dctFile =+ new DataInputStream(Files.newInputStream(Paths.get(dctFilePath)))) {++ // GB2312 characters 0 - 6768+ for (i = GB2312_FIRST_CHAR; i < GB2312_FIRST_CHAR + CHAR_NUM_IN_FILE; i++) {+ // if (i == 5231)+ // System.out.println(i);+dctFile.read(intBuffer);- buffer[2] = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt(); // handle-- // wordItemTable[i][j].frequency = buffer[0];- wordItem_frequencyTable[i][j] = buffer[0];-- length = buffer[1];- if (length > 0) {- byte[] lchBuffer = new byte[length];- dctFile.read(lchBuffer);- tmpword = new String(lchBuffer, "GB2312");- // indexTable[i].wordItems[j].word = tmpword;- // wordItemTable[i][j].charArray = tmpword.toCharArray();- wordItem_charArrayTable[i][j] = tmpword.toCharArray();- } else {- // wordItemTable[i][j].charArray = null;- wordItem_charArrayTable[i][j] = null;+ // the dictionary was developed for C, and byte order must be converted to work with Java+ cnt = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt();+ if (cnt <= 0) {+ wordItem_charArrayTable[i] = null;+ wordItem_frequencyTable[i] = null;+ continue;+ }+ wordItem_charArrayTable[i] = new char[cnt][];+ wordItem_frequencyTable[i] = new int[cnt];+ total += cnt;+ int j = 0;+ while (j < cnt) {+ // wordItemTable[i][j] = new WordItem();+ dctFile.read(intBuffer);+ buffer[0] =+ ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt(); // frequency+ dctFile.read(intBuffer);+ buffer[1] = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt(); // length+ dctFile.read(intBuffer);+ buffer[2] = ByteBuffer.wrap(intBuffer).order(ByteOrder.LITTLE_ENDIAN).getInt(); // handle++ // wordItemTable[i][j].frequency = buffer[0];+ wordItem_frequencyTable[i][j] = buffer[0];++ length = buffer[1];+ if (length > 0) {+ byte[] lchBuffer = new byte[length];+ dctFile.read(lchBuffer);+ tmpword = new String(lchBuffer, "GB2312");+ // indexTable[i].wordItems[j].word = tmpword;+ // wordItemTable[i][j].charArray = tmpword.toCharArray();+ wordItem_charArrayTable[i][j] = tmpword.toCharArray();+ } else {+ // wordItemTable[i][j].charArray = null;+ wordItem_charArrayTable[i][j] = null;+ }+ // System.out.println(indexTable[i].wordItems[j]);+ j++;}- // System.out.println(indexTable[i].wordItems[j]);- j++;- }- String str = getCCByGB2312Id(i);- setTableIndex(str.charAt(0), i);+ String str = getCCByGB2312Id(i);+ setTableIndex(str.charAt(0), i);+ }}- dctFile.close();return total;}diff --git a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DirContentSource.java b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DirContentSource.javaindex 0370bbfc029..35afd4d55e6 100644--- a/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DirContentSource.java+++ b/lucene/benchmark/src/java/org/apache/lucene/benchmark/byTask/feeds/DirContentSource.java@@ -210,18 +210,23 @@ public class DirContentSource extends ContentSource {name = f.toRealPath() + "_" + iteration;}- BufferedReader reader = Files.newBufferedReader(f, StandardCharsets.UTF_8);String line = null;- // First line is the date, 3rd is the title, rest is body- String dateStr = reader.readLine();- reader.readLine(); // skip an empty line- String title = reader.readLine();- reader.readLine(); // skip an empty line- StringBuilder bodyBuf = new StringBuilder(1024);- while ((line = reader.readLine()) != null) {- bodyBuf.append(line).append(' ');+ String title = null;+ String dateStr = null;+ StringBuilder bodyBuf = null;++ // Use try-with-resources to ensure the reader is closed+ try (BufferedReader reader = Files.newBufferedReader(f, StandardCharsets.UTF_8)) {+ // First line is the date, 3rd is the title, rest is body+ dateStr = reader.readLine();+ reader.readLine(); // skip an empty line+ title = reader.readLine();+ reader.readLine(); // skip an empty line+ bodyBuf = new StringBuilder(1024);+ while ((line = reader.readLine()) != null) {+ bodyBuf.append(line).append(' ');+ }}- reader.close();addBytes(Files.size(f));Date date = parseDate(dateStr);diff --git a/lucene/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.java b/lucene/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.javaindex 05354c12da8..7a0fb8ee879 100644--- a/lucene/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.java+++ b/lucene/benchmark/src/java/org/apache/lucene/benchmark/quality/trec/QueryDriver.java@@ -57,45 +57,45 @@ public class QueryDriver {new SubmissionReport(new PrintWriter(Files.newBufferedWriter(submissionFile, StandardCharsets.UTF_8)),"lucene");- FSDirectory dir = FSDirectory.open(Paths.get(args[3]));String fieldSpec = args.length == 5 ? args[4] : "T"; // default to Title-only if not specified.- IndexReader reader = DirectoryReader.open(dir);- IndexSearcher searcher = new IndexSearcher(reader);- int maxResults = 1000;- String docNameField = "docname";+ // --- Use try-with-resources for FSDirectory and IndexReader ---+ try (FSDirectory dir = FSDirectory.open(Paths.get(args[3]));+ IndexReader reader = DirectoryReader.open(dir)) {+ IndexSearcher searcher = new IndexSearcher(reader);+ int maxResults = 1000;+ String docNameField = "docname";- PrintWriter logger =- new PrintWriter(new OutputStreamWriter(System.out, Charset.defaultCharset()), true);+ PrintWriter logger =+ new PrintWriter(new OutputStreamWriter(System.out, Charset.defaultCharset()), true);- // use trec utilities to read trec topics into quality queries- TrecTopicsReader qReader = new TrecTopicsReader();- QualityQuery[] qqs =- qReader.readQueries(Files.newBufferedReader(topicsFile, StandardCharsets.UTF_8));+ // use trec utilities to read trec topics into quality queries+ TrecTopicsReader qReader = new TrecTopicsReader();+ QualityQuery[] qqs =+ qReader.readQueries(Files.newBufferedReader(topicsFile, StandardCharsets.UTF_8));- // prepare judge, with trec utilities that read from a QRels file- Judge judge = new TrecJudge(Files.newBufferedReader(qrelsFile, StandardCharsets.UTF_8));+ // prepare judge, with trec utilities that read from a QRels file+ Judge judge = new TrecJudge(Files.newBufferedReader(qrelsFile, StandardCharsets.UTF_8));- // validate topics & judgments match each other- judge.validateData(qqs, logger);+ // validate topics & judgments match each other+ judge.validateData(qqs, logger);- Set<String> fieldSet = new HashSet<>();- if (fieldSpec.indexOf('T') >= 0) fieldSet.add("title");- if (fieldSpec.indexOf('D') >= 0) fieldSet.add("description");- if (fieldSpec.indexOf('N') >= 0) fieldSet.add("narrative");+ Set<String> fieldSet = new HashSet<>();+ if (fieldSpec.indexOf('T') >= 0) fieldSet.add("title");+ if (fieldSpec.indexOf('D') >= 0) fieldSet.add("description");+ if (fieldSpec.indexOf('N') >= 0) fieldSet.add("narrative");- // set the parsing of quality queries into Lucene queries.- QualityQueryParser qqParser = new SimpleQQParser(fieldSet.toArray(new String[0]), "body");+ // set the parsing of quality queries into Lucene queries.+ QualityQueryParser qqParser = new SimpleQQParser(fieldSet.toArray(new String[0]), "body");- // run the benchmark- QualityBenchmark qrun = new QualityBenchmark(qqs, qqParser, searcher, docNameField);- qrun.setMaxResults(maxResults);- QualityStats[] stats = qrun.execute(judge, submitLog, logger);+ // run the benchmark+ QualityBenchmark qrun = new QualityBenchmark(qqs, qqParser, searcher, docNameField);+ qrun.setMaxResults(maxResults);+ QualityStats[] stats = qrun.execute(judge, submitLog, logger);- // print an avarage sum of the results- QualityStats avg = QualityStats.average(stats);- avg.log("SUMMARY", 2, logger, " ");- reader.close();- dir.close();+ // print an avarage sum of the results+ QualityStats avg = QualityStats.average(stats);+ avg.log("SUMMARY", 2, logger, " ");+ }}}