Apache Nutch is an extensible and scalable web crawler
NUTCH-3200 Close Closeable resources flagged by Sonar java:S2095 (#948)
Lewis John McGibbney21ba9a8a8171
diff --git a/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java b/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.javaindex 04a3a1303..6bfb6085f 100644--- a/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java+++ b/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java@@ -133,111 +133,111 @@ public class AdaptiveFetchSchedule extends AbstractFetchSchedule {private void setHostSpecificIntervals(String fileName,float defaultMin, float defaultMax) throws IOException {// Setup for reading the config file.- Reader configReader = null;- configReader = conf.getConfResourceAsReader(fileName);+ Reader configReader = conf.getConfResourceAsReader(fileName);if (configReader == null) {configReader = new FileReader(fileName, StandardCharsets.UTF_8);}- BufferedReader reader = new BufferedReader(configReader);- String line;- int lineNo = 0;+ try (BufferedReader reader = new BufferedReader(configReader)) {+ String line;+ int lineNo = 0;- // Read the file line by line.- while ((line = reader.readLine()) != null) {- lineNo++;+ // Read the file line by line.+ while ((line = reader.readLine()) != null) {+ lineNo++;- // Skip blank lines and comments.- if (StringUtils.isBlank(line) || line.startsWith("#")) {- continue;- }+ // Skip blank lines and comments.+ if (StringUtils.isBlank(line) || line.startsWith("#")) {+ continue;+ }- // Trim and partition the line.- line = line.trim();- String[] parts = line.split("\\s+");+ // Trim and partition the line.+ line = line.trim();+ String[] parts = line.split("\\s+");- // There should be three parts.- if (parts.length != 3) {- LOG.error(- "Malformed (domain, min_interval, max_interval) triplet on line {} of the config. file: `{}`",- lineNo, line);- continue;- }+ // There should be three parts.+ if (parts.length != 3) {+ LOG.error(+ "Malformed (domain, min_interval, max_interval) triplet on line {} of the config. file: `{}`",+ lineNo, line);+ continue;+ }- // Normalize the parts.- String host = parts[0].trim().toLowerCase(Locale.ROOT);- String minInt = parts[1].trim();- String maxInt = parts[2].trim();-- // "0" and "default" both mean `use default interval`; normalize to "0".- if (minInt.equalsIgnoreCase("default")) { minInt = "0"; }- if (maxInt.equalsIgnoreCase("default")) { maxInt = "0"; }-- // Convert intervals to float and ignore the line in case of failure.- float m, M;- try {- m = Float.parseFloat(minInt);- M = Float.parseFloat(maxInt);- } catch (NumberFormatException e) {- LOG.error(- "Improper fetch intervals given on line {} in the config. file `{}`: {}",- lineNo, line, e.toString());- continue;- }+ // Normalize the parts.+ String host = parts[0].trim().toLowerCase(Locale.ROOT);+ String minInt = parts[1].trim();+ String maxInt = parts[2].trim();++ // "0" and "default" both mean `use default interval`; normalize to "0".+ if (minInt.equalsIgnoreCase("default")) { minInt = "0"; }+ if (maxInt.equalsIgnoreCase("default")) { maxInt = "0"; }++ // Convert intervals to float and ignore the line in case of failure.+ float m, M;+ try {+ m = Float.parseFloat(minInt);+ M = Float.parseFloat(maxInt);+ } catch (NumberFormatException e) {+ LOG.error(+ "Improper fetch intervals given on line {} in the config. file `{}`: {}",+ lineNo, line, e.toString());+ continue;+ }- // If both intervals are set to default,- // ignore the line and issue a warning.- if (m == 0 && M == 0) {- LOG.warn(- "Ignoring default interval values on line {} of config. file: `{}`",- lineNo, line);- continue;- }+ // If both intervals are set to default,+ // ignore the line and issue a warning.+ if (m == 0 && M == 0) {+ LOG.warn(+ "Ignoring default interval values on line {} of config. file: `{}`",+ lineNo, line);+ continue;+ }- // Replace the zero with the default value.- if (m == 0) {- m = defaultMin;- } else if (M == 0) {- M = defaultMax;- }+ // Replace the zero with the default value.+ if (m == 0) {+ m = defaultMin;+ } else if (M == 0) {+ M = defaultMax;+ }- // Intervals cannot be negative and the min cannot be above the max- // (we assume here that the default values satisfy this).- if (m < 0 || M < 0) {- LOG.error(- "Improper fetch intervals given on line {} in the config. file: `{}`: intervals cannot be negative",- lineNo, line);- continue;- }+ // Intervals cannot be negative and the min cannot be above the max+ // (we assume here that the default values satisfy this).+ if (m < 0 || M < 0) {+ LOG.error(+ "Improper fetch intervals given on line {} in the config. file: `{}`: intervals cannot be negative",+ lineNo, line);+ continue;+ }- if (m > M) {- LOG.error(- "Improper fetch intervals given on line {} in the config. file: `{}`: min. interval cannot be above max. interval",- lineNo, line);- continue;- }+ if (m > M) {+ LOG.error(+ "Improper fetch intervals given on line {} in the config. file: `{}`: min. interval cannot be above max. interval",+ lineNo, line);+ continue;+ }- // The custom intervals should respect the boundaries of the default values.- if (m < defaultMin) {- LOG.error(- "Min. interval out of bounds ({}) on line {} in the config. file: `{}`",- defaultMin, lineNo, line);- continue;- }+ // The custom intervals should respect the boundaries of the default values.+ if (m < defaultMin) {+ LOG.error(+ "Min. interval out of bounds ({}) on line {} in the config. file: `{}`",+ defaultMin, lineNo, line);+ continue;+ }- if (M > defaultMax) {- LOG.error(- "Max. interval out of bounds ({}) on line {} in the config. file: `{}`",- defaultMax, lineNo, line);- continue;- }+ if (M > defaultMax) {+ LOG.error(+ "Max. interval out of bounds ({}) on line {} in the config. file: `{}`",+ defaultMax, lineNo, line);+ continue;+ }- // If all is well, store the specific intervals.- hostSpecificMinInterval.put(host, m);- LOG.debug("Added custom min. interval {} for host {}.", m, host);+ // If all is well, store the specific intervals.+ hostSpecificMinInterval.put(host, m);+ LOG.debug("Added custom min. interval {} for host {}.", m, host);- hostSpecificMaxInterval.put(host, M);- LOG.debug("Added custom max. interval {} for host {}.", M, host);+ hostSpecificMaxInterval.put(host, M);+ LOG.debug("Added custom max. interval {} for host {}.", M, host);+ }}}diff --git a/src/java/org/apache/nutch/crawl/TextProfileSignature.java b/src/java/org/apache/nutch/crawl/TextProfileSignature.javaindex 63583a694..f54d230ae 100644--- a/src/java/org/apache/nutch/crawl/TextProfileSignature.java+++ b/src/java/org/apache/nutch/crawl/TextProfileSignature.java@@ -195,17 +195,16 @@ public class TextProfileSignature extends Signature {HashMap<String, byte[]> res = new HashMap<>();File[] files = new File(args[0]).listFiles();for (int i = 0; i < files.length; i++) {- FileInputStream fis = new FileInputStream(files[i]);- BufferedReader br = new BufferedReader(- new InputStreamReader(fis, StandardCharsets.UTF_8));StringBuffer text = new StringBuffer();- String line = null;- while ((line = br.readLine()) != null) {- if (text.length() > 0)- text.append("\n");- text.append(line);+ try (BufferedReader br = new BufferedReader(new InputStreamReader(+ new FileInputStream(files[i]), StandardCharsets.UTF_8))) {+ String line = null;+ while ((line = br.readLine()) != null) {+ if (text.length() > 0)+ text.append("\n");+ text.append(line);+ }}- br.close();byte[] signature = sig.calculate(null, new ParseImpl(text.toString(),null));res.put(files[i].toString(), signature);diff --git a/src/java/org/apache/nutch/protocol/RobotRulesParser.java b/src/java/org/apache/nutch/protocol/RobotRulesParser.javaindex 5b0412910..3efa0fc3b 100644--- a/src/java/org/apache/nutch/protocol/RobotRulesParser.java+++ b/src/java/org/apache/nutch/protocol/RobotRulesParser.java@@ -398,30 +398,30 @@ public abstract class RobotRulesParser implements Tool {System.out.println("Testing robots.txt for agent names: "+ (agentNames.isEmpty() ? "* (any other agent)" : agentNames));- LineNumberReader testsIn = new LineNumberReader(- new FileReader(urlFile, StandardCharsets.UTF_8));- String testPath;- testPath = testsIn.readLine();- while (testPath != null) {- testPath = testPath.trim();- try {- // testPath can be just a path or a complete URL- URL url = new URL(testPath);- String status;- if (isAllowListed(url)) {- status = "allowlisted";- } else if (rules.isAllowed(testPath)) {- status = "allowed";- } else {- status = "not allowed";+ try (LineNumberReader testsIn = new LineNumberReader(+ new FileReader(urlFile, StandardCharsets.UTF_8))) {+ String testPath;+ testPath = testsIn.readLine();+ while (testPath != null) {+ testPath = testPath.trim();+ try {+ // testPath can be just a path or a complete URL+ URL url = new URL(testPath);+ String status;+ if (isAllowListed(url)) {+ status = "allowlisted";+ } else if (rules.isAllowed(testPath)) {+ status = "allowed";+ } else {+ status = "not allowed";+ }+ System.out.println(status + ":\t" + testPath);+ } catch (MalformedURLException e) {+ LOG.warn("Not a valid URL: {}", testPath);}- System.out.println(status + ":\t" + testPath);- } catch (MalformedURLException e) {- LOG.warn("Not a valid URL: {}", testPath);+ testPath = testsIn.readLine();}- testPath = testsIn.readLine();}- testsIn.close();} catch (IOException e) {LOG.error("Failed to run:", e);return -1;@@ -476,9 +476,9 @@ public abstract class RobotRulesParser implements Tool {try {int contentLength = url.openConnection().getContentLength();byte[] robotsBytes = new byte[contentLength];- InputStream openStream = url.openStream();- openStream.read(robotsBytes);- openStream.close();+ try (InputStream openStream = url.openStream()) {+ openStream.read(robotsBytes);+ }rules = robotParser.parseContent(url.toString(), robotsBytes,"text/plain", agentNames);} catch (IOException e) {diff --git a/src/java/org/apache/nutch/scoring/webgraph/LinkRank.java b/src/java/org/apache/nutch/scoring/webgraph/LinkRank.javaindex 80d6ac7cf..de15e3d66 100644--- a/src/java/org/apache/nutch/scoring/webgraph/LinkRank.java+++ b/src/java/org/apache/nutch/scoring/webgraph/LinkRank.java@@ -141,25 +141,26 @@ public class LinkRank extends Configured implements Tool {}Path numLinksFile = numLinksFiles[0].getPath();LOG.info("Reading numlinks temp file {}", numLinksFile);- FSDataInputStream readLinks = fs.open(numLinksFile);CompressionCodecFactory cf = new CompressionCodecFactory(conf);CompressionCodec codec = cf.getCodec(numLinksFiles[0].getPath());- InputStream streamLinks;- if (codec == null) {- LOG.debug("No compression codec found for {}, trying uncompressed",- numLinksFile);- streamLinks = readLinks;- } else {- LOG.info("Compression codec of numlinks temp file: {}",- codec.getDefaultExtension());- readLinks.seek(0);- streamLinks = codec.createInputStream(readLinks);+ String numLinksLine;+ try (FSDataInputStream readLinks = fs.open(numLinksFile)) {+ InputStream streamLinks;+ if (codec == null) {+ LOG.debug("No compression codec found for {}, trying uncompressed",+ numLinksFile);+ streamLinks = readLinks;+ } else {+ LOG.info("Compression codec of numlinks temp file: {}",+ codec.getDefaultExtension());+ readLinks.seek(0);+ streamLinks = codec.createInputStream(readLinks);+ }+ try (BufferedReader buffer = new BufferedReader(+ new InputStreamReader(streamLinks, StandardCharsets.UTF_8))) {+ numLinksLine = buffer.readLine();+ }}- BufferedReader buffer = new BufferedReader(- new InputStreamReader(streamLinks, StandardCharsets.UTF_8));-- String numLinksLine = buffer.readLine();- readLinks.close();// check if there are links to process, if none, webgraph might be emptyif (numLinksLine == null || numLinksLine.length() == 0) {diff --git a/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java b/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.javaindex 39935016d..fe714218e 100644--- a/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java+++ b/src/java/org/apache/nutch/tools/CommonCrawlDataDumper.java@@ -269,182 +269,180 @@ public class CommonCrawlDataDumper extends NutchTool implements Tool {}}- LinkDbReader linkDbReader = null;- if (linkdb != null) {- linkDbReader = new LinkDbReader(nutchConfig, new Path(linkdb.toString()));- }- if (parts == null || parts.size() == 0) {- LOG.error( "No segment directories found in {} ",- segmentRootDir.getAbsolutePath());- this.errorTracker.recordError(ErrorTracker.ErrorType.OTHER);- return;- }- LOG.info("Found {} segment parts", parts.size());- if (gzip && !warc) {- fileList = new ArrayList<>();- constructNewStream(outputDir);- }+ try (LinkDbReader linkDbReader = linkdb != null+ ? new LinkDbReader(nutchConfig, new Path(linkdb.toString()))+ : null) {+ if (parts == null || parts.size() == 0) {+ LOG.error( "No segment directories found in {} ",+ segmentRootDir.getAbsolutePath());+ this.errorTracker.recordError(ErrorTracker.ErrorType.OTHER);+ return;+ }+ LOG.info("Found {} segment parts", parts.size());+ if (gzip && !warc) {+ fileList = new ArrayList<>();+ constructNewStream(outputDir);+ }- for (Path segmentPart : parts) {- LOG.info("Processing segment Part : [ {} ]", segmentPart);- try {- SequenceFile.Reader reader = new SequenceFile.Reader(nutchConfig,- SequenceFile.Reader.file(segmentPart));+ for (Path segmentPart : parts) {+ LOG.info("Processing segment Part : [ {} ]", segmentPart);+ try (SequenceFile.Reader reader = new SequenceFile.Reader(nutchConfig,+ SequenceFile.Reader.file(segmentPart))) {- Writable key = (Writable) reader.getKeyClass().getConstructor().newInstance();+ Writable key = (Writable) reader.getKeyClass().getConstructor().newInstance();- Content content = null;- while (reader.next(key)) {- content = new Content();- reader.getCurrentValue(content);- Metadata metadata = content.getMetadata();- String url = key.toString();+ Content content = null;+ while (reader.next(key)) {+ content = new Content();+ reader.getCurrentValue(content);+ Metadata metadata = content.getMetadata();+ String url = key.toString();- String baseName = FilenameUtils.getBaseName(url);- String extensionName = FilenameUtils.getExtension(url);+ String baseName = FilenameUtils.getBaseName(url);+ String extensionName = FilenameUtils.getExtension(url);- if (!extension.isEmpty()) {- extensionName = extension;- } else if ((extensionName == null) || extensionName.isEmpty()) {- extensionName = "html";- }+ if (!extension.isEmpty()) {+ extensionName = extension;+ } else if ((extensionName == null) || extensionName.isEmpty()) {+ extensionName = "html";+ }- String outputFullPath = null;- String outputRelativePath = null;- String filename = null;- String timestamp = null;- String reverseKey = null;+ String outputFullPath = null;+ String outputRelativePath = null;+ String filename = null;+ String timestamp = null;+ String reverseKey = null;++ if (epochFilename || config.getReverseKey()) {+ try {+ long epoch = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z")+ .parse(getDate(metadata.get("Date"))).getTime();+ timestamp = String.valueOf(epoch);+ } catch (ParseException pe) {+ LOG.warn(pe.getMessage());+ }- if (epochFilename || config.getReverseKey()) {- try {- long epoch = new SimpleDateFormat("EEE, d MMM yyyy HH:mm:ss z")- .parse(getDate(metadata.get("Date"))).getTime();- timestamp = String.valueOf(epoch);- } catch (ParseException pe) {- LOG.warn(pe.getMessage());+ reverseKey = reverseUrl(url);+ config.setReverseKeyValue(+ reverseKey.replace("/", "_") + "_" + DigestUtils.sha1Hex(url)+ + "_" + timestamp);}- reverseKey = reverseUrl(url);- config.setReverseKeyValue(- reverseKey.replace("/", "_") + "_" + DigestUtils.sha1Hex(url)- + "_" + timestamp);- }-- if (!warc) {- if (epochFilename) {- outputFullPath = DumpFileUtil- .createFileNameFromUrl(outputDir.getAbsolutePath(),- reverseKey, url, timestamp, extensionName, !gzip);- outputRelativePath = outputFullPath- .substring(0, outputFullPath.lastIndexOf(File.separator) - 1);- filename = content.getMetadata().get(Metadata.DATE) + "."- + extensionName;- } else {- String md5Ofurl = DumpFileUtil.getUrlMD5(url);- String fullDir = DumpFileUtil- .createTwoLevelsDirectory(outputDir.getAbsolutePath(),- md5Ofurl, !gzip);- filename = DumpFileUtil- .createFileName(md5Ofurl, baseName, extensionName);- outputFullPath = String.format(Locale.ROOT, "%s/%s", fullDir, filename);-- String[] fullPathLevels = fullDir- .split(Pattern.quote(File.separator));- String firstLevelDirName = fullPathLevels[fullPathLevels.length- - 2];- String secondLevelDirName = fullPathLevels[fullPathLevels.length- - 1];- outputRelativePath = firstLevelDirName + secondLevelDirName;+ if (!warc) {+ if (epochFilename) {+ outputFullPath = DumpFileUtil+ .createFileNameFromUrl(outputDir.getAbsolutePath(),+ reverseKey, url, timestamp, extensionName, !gzip);+ outputRelativePath = outputFullPath+ .substring(0, outputFullPath.lastIndexOf(File.separator) - 1);+ filename = content.getMetadata().get(Metadata.DATE) + "."+ + extensionName;+ } else {+ String md5Ofurl = DumpFileUtil.getUrlMD5(url);+ String fullDir = DumpFileUtil+ .createTwoLevelsDirectory(outputDir.getAbsolutePath(),+ md5Ofurl, !gzip);+ filename = DumpFileUtil+ .createFileName(md5Ofurl, baseName, extensionName);+ outputFullPath = String.format(Locale.ROOT, "%s/%s", fullDir, filename);++ String[] fullPathLevels = fullDir+ .split(Pattern.quote(File.separator));+ String firstLevelDirName = fullPathLevels[fullPathLevels.length+ - 2];+ String secondLevelDirName = fullPathLevels[fullPathLevels.length+ - 1];+ outputRelativePath = firstLevelDirName + secondLevelDirName;+ }}- }- // Encode all filetypes if no mimetypes have been given- Boolean filter = (mimeTypes == null);-- String jsonData = "";- try {- String mimeType = new Tika().detect(content.getContent());- // Maps file to JSON-based structure-- Set<String> inUrls = null; //there may be duplicates, so using set- if (linkDbReader != null) {- Inlinks inlinks = linkDbReader.getInlinks((Text) key);- if (inlinks != null) {- Iterator<Inlink> iterator = inlinks.iterator();- inUrls = new LinkedHashSet<>();- while (inUrls.size() <= MAX_INLINKS && iterator.hasNext()){- inUrls.add(iterator.next().getFromUrl());+ // Encode all filetypes if no mimetypes have been given+ Boolean filter = (mimeTypes == null);++ String jsonData = "";+ try {+ String mimeType = new Tika().detect(content.getContent());+ // Maps file to JSON-based structure++ Set<String> inUrls = null; //there may be duplicates, so using set+ if (linkDbReader != null) {+ Inlinks inlinks = linkDbReader.getInlinks((Text) key);+ if (inlinks != null) {+ Iterator<Inlink> iterator = inlinks.iterator();+ inUrls = new LinkedHashSet<>();+ while (inUrls.size() <= MAX_INLINKS && iterator.hasNext()){+ inUrls.add(iterator.next().getFromUrl());+ }}}- }- //TODO: Make this Jackson Format implementation reusable- try (CommonCrawlFormat format = CommonCrawlFormatFactory- .getCommonCrawlFormat(warc ? "WARC" : "JACKSON", nutchConfig, config)) {- if (inUrls != null) {- format.setInLinks(new ArrayList<>(inUrls));+ //TODO: Make this Jackson Format implementation reusable+ try (CommonCrawlFormat format = CommonCrawlFormatFactory+ .getCommonCrawlFormat(warc ? "WARC" : "JACKSON", nutchConfig, config)) {+ if (inUrls != null) {+ format.setInLinks(new ArrayList<>(inUrls));+ }+ jsonData = format.getJsonData(url, content, metadata);}- jsonData = format.getJsonData(url, content, metadata);- }- collectStats(typeCounts, mimeType);- // collects statistics for the given mimetypes- if ((mimeType != null) && (mimeTypes != null) && Arrays- .asList(mimeTypes).contains(mimeType)) {- collectStats(filteredCounts, mimeType);- filter = true;+ collectStats(typeCounts, mimeType);+ // collects statistics for the given mimetypes+ if ((mimeType != null) && (mimeTypes != null) && Arrays+ .asList(mimeTypes).contains(mimeType)) {+ collectStats(filteredCounts, mimeType);+ filter = true;+ }+ } catch (IOException ioe) {+ LOG.error("Fatal error in creating JSON data: {}", ioe.getMessage());+ return;}- } catch (IOException ioe) {- LOG.error("Fatal error in creating JSON data: {}", ioe.getMessage());- return;- }- if (!warc) {- if (filter) {- byte[] byteData = serializeCBORData(jsonData);-- if (!gzip) {- File outputFile = new File(outputFullPath);- if (outputFile.exists()) {- LOG.info("Skipping writing: [{}]: file already exists", outputFullPath);- } else {- LOG.info("Writing: [{}]", outputFullPath);- IOUtils.copy(new ByteArrayInputStream(byteData),- new FileOutputStream(outputFile));- }- } else {- if (fileList.contains(outputFullPath)) {- LOG.info("Skipping compressing: [{}]: file already exists", outputFullPath);+ if (!warc) {+ if (filter) {+ byte[] byteData = serializeCBORData(jsonData);++ if (!gzip) {+ File outputFile = new File(outputFullPath);+ if (outputFile.exists()) {+ LOG.info("Skipping writing: [{}]: file already exists", outputFullPath);+ } else {+ LOG.info("Writing: [{}]", outputFullPath);+ IOUtils.copy(new ByteArrayInputStream(byteData),+ new FileOutputStream(outputFile));+ }} else {- fileList.add(outputFullPath);- LOG.info("Compressing: [{}]", outputFullPath);- //TarArchiveEntry tarEntry = new TarArchiveEntry(firstLevelDirName + File.separator + secondLevelDirName + File.separator + filename);- TarArchiveEntry tarEntry = new TarArchiveEntry(- outputRelativePath + File.separator + filename);- tarEntry.setSize(byteData.length);- tarOutput.putArchiveEntry(tarEntry);- tarOutput.write(byteData);- tarOutput.closeArchiveEntry();+ if (fileList.contains(outputFullPath)) {+ LOG.info("Skipping compressing: [{}]: file already exists", outputFullPath);+ } else {+ fileList.add(outputFullPath);+ LOG.info("Compressing: [{}]", outputFullPath);+ //TarArchiveEntry tarEntry = new TarArchiveEntry(firstLevelDirName + File.separator + secondLevelDirName + File.separator + filename);+ TarArchiveEntry tarEntry = new TarArchiveEntry(+ outputRelativePath + File.separator + filename);+ tarEntry.setSize(byteData.length);+ tarOutput.putArchiveEntry(tarEntry);+ tarOutput.write(byteData);+ tarOutput.closeArchiveEntry();+ }}}}}+ } catch (Exception e){+ LOG.warn("SKIPPED: {} Because : {}", segmentPart, e.getMessage());+ } finally {+ fs.close();}- reader.close();- } catch (Exception e){- LOG.warn("SKIPPED: {} Because : {}", segmentPart, e.getMessage());- } finally {- fs.close();}- }- if (gzip && !warc) {- closeStream();- }+ if (gzip && !warc) {+ closeStream();+ }- if (!typeCounts.isEmpty()) {- LOG.info("CommonsCrawlDataDumper File Stats: {}", DumpFileUtil- .displayFileTypes(typeCounts, filteredCounts));- }+ if (!typeCounts.isEmpty()) {+ LOG.info("CommonsCrawlDataDumper File Stats: {}", DumpFileUtil+ .displayFileTypes(typeCounts, filteredCounts));+ }+ }}private void closeStream() {diff --git a/src/java/org/apache/nutch/tools/FileDumper.java b/src/java/org/apache/nutch/tools/FileDumper.javaindex 8afc390c6..35d096351 100644--- a/src/java/org/apache/nutch/tools/FileDumper.java+++ b/src/java/org/apache/nutch/tools/FileDumper.java@@ -169,114 +169,107 @@ public class FileDumper {continue;}- SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(file));-- Writable key = (Writable) reader.getKeyClass().getConstructor().newInstance();- Content content = null;-- while (reader.next(key)) {- content = new Content();- reader.getCurrentValue(content);- String url = key.toString();- String baseName = FilenameUtils.getBaseName(url);- String extension = FilenameUtils.getExtension(url);- if (extension == null || (extension != null && extension.equals(""))) {- extension = "html";- }+ try (SequenceFile.Reader reader = new SequenceFile.Reader(conf,+ SequenceFile.Reader.file(file))) {++ Writable key = (Writable) reader.getKeyClass().getConstructor().newInstance();+ Content content = null;++ while (reader.next(key)) {+ content = new Content();+ reader.getCurrentValue(content);+ String url = key.toString();+ String baseName = FilenameUtils.getBaseName(url);+ String extension = FilenameUtils.getExtension(url);+ if (extension == null || (extension != null && extension.equals(""))) {+ extension = "html";+ }- ByteArrayInputStream bas = null;- Boolean filter = false;- try {- bas = new ByteArrayInputStream(content.getContent());- String mimeType = new Tika().detect(content.getContent());- collectStats(typeCounts, mimeType);- if (mimeType != null) {- if (mimeTypes == null- || Arrays.asList(mimeTypes).contains(mimeType)) {- collectStats(filteredCounts, mimeType);- filter = true;+ ByteArrayInputStream bas = null;+ Boolean filter = false;+ try {+ bas = new ByteArrayInputStream(content.getContent());+ String mimeType = new Tika().detect(content.getContent());+ collectStats(typeCounts, mimeType);+ if (mimeType != null) {+ if (mimeTypes == null+ || Arrays.asList(mimeTypes).contains(mimeType)) {+ collectStats(filteredCounts, mimeType);+ filter = true;+ }}- }- } catch (Exception e) {- e.printStackTrace();- LOG.warn("Tika is unable to detect type for: [{}]", url);- } finally {- if (bas != null) {- try {- bas.close();- } catch (Exception ignore) {+ } catch (Exception e) {+ e.printStackTrace();+ LOG.warn("Tika is unable to detect type for: [{}]", url);+ } finally {+ if (bas != null) {+ try {+ bas.close();+ } catch (Exception ignore) {+ }}}- }- if (filter) {- if (!mimeTypeStats) {- String md5Ofurl = DumpFileUtil.getUrlMD5(url);+ if (filter) {+ if (!mimeTypeStats) {+ String md5Ofurl = DumpFileUtil.getUrlMD5(url);- String fullDir = outputDir.getAbsolutePath();- if (!flatDir && !reverseURLDump) {- fullDir = DumpFileUtil.createTwoLevelsDirectory(fullDir, md5Ofurl);- }+ String fullDir = outputDir.getAbsolutePath();+ if (!flatDir && !reverseURLDump) {+ fullDir = DumpFileUtil.createTwoLevelsDirectory(fullDir, md5Ofurl);+ }- if (!Strings.isNullOrEmpty(fullDir)) {- String outputFullPath;+ if (!Strings.isNullOrEmpty(fullDir)) {+ String outputFullPath;- if (reverseURLDump) {- String[] reversedURL = TableUtil.reverseUrl(url).split(":");- reversedURL[0] = reversedURL[0].replace('.', '/');+ if (reverseURLDump) {+ String[] reversedURL = TableUtil.reverseUrl(url).split(":");+ reversedURL[0] = reversedURL[0].replace('.', '/');- String reversedURLPath = reversedURL[0] + "/"- + DigestUtils.sha256Hex(url).toUpperCase(Locale.ROOT);- outputFullPath = String.format(Locale.ROOT, "%s/%s",- fullDir, reversedURLPath);+ String reversedURLPath = reversedURL[0] + "/"+ + DigestUtils.sha256Hex(url).toUpperCase(Locale.ROOT);+ outputFullPath = String.format(Locale.ROOT, "%s/%s",+ fullDir, reversedURLPath);- // We'll drop the trailing file name and create the nested structure if it doesn't already exist.- String[] splitPath = outputFullPath.split("/");- File fullOutputDir = new File(org.apache.commons.lang3.StringUtils.join(Arrays.copyOf(splitPath, splitPath.length - 1), "/"));+ // We'll drop the trailing file name and create the nested structure if it doesn't already exist.+ String[] splitPath = outputFullPath.split("/");+ File fullOutputDir = new File(org.apache.commons.lang3.StringUtils.join(Arrays.copyOf(splitPath, splitPath.length - 1), "/"));- if (!fullOutputDir.exists()) {- if(!fullOutputDir.mkdirs());- throw new Exception("Unable to create: ["- + fullOutputDir.getAbsolutePath() + "]");+ if (!fullOutputDir.exists()) {+ if(!fullOutputDir.mkdirs());+ throw new Exception("Unable to create: ["+ + fullOutputDir.getAbsolutePath() + "]");+ }+ } else {+ outputFullPath = String.format(Locale.ROOT, "%s/%s",+ fullDir, DumpFileUtil.createFileName(md5Ofurl, baseName,+ extension));}- } else {- outputFullPath = String.format(Locale.ROOT, "%s/%s",- fullDir, DumpFileUtil.createFileName(md5Ofurl, baseName,- extension));- }- filenameToUrl.put(outputFullPath, url);- File outputFile = new File(outputFullPath);-- if (!outputFile.exists()) {- LOG.info("Writing: [{}]", outputFullPath);-- // Modified to prevent FileNotFoundException (Invalid Argument)- FileOutputStream output = null;- try {- output = new FileOutputStream(outputFile);- IOUtils.write(content.getContent(), output);- } catch (Exception e) {- LOG.warn("Write Error: [{}]", outputFullPath);- e.printStackTrace();- } finally {- if (output != null) {+ filenameToUrl.put(outputFullPath, url);+ File outputFile = new File(outputFullPath);++ if (!outputFile.exists()) {+ LOG.info("Writing: [{}]", outputFullPath);++ // Modified to prevent FileNotFoundException (Invalid Argument)+ try (FileOutputStream output = new FileOutputStream(+ outputFile)) {+ IOUtils.write(content.getContent(), output);output.flush();- try {- output.close();- } catch (Exception ignore) {- }+ } catch (Exception e) {+ LOG.warn("Write Error: [{}]", outputFullPath);+ e.printStackTrace();}+ fileCount++;+ } else {+ LOG.info("Skipping writing: [{}]: file already exists",+ outputFullPath);}- fileCount++;- } else {- LOG.info("Skipping writing: [{}]: file already exists",- outputFullPath);}}}}}- reader.close();} finally {if (doutputStream != null) {try {diff --git a/src/java/org/apache/nutch/tools/ResolveUrls.java b/src/java/org/apache/nutch/tools/ResolveUrls.javaindex ad38307a6..ddfa6cf21 100644--- a/src/java/org/apache/nutch/tools/ResolveUrls.java+++ b/src/java/org/apache/nutch/tools/ResolveUrls.java@@ -107,19 +107,19 @@ public class ResolveUrls {pool = Executors.newFixedThreadPool(numThreads);// read in the urls file and loop through each line, one url per line- BufferedReader buffRead = new BufferedReader(new FileReader(new File(- urlsFile), StandardCharsets.UTF_8));- String urlStr = null;- while ((urlStr = buffRead.readLine()) != null) {-- // spin up a resolver thread per url- LOG.info("Starting: {}", urlStr);- pool.execute(new ResolverThread(urlStr));+ try (BufferedReader buffRead = new BufferedReader(new FileReader(+ new File(urlsFile), StandardCharsets.UTF_8))) {+ String urlStr = null;+ while ((urlStr = buffRead.readLine()) != null) {++ // spin up a resolver thread per url+ LOG.info("Starting: {}", urlStr);+ pool.execute(new ResolverThread(urlStr));+ }}- // close the file and wait for up to 60 seconds before shutting down- // the thread pool to give urls time to finish resolving- buffRead.close();+ // wait for up to 60 seconds before shutting down the thread pool to+ // give urls time to finish resolvingpool.awaitTermination(60, TimeUnit.SECONDS);} catch (Exception e) {diff --git a/src/plugin/feed/src/java/org/apache/nutch/parse/feed/FeedParser.java b/src/plugin/feed/src/java/org/apache/nutch/parse/feed/FeedParser.javaindex a444f4312..720f5a6d1 100644--- a/src/plugin/feed/src/java/org/apache/nutch/parse/feed/FeedParser.java+++ b/src/plugin/feed/src/java/org/apache/nutch/parse/feed/FeedParser.java@@ -198,9 +198,9 @@ public class FeedParser implements Parser {parser.setConf(conf);File file = new File(name);byte[] bytes = new byte[(int) file.length()];- DataInputStream in = new DataInputStream(new FileInputStream(file));- in.readFully(bytes);- in.close();+ try (DataInputStream in = new DataInputStream(new FileInputStream(file))) {+ in.readFully(bytes);+ }ParseResult parseResult = parser.getParse(new Content(url, url, bytes,"application/rss+xml", new Metadata(), conf));for (Entry<Text, Parse> entry : parseResult) {diff --git a/src/plugin/parse-js/src/java/org/apache/nutch/parse/js/JSParseFilter.java b/src/plugin/parse-js/src/java/org/apache/nutch/parse/js/JSParseFilter.javaindex 172b2030a..8e83572f3 100644--- a/src/plugin/parse-js/src/java/org/apache/nutch/parse/js/JSParseFilter.java+++ b/src/plugin/parse-js/src/java/org/apache/nutch/parse/js/JSParseFilter.java@@ -18,7 +18,6 @@ package org.apache.nutch.parse.js;import java.io.BufferedReader;import java.io.FileInputStream;-import java.io.InputStream;import java.io.InputStreamReader;import java.lang.invoke.MethodHandles;import java.net.MalformedURLException;@@ -278,14 +277,14 @@ public class JSParseFilter implements HtmlParseFilter, Parser {System.err.println(JSParseFilter.class.getName() + " file.js baseURL");return;}- InputStream in = new FileInputStream(args[0]);- BufferedReader br = new BufferedReader(- new InputStreamReader(in, StandardCharsets.UTF_8));StringBuffer sb = new StringBuffer();- String line = null;- while ((line = br.readLine()) != null)- sb.append(line + "\n");- br.close();+ try (BufferedReader br = new BufferedReader(+ new InputStreamReader(new FileInputStream(args[0]),+ StandardCharsets.UTF_8))) {+ String line = null;+ while ((line = br.readLine()) != null)+ sb.append(line + "\n");+ }JSParseFilter parseFilter = new JSParseFilter();parseFilter.setConf(NutchConfiguration.create());diff --git a/src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipParser.java b/src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipParser.javaindex 315f7e79b..7dc08fd7b 100644--- a/src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipParser.java+++ b/src/plugin/parse-zip/src/java/org/apache/nutch/parse/zip/ZipParser.java@@ -124,10 +124,11 @@ public class ZipParser implements Parser {}File file = new File(args[0]);String url = "file:"+file.getCanonicalPath();- FileInputStream in = new FileInputStream(file);- byte[] bytes = new byte[in.available()];- in.read(bytes);- in.close();+ byte[] bytes;+ try (FileInputStream in = new FileInputStream(file)) {+ bytes = new byte[in.available()];+ in.read(bytes);+ }Configuration conf = NutchConfiguration.create();ZipParser parser = new ZipParser();parser.setConf(conf);diff --git a/src/plugin/parsefilter-naivebayes/src/java/org/apache/nutch/parsefilter/naivebayes/Classify.java b/src/plugin/parsefilter-naivebayes/src/java/org/apache/nutch/parsefilter/naivebayes/Classify.javaindex 4decb7a35..362c341cc 100644--- a/src/plugin/parsefilter-naivebayes/src/java/org/apache/nutch/parsefilter/naivebayes/Classify.java+++ b/src/plugin/parsefilter-naivebayes/src/java/org/apache/nutch/parsefilter/naivebayes/Classify.java@@ -70,23 +70,23 @@ public class Classify {Configuration configuration = new Configuration();FileSystem fs = FileSystem.get(configuration);- BufferedReader bufferedReader = new BufferedReader(new InputStreamReader(- fs.open(new Path("naivebayes-model")), StandardCharsets.UTF_8));-- uniquewords_size = Integer.parseInt(bufferedReader.readLine());- bufferedReader.readLine();-- numof_ir = Integer.parseInt(bufferedReader.readLine());- numwords_ir = Integer.parseInt(bufferedReader.readLine());- wordfreq_ir = unflattenToHashmap(bufferedReader.readLine());- bufferedReader.readLine();- numof_r = Integer.parseInt(bufferedReader.readLine());- numwords_r = Integer.parseInt(bufferedReader.readLine());- wordfreq_r = unflattenToHashmap(bufferedReader.readLine());-- ismodel = true;-- bufferedReader.close();+ try (BufferedReader bufferedReader = new BufferedReader(+ new InputStreamReader(fs.open(new Path("naivebayes-model")),+ StandardCharsets.UTF_8))) {++ uniquewords_size = Integer.parseInt(bufferedReader.readLine());+ bufferedReader.readLine();++ numof_ir = Integer.parseInt(bufferedReader.readLine());+ numwords_ir = Integer.parseInt(bufferedReader.readLine());+ wordfreq_ir = unflattenToHashmap(bufferedReader.readLine());+ bufferedReader.readLine();+ numof_r = Integer.parseInt(bufferedReader.readLine());+ numwords_r = Integer.parseInt(bufferedReader.readLine());+ wordfreq_r = unflattenToHashmap(bufferedReader.readLine());++ ismodel = true;+ }}diff --git a/src/plugin/parsefilter-naivebayes/src/java/org/apache/nutch/parsefilter/naivebayes/Train.java b/src/plugin/parsefilter-naivebayes/src/java/org/apache/nutch/parsefilter/naivebayes/Train.javaindex ab021334b..0b23705f4 100644--- a/src/plugin/parsefilter-naivebayes/src/java/org/apache/nutch/parsefilter/naivebayes/Train.java+++ b/src/plugin/parsefilter-naivebayes/src/java/org/apache/nutch/parsefilter/naivebayes/Train.java@@ -90,34 +90,36 @@ public class Train {Configuration configuration = new Configuration();FileSystem fs = FileSystem.get(configuration);- BufferedReader bufferedReader = new BufferedReader(- configuration.getConfResourceAsReader(filepath));+ try (BufferedReader bufferedReader = new BufferedReader(+ configuration.getConfResourceAsReader(filepath))) {- while ((line = bufferedReader.readLine()) != null) {+ while ((line = bufferedReader.readLine()) != null) {- target = line.split("\t")[0];+ target = line.split("\t")[0];- line = replacefirstoccuranceof(target + "\t", line);+ line = replacefirstoccuranceof(target + "\t", line);- linearray = line.replaceAll("[^a-zA-Z ]", "").toLowerCase(Locale.ROOT)- .split(" ");+ linearray = line.replaceAll("[^a-zA-Z ]", "").toLowerCase(Locale.ROOT)+ .split(" ");- // update the data structures- if (target.equals("0")) {+ // update the data structures+ if (target.equals("0")) {- numof_ir += 1;- numwords_ir += linearray.length;- for (int i = 0; i < linearray.length; i++) {- uniquewords.add(linearray[i]);- updateHashMap(wordfreq_ir, linearray[i]);- }- } else {+ numof_ir += 1;+ numwords_ir += linearray.length;+ for (int i = 0; i < linearray.length; i++) {+ uniquewords.add(linearray[i]);+ updateHashMap(wordfreq_ir, linearray[i]);+ }+ } else {++ numof_r += 1;+ numwords_r += linearray.length;+ for (int i = 0; i < linearray.length; i++) {+ uniquewords.add(linearray[i]);+ updateHashMap(wordfreq_r, linearray[i]);+ }- numof_r += 1;- numwords_r += linearray.length;- for (int i = 0; i < linearray.length; i++) {- uniquewords.add(linearray[i]);- updateHashMap(wordfreq_r, linearray[i]);}}@@ -128,22 +130,19 @@ public class Train {Path path = new Path("naivebayes-model");- Writer writer = new BufferedWriter(new OutputStreamWriter(fs.create(path,- true), StandardCharsets.UTF_8));-- writer.write(String.valueOf(uniquewords.size()) + "\n");- writer.write("0\n");- writer.write(String.valueOf(numof_ir) + "\n");- writer.write(String.valueOf(numwords_ir) + "\n");- writer.write(flattenHashMap(wordfreq_ir) + "\n");- writer.write("1\n");- writer.write(String.valueOf(numof_r) + "\n");- writer.write(String.valueOf(numwords_r) + "\n");- writer.write(flattenHashMap(wordfreq_r) + "\n");-- writer.close();-- bufferedReader.close();+ try (Writer writer = new BufferedWriter(new OutputStreamWriter(fs.create(+ path, true), StandardCharsets.UTF_8))) {++ writer.write(String.valueOf(uniquewords.size()) + "\n");+ writer.write("0\n");+ writer.write(String.valueOf(numof_ir) + "\n");+ writer.write(String.valueOf(numwords_ir) + "\n");+ writer.write(flattenHashMap(wordfreq_ir) + "\n");+ writer.write("1\n");+ writer.write(String.valueOf(numof_r) + "\n");+ writer.write(String.valueOf(numwords_r) + "\n");+ writer.write(flattenHashMap(wordfreq_r) + "\n");+ }}diff --git a/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java b/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.javaindex 266aab165..143ef3d1e 100644--- a/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java+++ b/src/plugin/protocol-file/src/java/org/apache/nutch/protocol/file/FileResponse.java@@ -204,17 +204,17 @@ public class FileResponse {this.content = new byte[len];- java.io.InputStream is = new java.io.FileInputStream(f);- int offset = 0;- int n = 0;- while (offset < len- && (n = is.read(this.content, offset, len - offset)) >= 0) {- offset += n;- }- if (offset < len) { // keep whatever already have, but issue a warning- File.LOG.warn("not enough bytes read from file: {}", f.getPath());+ try (java.io.InputStream is = new java.io.FileInputStream(f)) {+ int offset = 0;+ int n = 0;+ while (offset < len+ && (n = is.read(this.content, offset, len - offset)) >= 0) {+ offset += n;+ }+ if (offset < len) { // keep whatever already have, but issue a warning+ File.LOG.warn("not enough bytes read from file: {}", f.getPath());+ }}- is.close();// set headersheaders.set(Response.CONTENT_LENGTH, Long.valueOf(size).toString());