Apache Nutch is an extensible and scalable web crawler
NUTCH-1807 Avoid methods relying on system-specific default locale / charset
Sebastian Nagelc180c0d2961f
diff --git a/src/java/org/apache/nutch/indexer/IndexerMapReduce.java b/src/java/org/apache/nutch/indexer/IndexerMapReduce.javaindex 14ef3de67..448316e08 100644--- a/src/java/org/apache/nutch/indexer/IndexerMapReduce.java+++ b/src/java/org/apache/nutch/indexer/IndexerMapReduce.java@@ -18,34 +18,33 @@ package org.apache.nutch.indexer;import java.io.IOException;import java.lang.invoke.MethodHandles;+import java.nio.charset.Charset;import java.util.Collection;import java.util.Locale;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;import org.apache.commons.codec.binary.Base64;import org.apache.commons.codec.binary.StringUtils;-import org.apache.hadoop.conf.Configured;import org.apache.hadoop.conf.Configuration;+import org.apache.hadoop.conf.Configured;import org.apache.hadoop.fs.Path;import org.apache.hadoop.io.Text;import org.apache.hadoop.io.Writable;import org.apache.hadoop.mapreduce.Counter;-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.Job;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.Reducer;+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;import org.apache.nutch.crawl.CrawlDatum;import org.apache.nutch.crawl.CrawlDb;import org.apache.nutch.crawl.Inlinks;import org.apache.nutch.crawl.LinkDb;import org.apache.nutch.crawl.NutchWritable;+import org.apache.nutch.metadata.Metadata;+import org.apache.nutch.metadata.Nutch;import org.apache.nutch.metrics.ErrorTracker;import org.apache.nutch.metrics.LatencyTracker;import org.apache.nutch.metrics.NutchMetrics;-import org.apache.nutch.metadata.Metadata;-import org.apache.nutch.metadata.Nutch;import org.apache.nutch.net.URLFilters;import org.apache.nutch.net.URLNormalizers;import org.apache.nutch.parse.Parse;@@ -56,6 +55,9 @@ import org.apache.nutch.parse.ParseText;import org.apache.nutch.protocol.Content;import org.apache.nutch.scoring.ScoringFilterException;import org.apache.nutch.scoring.ScoringFilters;+import org.apache.nutch.segment.SegmentReader;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;/*** <p>@@ -104,13 +106,13 @@ public class IndexerMapReduce extends Configured {/*** Normalizes and trims extra whitespace from the given url.- *+ ** @param url* The url to normalize.- *+ ** @return The normalized url.*/- private static String normalizeUrl(String url, boolean normalize,+ private static String normalizeUrl(String url, boolean normalize,URLNormalizers urlNormalizers) {if (!normalize) {return url;@@ -135,13 +137,13 @@ public class IndexerMapReduce extends Configured {/*** Filters the given url.- *+ ** @param url* The url to filter.- *+ ** @return The filtered url or null.*/- private static String filterUrl(String url, boolean filter,+ private static String filterUrl(String url, boolean filter,URLFilters urlFilters) {if (!filter) {return url;@@ -156,7 +158,7 @@ public class IndexerMapReduce extends Configured {return url;}- public static class IndexerMapper extends+ public static class IndexerMapper extendsMapper<Text, Writable, Text, NutchWritable> {// using normalizers and/or filters@@ -170,25 +172,25 @@ public class IndexerMapReduce extends Configured {@Overridepublic void setup(Mapper<Text, Writable, Text, NutchWritable>.Context context){Configuration conf = context.getConfiguration();-+normalize = conf.getBoolean(URL_NORMALIZING, false);filter = conf.getBoolean(URL_FILTERING, false);-+if (normalize) {urlNormalizers = new URLNormalizers(conf,URLNormalizers.SCOPE_INDEXER);- }+ }if (filter) {urlFilters = new URLFilters(conf);- }+ }}@Overridepublic void map(Text key, Writable value,Context context) throws IOException, InterruptedException {- String urlString = filterUrl(normalizeUrl(key.toString(), normalize,+ String urlString = filterUrl(normalizeUrl(key.toString(), normalize,urlNormalizers), filter, urlFilters);if (urlString == null) {return;@@ -233,7 +235,7 @@ public class IndexerMapReduce extends Configured {private Counter deletedByIndexingFilterCounter;private Counter skippedByIndexingFilterCounter;private Counter indexedCounter;-+// Error tracker with cached countersprivate ErrorTracker errorTracker;@@ -423,7 +425,7 @@ public class IndexerMapReduce extends Configured {// add digest, used by dedupdoc.add("digest", metadata.get(Nutch.SIGNATURE_KEY));-+final Parse parse = new ParseImpl(parseText, parseData);float boost = 1.0f;// run scoring filters@@ -444,7 +446,7 @@ public class IndexerMapReduce extends Configured {if (dbDatum != null) {// Indexing filters may also be interested in the signaturefetchDatum.setSignature(dbDatum.getSignature());-+// extract information from dbDatum and pass it to// fetchDatum so that indexing filters can use itfinal Text url = (Text) dbDatum.getMetaData().get(@@ -489,11 +491,13 @@ public class IndexerMapReduce extends Configured {if (base64) {// optionally encode as base64// Note: we need a form which works with many versions of commons-code (1.4, 1.11 and upwards),- // cf. NUTCH-2706. The following returns a chunked string for commons-coded 1.4:+ // cf. NUTCH-2706. The following returns a chunked string for commons-code 1.4:// binary = Base64.encodeBase64String(content.getContent());binary = StringUtils.newStringUtf8(Base64.encodeBase64(content.getContent(), false, false));} else {- binary = new String(content.getContent());+ // try to decode, fall-back UTF-8+ Charset charset = SegmentReader.getCharset(parseData.getParseMeta());+ binary = content.toString(charset);}doc.add("binaryContent", binary);}diff --git a/src/java/org/apache/nutch/protocol/RobotRulesParser.java b/src/java/org/apache/nutch/protocol/RobotRulesParser.javaindex c5e267ef6..5b0412910 100644--- a/src/java/org/apache/nutch/protocol/RobotRulesParser.java+++ b/src/java/org/apache/nutch/protocol/RobotRulesParser.java@@ -24,17 +24,16 @@ import java.io.LineNumberReader;import java.lang.invoke.MethodHandles;import java.net.MalformedURLException;import java.net.URL;+import java.nio.charset.StandardCharsets;import java.util.Collection;import java.util.HashSet;import java.util.Hashtable;import java.util.LinkedHashSet;import java.util.LinkedList;import java.util.List;+import java.util.Locale;import java.util.Set;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;-import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.io.Text;import org.apache.hadoop.util.StringUtils;@@ -42,6 +41,8 @@ import org.apache.hadoop.util.Tool;import org.apache.hadoop.util.ToolRunner;import org.apache.nutch.util.NutchConfiguration;import org.apache.nutch.util.SuffixStringMatcher;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;import crawlercommons.robots.BaseRobotRules;import crawlercommons.robots.SimpleRobotRules;@@ -52,7 +53,7 @@ import crawlercommons.robots.SimpleRobotRulesParser;* This class uses crawler-commons for handling the parsing of* {@code robots.txt} files. It emits SimpleRobotRules objects, which describe* the download permissions as described in SimpleRobotRulesParser.- *+ ** Protocol-specific implementations have to implement the method* {@link getRobotRulesSet}.*/@@ -62,7 +63,7 @@ public abstract class RobotRulesParser implements Tool {.getLogger(MethodHandles.lookup().lookupClass());protected static final Hashtable<String, BaseRobotRules> CACHE = new Hashtable<>();-+/*** A {@link BaseRobotRules} object appropriate for use when the* {@code robots.txt} file is empty or missing; all requests are allowed.@@ -102,7 +103,7 @@ public abstract class RobotRulesParser implements Tool {/** set of host names or IPs to be explicitly excluded from robots.txt checking */protected Set<String> allowList = new HashSet<>();-+/* Matcher user for efficiently matching URLs against a set of suffixes. */private SuffixStringMatcher matcher = null;@@ -131,7 +132,7 @@ public abstract class RobotRulesParser implements Tool {* skip wildcard "*" - crawler-commons' SimpleRobotRulesParser expects an* empty set of agent names to use the wildcard rules*/- agentNames.add(agentName.toLowerCase());+ agentNames.add(agentName.toLowerCase(Locale.ROOT));}// If there are any other agents specified, append those to the list of@@ -139,7 +140,7 @@ public abstract class RobotRulesParser implements Tool {String[] otherAgents = conf.getStrings("http.robots.agents");if (otherAgents != null && otherAgents.length > 0) {for (String otherAgent : otherAgents) {- otherAgent = otherAgent.toLowerCase();+ otherAgent = otherAgent.toLowerCase(Locale.ROOT);if (otherAgent.equals("*") || otherAgent.equalsIgnoreCase(agentName)) {/** skip wildcard "*" or agent name itself (required for backward@@ -166,7 +167,7 @@ public abstract class RobotRulesParser implements Tool {}allowList.add(confAllowList[i]);}-+if (allowList.size() > 0) {matcher = new SuffixStringMatcher(allowList);LOG.info("Allowlisted hosts: {}", allowList);@@ -190,18 +191,18 @@ public abstract class RobotRulesParser implements Tool {public boolean isAllowListed(URL url) {boolean match = false;String urlString = url.getHost();-+if (matcher != null) {match = matcher.matches(urlString);}-+return match;}/*** Parses the robots content using the {@link SimpleRobotRulesParser} from* crawler-commons- *+ ** @param url* The robots.txt URL* @param content@@ -222,7 +223,7 @@ public abstract class RobotRulesParser implements Tool {/*** Parses the robots content using the {@link SimpleRobotRulesParser} from* crawler-commons- *+ ** @param url* The robots.txt URL* @param content@@ -397,7 +398,8 @@ public abstract class RobotRulesParser implements Tool {System.out.println("Testing robots.txt for agent names: "+ (agentNames.isEmpty() ? "* (any other agent)" : agentNames));- LineNumberReader testsIn = new LineNumberReader(new FileReader(urlFile));+ LineNumberReader testsIn = new LineNumberReader(+ new FileReader(urlFile, StandardCharsets.UTF_8));String testPath;testPath = testsIn.readLine();while (testPath != null) {@@ -440,7 +442,7 @@ public abstract class RobotRulesParser implements Tool {/** Make sure that agent name is not empty so that* RobotRulesParser.setConf() does not complain.- *+ ** If provided the agent names passed as command-line argument are* checked, see RobotRulesParser.run(...). Also http.agent.name is then* filled taking the first agent name from command-line.diff --git a/src/java/org/apache/nutch/segment/SegmentReader.java b/src/java/org/apache/nutch/segment/SegmentReader.javaindex 3b386d24e..817f6bcfb 100644--- a/src/java/org/apache/nutch/segment/SegmentReader.java+++ b/src/java/org/apache/nutch/segment/SegmentReader.java@@ -38,29 +38,27 @@ import java.util.List;import java.util.Map;import java.util.Random;-import org.slf4j.Logger;-import org.slf4j.LoggerFactory;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.conf.Configured;import org.apache.hadoop.fs.FileStatus;import org.apache.hadoop.fs.FileSystem;import org.apache.hadoop.fs.Path;-import org.apache.hadoop.io.MapFile;import org.apache.hadoop.io.IOUtils;+import org.apache.hadoop.io.MapFile;import org.apache.hadoop.io.SequenceFile;import org.apache.hadoop.io.Text;import org.apache.hadoop.io.Writable;import org.apache.hadoop.io.WritableComparable;-import org.apache.hadoop.util.StringUtils;-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;import org.apache.hadoop.mapreduce.Job;-import org.apache.hadoop.mapreduce.lib.output.MapFileOutputFormat;import org.apache.hadoop.mapreduce.Mapper;import org.apache.hadoop.mapreduce.RecordWriter;import org.apache.hadoop.mapreduce.Reducer;import org.apache.hadoop.mapreduce.TaskAttemptContext;+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;+import org.apache.hadoop.mapreduce.lib.output.MapFileOutputFormat;+import org.apache.hadoop.util.StringUtils;import org.apache.hadoop.util.Tool;import org.apache.hadoop.util.ToolRunner;import org.apache.nutch.crawl.CrawlDatum;@@ -73,6 +71,8 @@ import org.apache.nutch.util.HadoopFSUtil;import org.apache.nutch.util.NutchConfiguration;import org.apache.nutch.util.NutchJob;import org.apache.nutch.util.SegmentReaderUtil;+import org.slf4j.Logger;+import org.slf4j.LoggerFactory;/** Dump the content of a segment. */public class SegmentReader extends Configured implements Tool {@@ -539,7 +539,7 @@ public class SegmentReader extends Configured implements Tool {/*** Try to get HTML encoding from parse metadata. Try* {@link Metadata#CHAR_ENCODING_FOR_CONVERSION}, then- * {@link Metadata#CONTENT_ENCODING} then fallback+ * {@link Metadata#CONTENT_ENCODING} then fall back to* {@link java.nio.charset.StandardCharsets#UTF_8}* @param parseMeta a populated {@link Metadata}* @return {@link Charset}diff --git a/src/java/org/apache/nutch/tools/AbstractCommonCrawlFormat.java b/src/java/org/apache/nutch/tools/AbstractCommonCrawlFormat.javaindex 00e293900..b31fa3d5a 100644--- a/src/java/org/apache/nutch/tools/AbstractCommonCrawlFormat.java+++ b/src/java/org/apache/nutch/tools/AbstractCommonCrawlFormat.java@@ -37,7 +37,7 @@ import org.slf4j.LoggerFactory;import com.ibm.icu.text.SimpleDateFormat;/**- * Abstract class that implements { @see org.apache.nutch.tools.CommonCrawlFormat } interface.+ * Abstract class that implements { @see org.apache.nutch.tools.CommonCrawlFormat } interface.**/public abstract class AbstractCommonCrawlFormat implements CommonCrawlFormat {@@ -330,7 +330,7 @@ public abstract class AbstractCommonCrawlFormat implements CommonCrawlFormat {}protected String getResponseContent() {- return new String(content.getContent());+ return content.toString();}protected String getKey() {