Skip to content

armgit / nutch

Public Imported from GitHub
0

Apache Nutch is an extensible and scalable web crawler

NUTCH-1807 Avoid methods relying on system-specific default locale / charset

Sebastian Nagelc180c0d2961f
diff --git a/src/java/org/apache/nutch/indexer/IndexerMapReduce.java b/src/java/org/apache/nutch/indexer/IndexerMapReduce.java
index 14ef3de67..448316e08 100644
--- a/src/java/org/apache/nutch/indexer/IndexerMapReduce.java
+++ b/src/java/org/apache/nutch/indexer/IndexerMapReduce.java
@@ -18,34 +18,33 @@ package org.apache.nutch.indexer;
import java.io.IOException;
import java.lang.invoke.MethodHandles;
+import java.nio.charset.Charset;
import java.util.Collection;
import java.util.Locale;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.commons.codec.binary.Base64;
import org.apache.commons.codec.binary.StringUtils;
-import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.mapreduce.Counter;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
import org.apache.nutch.crawl.CrawlDatum;
import org.apache.nutch.crawl.CrawlDb;
import org.apache.nutch.crawl.Inlinks;
import org.apache.nutch.crawl.LinkDb;
import org.apache.nutch.crawl.NutchWritable;
+import org.apache.nutch.metadata.Metadata;
+import org.apache.nutch.metadata.Nutch;
import org.apache.nutch.metrics.ErrorTracker;
import org.apache.nutch.metrics.LatencyTracker;
import org.apache.nutch.metrics.NutchMetrics;
-import org.apache.nutch.metadata.Metadata;
-import org.apache.nutch.metadata.Nutch;
import org.apache.nutch.net.URLFilters;
import org.apache.nutch.net.URLNormalizers;
import org.apache.nutch.parse.Parse;
@@ -56,6 +55,9 @@ import org.apache.nutch.parse.ParseText;
import org.apache.nutch.protocol.Content;
import org.apache.nutch.scoring.ScoringFilterException;
import org.apache.nutch.scoring.ScoringFilters;
+import org.apache.nutch.segment.SegmentReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* <p>
@@ -104,13 +106,13 @@ public class IndexerMapReduce extends Configured {
/**
* Normalizes and trims extra whitespace from the given url.
- *
+ *
* @param url
* The url to normalize.
- *
+ *
* @return The normalized url.
*/
- private static String normalizeUrl(String url, boolean normalize,
+ private static String normalizeUrl(String url, boolean normalize,
URLNormalizers urlNormalizers) {
if (!normalize) {
return url;
@@ -135,13 +137,13 @@ public class IndexerMapReduce extends Configured {
/**
* Filters the given url.
- *
+ *
* @param url
* The url to filter.
- *
+ *
* @return The filtered url or null.
*/
- private static String filterUrl(String url, boolean filter,
+ private static String filterUrl(String url, boolean filter,
URLFilters urlFilters) {
if (!filter) {
return url;
@@ -156,7 +158,7 @@ public class IndexerMapReduce extends Configured {
return url;
}
- public static class IndexerMapper extends
+ public static class IndexerMapper extends
Mapper<Text, Writable, Text, NutchWritable> {
// using normalizers and/or filters
@@ -170,25 +172,25 @@ public class IndexerMapReduce extends Configured {
@Override
public void setup(Mapper<Text, Writable, Text, NutchWritable>.Context context){
Configuration conf = context.getConfiguration();
-
+
normalize = conf.getBoolean(URL_NORMALIZING, false);
filter = conf.getBoolean(URL_FILTERING, false);
-
+
if (normalize) {
urlNormalizers = new URLNormalizers(conf,
URLNormalizers.SCOPE_INDEXER);
- }
+ }
if (filter) {
urlFilters = new URLFilters(conf);
- }
+ }
}
@Override
public void map(Text key, Writable value,
Context context) throws IOException, InterruptedException {
- String urlString = filterUrl(normalizeUrl(key.toString(), normalize,
+ String urlString = filterUrl(normalizeUrl(key.toString(), normalize,
urlNormalizers), filter, urlFilters);
if (urlString == null) {
return;
@@ -233,7 +235,7 @@ public class IndexerMapReduce extends Configured {
private Counter deletedByIndexingFilterCounter;
private Counter skippedByIndexingFilterCounter;
private Counter indexedCounter;
-
+
// Error tracker with cached counters
private ErrorTracker errorTracker;
@@ -423,7 +425,7 @@ public class IndexerMapReduce extends Configured {
// add digest, used by dedup
doc.add("digest", metadata.get(Nutch.SIGNATURE_KEY));
-
+
final Parse parse = new ParseImpl(parseText, parseData);
float boost = 1.0f;
// run scoring filters
@@ -444,7 +446,7 @@ public class IndexerMapReduce extends Configured {
if (dbDatum != null) {
// Indexing filters may also be interested in the signature
fetchDatum.setSignature(dbDatum.getSignature());
-
+
// extract information from dbDatum and pass it to
// fetchDatum so that indexing filters can use it
final Text url = (Text) dbDatum.getMetaData().get(
@@ -489,11 +491,13 @@ public class IndexerMapReduce extends Configured {
if (base64) {
// optionally encode as base64
// Note: we need a form which works with many versions of commons-code (1.4, 1.11 and upwards),
- // cf. NUTCH-2706. The following returns a chunked string for commons-coded 1.4:
+ // cf. NUTCH-2706. The following returns a chunked string for commons-code 1.4:
// binary = Base64.encodeBase64String(content.getContent());
binary = StringUtils.newStringUtf8(Base64.encodeBase64(content.getContent(), false, false));
} else {
- binary = new String(content.getContent());
+ // try to decode, fall-back UTF-8
+ Charset charset = SegmentReader.getCharset(parseData.getParseMeta());
+ binary = content.toString(charset);
}
doc.add("binaryContent", binary);
}
diff --git a/src/java/org/apache/nutch/protocol/RobotRulesParser.java b/src/java/org/apache/nutch/protocol/RobotRulesParser.java
index c5e267ef6..5b0412910 100644
--- a/src/java/org/apache/nutch/protocol/RobotRulesParser.java
+++ b/src/java/org/apache/nutch/protocol/RobotRulesParser.java
@@ -24,17 +24,16 @@ import java.io.LineNumberReader;
import java.lang.invoke.MethodHandles;
import java.net.MalformedURLException;
import java.net.URL;
+import java.nio.charset.StandardCharsets;
import java.util.Collection;
import java.util.HashSet;
import java.util.Hashtable;
import java.util.LinkedHashSet;
import java.util.LinkedList;
import java.util.List;
+import java.util.Locale;
import java.util.Set;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.util.StringUtils;
@@ -42,6 +41,8 @@ import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.nutch.util.NutchConfiguration;
import org.apache.nutch.util.SuffixStringMatcher;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import crawlercommons.robots.BaseRobotRules;
import crawlercommons.robots.SimpleRobotRules;
@@ -52,7 +53,7 @@ import crawlercommons.robots.SimpleRobotRulesParser;
* This class uses crawler-commons for handling the parsing of
* {@code robots.txt} files. It emits SimpleRobotRules objects, which describe
* the download permissions as described in SimpleRobotRulesParser.
- *
+ *
* Protocol-specific implementations have to implement the method
* {@link getRobotRulesSet}.
*/
@@ -62,7 +63,7 @@ public abstract class RobotRulesParser implements Tool {
.getLogger(MethodHandles.lookup().lookupClass());
protected static final Hashtable<String, BaseRobotRules> CACHE = new Hashtable<>();
-
+
/**
* A {@link BaseRobotRules} object appropriate for use when the
* {@code robots.txt} file is empty or missing; all requests are allowed.
@@ -102,7 +103,7 @@ public abstract class RobotRulesParser implements Tool {
/** set of host names or IPs to be explicitly excluded from robots.txt checking */
protected Set<String> allowList = new HashSet<>();
-
+
/* Matcher user for efficiently matching URLs against a set of suffixes. */
private SuffixStringMatcher matcher = null;
@@ -131,7 +132,7 @@ public abstract class RobotRulesParser implements Tool {
* skip wildcard "*" - crawler-commons' SimpleRobotRulesParser expects an
* empty set of agent names to use the wildcard rules
*/
- agentNames.add(agentName.toLowerCase());
+ agentNames.add(agentName.toLowerCase(Locale.ROOT));
}
// If there are any other agents specified, append those to the list of
@@ -139,7 +140,7 @@ public abstract class RobotRulesParser implements Tool {
String[] otherAgents = conf.getStrings("http.robots.agents");
if (otherAgents != null && otherAgents.length > 0) {
for (String otherAgent : otherAgents) {
- otherAgent = otherAgent.toLowerCase();
+ otherAgent = otherAgent.toLowerCase(Locale.ROOT);
if (otherAgent.equals("*") || otherAgent.equalsIgnoreCase(agentName)) {
/*
* skip wildcard "*" or agent name itself (required for backward
@@ -166,7 +167,7 @@ public abstract class RobotRulesParser implements Tool {
}
allowList.add(confAllowList[i]);
}
-
+
if (allowList.size() > 0) {
matcher = new SuffixStringMatcher(allowList);
LOG.info("Allowlisted hosts: {}", allowList);
@@ -190,18 +191,18 @@ public abstract class RobotRulesParser implements Tool {
public boolean isAllowListed(URL url) {
boolean match = false;
String urlString = url.getHost();
-
+
if (matcher != null) {
match = matcher.matches(urlString);
}
-
+
return match;
}
/**
* Parses the robots content using the {@link SimpleRobotRulesParser} from
* crawler-commons
- *
+ *
* @param url
* The robots.txt URL
* @param content
@@ -222,7 +223,7 @@ public abstract class RobotRulesParser implements Tool {
/**
* Parses the robots content using the {@link SimpleRobotRulesParser} from
* crawler-commons
- *
+ *
* @param url
* The robots.txt URL
* @param content
@@ -397,7 +398,8 @@ public abstract class RobotRulesParser implements Tool {
System.out.println("Testing robots.txt for agent names: "
+ (agentNames.isEmpty() ? "* (any other agent)" : agentNames));
- LineNumberReader testsIn = new LineNumberReader(new FileReader(urlFile));
+ LineNumberReader testsIn = new LineNumberReader(
+ new FileReader(urlFile, StandardCharsets.UTF_8));
String testPath;
testPath = testsIn.readLine();
while (testPath != null) {
@@ -440,7 +442,7 @@ public abstract class RobotRulesParser implements Tool {
/*
* Make sure that agent name is not empty so that
* RobotRulesParser.setConf() does not complain.
- *
+ *
* If provided the agent names passed as command-line argument are
* checked, see RobotRulesParser.run(...). Also http.agent.name is then
* filled taking the first agent name from command-line.
diff --git a/src/java/org/apache/nutch/segment/SegmentReader.java b/src/java/org/apache/nutch/segment/SegmentReader.java
index 3b386d24e..817f6bcfb 100644
--- a/src/java/org/apache/nutch/segment/SegmentReader.java
+++ b/src/java/org/apache/nutch/segment/SegmentReader.java
@@ -38,29 +38,27 @@ import java.util.List;
import java.util.Map;
import java.util.Random;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.io.MapFile;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.io.MapFile;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.io.WritableComparable;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
-import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
import org.apache.hadoop.mapreduce.Job;
-import org.apache.hadoop.mapreduce.lib.output.MapFileOutputFormat;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.RecordWriter;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.TaskAttemptContext;
+import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.input.SequenceFileInputFormat;
+import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
+import org.apache.hadoop.mapreduce.lib.output.MapFileOutputFormat;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
import org.apache.nutch.crawl.CrawlDatum;
@@ -73,6 +71,8 @@ import org.apache.nutch.util.HadoopFSUtil;
import org.apache.nutch.util.NutchConfiguration;
import org.apache.nutch.util.NutchJob;
import org.apache.nutch.util.SegmentReaderUtil;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/** Dump the content of a segment. */
public class SegmentReader extends Configured implements Tool {
@@ -539,7 +539,7 @@ public class SegmentReader extends Configured implements Tool {
/**
* Try to get HTML encoding from parse metadata. Try
* {@link Metadata#CHAR_ENCODING_FOR_CONVERSION}, then
- * {@link Metadata#CONTENT_ENCODING} then fallback
+ * {@link Metadata#CONTENT_ENCODING} then fall back to
* {@link java.nio.charset.StandardCharsets#UTF_8}
* @param parseMeta a populated {@link Metadata}
* @return {@link Charset}
diff --git a/src/java/org/apache/nutch/tools/AbstractCommonCrawlFormat.java b/src/java/org/apache/nutch/tools/AbstractCommonCrawlFormat.java
index 00e293900..b31fa3d5a 100644
--- a/src/java/org/apache/nutch/tools/AbstractCommonCrawlFormat.java
+++ b/src/java/org/apache/nutch/tools/AbstractCommonCrawlFormat.java
@@ -37,7 +37,7 @@ import org.slf4j.LoggerFactory;
import com.ibm.icu.text.SimpleDateFormat;
/**
- * Abstract class that implements { @see org.apache.nutch.tools.CommonCrawlFormat } interface.
+ * Abstract class that implements { @see org.apache.nutch.tools.CommonCrawlFormat } interface.
*
*/
public abstract class AbstractCommonCrawlFormat implements CommonCrawlFormat {
@@ -330,7 +330,7 @@ public abstract class AbstractCommonCrawlFormat implements CommonCrawlFormat {
}
protected String getResponseContent() {
- return new String(content.getContent());
+ return content.toString();
}
protected String getKey() {