Skip to content

armgit / nutch

Public Imported from GitHub
0

Apache Nutch is an extensible and scalable web crawler

NUTCH-1564: fix immediate refetch for pages not modified

Isabelle Giguered5dccfb0c3eb
diff --git a/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java b/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java
index 6575ccb88..38e3162b1 100644
--- a/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java
+++ b/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java
@@ -219,15 +219,15 @@ public class AdaptiveFetchSchedule extends AbstractFetchSchedule {
// The custom intervals should respect the boundaries of the default values.
if (m < defaultMin) {
LOG.error(
- "Min. interval out of bounds on line {} in the config. file: `{}`",
- lineNo, line);
+ "Min. interval out of bounds ({}) on line {} in the config. file: `{}`",
+ defaultMin, lineNo, line);
continue;
}
if (M > defaultMax) {
LOG.error(
- "Max. interval out of bounds on line {} in the config. file: `{}`",
- lineNo, line);
+ "Max. interval out of bounds ({}) on line {} in the config. file: `{}`",
+ defaultMax, lineNo, line);
continue;
}
@@ -338,6 +338,10 @@ public class AdaptiveFetchSchedule extends AbstractFetchSchedule {
if (delta > interval)
interval = delta;
refTime = fetchTime - Math.round(delta * SYNC_DELTA_RATE * 1000);
+ // make sure we are not in the past
+ if (refTime < fetchTime) {
+ refTime = fetchTime;
+ }
}
// Ensure the interval does not fall outside of bounds
@@ -389,7 +393,8 @@ public class AdaptiveFetchSchedule extends AbstractFetchSchedule {
(p.getFetchInterval() / SECONDS_PER_DAY), miss);
if (p.getFetchTime() <= curTime) {
fetchCnt++;
- fs.setFetchSchedule(new Text("http://www.example.com"), p, p
+ // why was "http://www.example.com" hard-coded here?
+ fs.setFetchSchedule(new Text(""), p, p
.getFetchTime(), p.getModifiedTime(), curTime, lastModified,
changed ? FetchSchedule.STATUS_MODIFIED
: FetchSchedule.STATUS_NOTMODIFIED);
diff --git a/src/java/org/apache/nutch/fetcher/FetcherThread.java b/src/java/org/apache/nutch/fetcher/FetcherThread.java
index 2abcfe9f6..62ab5c1b8 100644
--- a/src/java/org/apache/nutch/fetcher/FetcherThread.java
+++ b/src/java/org/apache/nutch/fetcher/FetcherThread.java
@@ -389,7 +389,7 @@ public class FetcherThread extends Thread {
}
continue;
}
- if (!rules.isAllowed(fit.u)) {
+ if (!rules.isAllowed(fit.u.toString())) {
// unblock
fetchQueues.finishFetchItem(fit, true);
LOG.info("Denied by robots.txt: {}", fit.url);
diff --git a/src/test/org/apache/nutch/crawl/TestAdaptiveFetchSchedule.java b/src/test/org/apache/nutch/crawl/TestAdaptiveFetchSchedule.java
index 377d49ec8..2ae06ecff 100644
--- a/src/test/org/apache/nutch/crawl/TestAdaptiveFetchSchedule.java
+++ b/src/test/org/apache/nutch/crawl/TestAdaptiveFetchSchedule.java
@@ -24,6 +24,12 @@ import org.junit.jupiter.api.Test;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+
+import java.time.Duration;
+import java.time.Instant;
+import java.util.Date;
+import java.util.Properties;
/**
* Test cases for AdaptiveFetchSchedule.
@@ -36,6 +42,8 @@ public class TestAdaptiveFetchSchedule {
private Configuration conf;
private long curTime, lastModified;
private int changed, interval, calculateInterval;
+
+ private static final long ONE_DAY = 86400;
@BeforeEach
public void setUp() throws Exception {
@@ -117,5 +125,49 @@ public class TestAdaptiveFetchSchedule {
}
}
+
+ /**
+ * Test https://issues.apache.org/jira/browse/NUTCH-1564
+ */
+ @Test
+ public void testSetFetchSchedule() {
+ conf.set("db.fetch.schedule.class", "org.apache.nutch.crawl.AdaptiveFetchSchedule");
+ conf.set("db.fetch.schedule.adaptive.sync_delta", "true"); // default
+ conf.set("db.fetch.schedule.adaptive.sync_delta_rate", "0.3"); // default
+ conf.set("db.fetch.interval.default", String.valueOf(ONE_DAY * 2)); // 2 days
+ conf.set("db.fetch.schedule.adaptive.min_interval", String.valueOf(ONE_DAY)); // 1 day
+ conf.set("db.fetch.schedule.adaptive.max_interval", String.valueOf(ONE_DAY * 7)); // 7 days
+ conf.set("db.fetch.interval.max", String.valueOf(ONE_DAY * 7)); // 7 days
+
+ // ignore adaptive-host-specific-intervals.txt
+ Text url = new Text("http://www.example2.com");
+
+ AdaptiveFetchSchedule fs = new AdaptiveFetchSchedule();
+ fs.setConf(conf);
+
+ CrawlDatum datum = prepareCrawlDatum();
+ Date fetchTime = Date.from(Instant.now());
+ // previous fetch 3 days ago
+ Date previousFetchTime = Date.from(Instant.now().minus(Duration.ofDays(3)));
+ // last modified 1 month ago
+ Date modifiedTime = Date.from(Instant.now().minus(Duration.ofDays(30)));
+ datum.setStatus(CrawlDatum.STATUS_FETCH_SUCCESS);
+ datum.setRetriesSinceFetch(0);
+ datum.setModifiedTime(modifiedTime.getTime());
+ datum.setFetchTime(fetchTime.getTime());
+
+ System.out.println("CrawlDatum fetchTime: " + fetchTime + "; modifiedTime: " + modifiedTime);
+
+ fs.setFetchSchedule(url, datum, previousFetchTime.getTime(), modifiedTime.getTime(),
+ fetchTime.getTime(), modifiedTime.getTime(), CrawlDatum.STATUS_DB_NOTMODIFIED);
+
+ Date nextFetchTime = new Date(datum.getFetchTime());
+ System.out.println("CrawlDatum next fetchTime: " + nextFetchTime);
+
+ assertTrue(nextFetchTime.after(fetchTime));
+ // adapt milliseconds to seconds
+ assertTrue((nextFetchTime.getTime() - fetchTime.getTime()) / 1000 >= ONE_DAY);
+ assertTrue((nextFetchTime.getTime() - fetchTime.getTime()) / 1000 <= ONE_DAY * 7);
+ }
}