Apache Nutch is an extensible and scalable web crawler
NUTCH-1564: fix immediate refetch for pages not modified
Isabelle Giguered5dccfb0c3eb
diff --git a/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java b/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.javaindex 6575ccb88..38e3162b1 100644--- a/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java+++ b/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java@@ -219,15 +219,15 @@ public class AdaptiveFetchSchedule extends AbstractFetchSchedule {// The custom intervals should respect the boundaries of the default values.if (m < defaultMin) {LOG.error(- "Min. interval out of bounds on line {} in the config. file: `{}`",- lineNo, line);+ "Min. interval out of bounds ({}) on line {} in the config. file: `{}`",+ defaultMin, lineNo, line);continue;}if (M > defaultMax) {LOG.error(- "Max. interval out of bounds on line {} in the config. file: `{}`",- lineNo, line);+ "Max. interval out of bounds ({}) on line {} in the config. file: `{}`",+ defaultMax, lineNo, line);continue;}@@ -338,6 +338,10 @@ public class AdaptiveFetchSchedule extends AbstractFetchSchedule {if (delta > interval)interval = delta;refTime = fetchTime - Math.round(delta * SYNC_DELTA_RATE * 1000);+ // make sure we are not in the past+ if (refTime < fetchTime) {+ refTime = fetchTime;+ }}// Ensure the interval does not fall outside of bounds@@ -389,7 +393,8 @@ public class AdaptiveFetchSchedule extends AbstractFetchSchedule {(p.getFetchInterval() / SECONDS_PER_DAY), miss);if (p.getFetchTime() <= curTime) {fetchCnt++;- fs.setFetchSchedule(new Text("http://www.example.com"), p, p+ // why was "http://www.example.com" hard-coded here?+ fs.setFetchSchedule(new Text(""), p, p.getFetchTime(), p.getModifiedTime(), curTime, lastModified,changed ? FetchSchedule.STATUS_MODIFIED: FetchSchedule.STATUS_NOTMODIFIED);diff --git a/src/java/org/apache/nutch/fetcher/FetcherThread.java b/src/java/org/apache/nutch/fetcher/FetcherThread.javaindex 2abcfe9f6..62ab5c1b8 100644--- a/src/java/org/apache/nutch/fetcher/FetcherThread.java+++ b/src/java/org/apache/nutch/fetcher/FetcherThread.java@@ -389,7 +389,7 @@ public class FetcherThread extends Thread {}continue;}- if (!rules.isAllowed(fit.u)) {+ if (!rules.isAllowed(fit.u.toString())) {// unblockfetchQueues.finishFetchItem(fit, true);LOG.info("Denied by robots.txt: {}", fit.url);diff --git a/src/test/org/apache/nutch/crawl/TestAdaptiveFetchSchedule.java b/src/test/org/apache/nutch/crawl/TestAdaptiveFetchSchedule.javaindex 377d49ec8..2ae06ecff 100644--- a/src/test/org/apache/nutch/crawl/TestAdaptiveFetchSchedule.java+++ b/src/test/org/apache/nutch/crawl/TestAdaptiveFetchSchedule.java@@ -24,6 +24,12 @@ import org.junit.jupiter.api.Test;import static org.hamcrest.CoreMatchers.is;import static org.hamcrest.MatcherAssert.assertThat;+import static org.junit.jupiter.api.Assertions.assertTrue;++import java.time.Duration;+import java.time.Instant;+import java.util.Date;+import java.util.Properties;/*** Test cases for AdaptiveFetchSchedule.@@ -36,6 +42,8 @@ public class TestAdaptiveFetchSchedule {private Configuration conf;private long curTime, lastModified;private int changed, interval, calculateInterval;++ private static final long ONE_DAY = 86400;@BeforeEachpublic void setUp() throws Exception {@@ -117,5 +125,49 @@ public class TestAdaptiveFetchSchedule {}}++ /**+ * Test https://issues.apache.org/jira/browse/NUTCH-1564+ */+ @Test+ public void testSetFetchSchedule() {+ conf.set("db.fetch.schedule.class", "org.apache.nutch.crawl.AdaptiveFetchSchedule");+ conf.set("db.fetch.schedule.adaptive.sync_delta", "true"); // default+ conf.set("db.fetch.schedule.adaptive.sync_delta_rate", "0.3"); // default+ conf.set("db.fetch.interval.default", String.valueOf(ONE_DAY * 2)); // 2 days+ conf.set("db.fetch.schedule.adaptive.min_interval", String.valueOf(ONE_DAY)); // 1 day+ conf.set("db.fetch.schedule.adaptive.max_interval", String.valueOf(ONE_DAY * 7)); // 7 days+ conf.set("db.fetch.interval.max", String.valueOf(ONE_DAY * 7)); // 7 days++ // ignore adaptive-host-specific-intervals.txt+ Text url = new Text("http://www.example2.com");++ AdaptiveFetchSchedule fs = new AdaptiveFetchSchedule();+ fs.setConf(conf);++ CrawlDatum datum = prepareCrawlDatum();+ Date fetchTime = Date.from(Instant.now());+ // previous fetch 3 days ago+ Date previousFetchTime = Date.from(Instant.now().minus(Duration.ofDays(3)));+ // last modified 1 month ago+ Date modifiedTime = Date.from(Instant.now().minus(Duration.ofDays(30)));+ datum.setStatus(CrawlDatum.STATUS_FETCH_SUCCESS);+ datum.setRetriesSinceFetch(0);+ datum.setModifiedTime(modifiedTime.getTime());+ datum.setFetchTime(fetchTime.getTime());++ System.out.println("CrawlDatum fetchTime: " + fetchTime + "; modifiedTime: " + modifiedTime);++ fs.setFetchSchedule(url, datum, previousFetchTime.getTime(), modifiedTime.getTime(),+ fetchTime.getTime(), modifiedTime.getTime(), CrawlDatum.STATUS_DB_NOTMODIFIED);++ Date nextFetchTime = new Date(datum.getFetchTime());+ System.out.println("CrawlDatum next fetchTime: " + nextFetchTime);++ assertTrue(nextFetchTime.after(fetchTime));+ // adapt milliseconds to seconds+ assertTrue((nextFetchTime.getTime() - fetchTime.getTime()) / 1000 >= ONE_DAY);+ assertTrue((nextFetchTime.getTime() - fetchTime.getTime()) / 1000 <= ONE_DAY * 7);+ }}