Apache Nutch is an extensible and scalable web crawler
NUTCH-1564: fix AdaptiveFetchSchedule for unmodified pages
Isabelle Giguere58687ec9ec11
diff --git a/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java b/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.javaindex 38e3162b1..aae385174 100644--- a/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java+++ b/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java@@ -35,6 +35,7 @@ import java.util.HashMap;import java.lang.invoke.MethodHandles;import java.net.URI;import java.net.URISyntaxException;+import java.time.Duration;/*** This class implements an adaptive re-fetch algorithm. This works as follows:@@ -332,21 +333,29 @@ public class AdaptiveFetchSchedule extends AbstractFetchSchedule {case FetchSchedule.STATUS_UNKNOWN:break;}++ // Ensure the interval does not fall outside of bounds+ float minInterval = (getCustomMinInterval(url) != null) ? getCustomMinInterval(url) : MIN_INTERVAL;+ float maxInterval = (getCustomMaxInterval(url) != null) ? getCustomMaxInterval(url) : MAX_INTERVAL;+if (SYNC_DELTA) {// try to synchronize with the time of change- long delta = (fetchTime - modifiedTime) / 1000L;- if (delta > interval)- interval = delta;- refTime = fetchTime - Math.round(delta * SYNC_DELTA_RATE * 1000);- // make sure we are not in the past- if (refTime < fetchTime) {- refTime = fetchTime;+ long delta = (fetchTime - modifiedTime);+ if (delta > (interval * 1000))+ interval = delta / 1000L;+ // offset: a fraction (sync_delta_rate) of the difference between the last modification time, and the last fetch time.+ long offset = Math.round(delta * SYNC_DELTA_RATE);+ long maxIntervalMillis = (long) maxInterval * 1000L;+ LOG.trace("delta (days): " + Duration.ofMillis(delta).toDays()+ + "; offset (days): " + Duration.ofMillis(offset).toDays()+ + "; maxInterval (days): " + Duration.ofMillis(maxIntervalMillis).toDays());+ // convert the offset to a ratio of max interval: avoid next fetchTime in the past, and mimic fetches within max interval+ if (delta > 0 && offset > maxIntervalMillis) {+ offset = offset / delta * maxIntervalMillis; // ex: 9/30*7 = 2.1}+ refTime = fetchTime - offset;}- // Ensure the interval does not fall outside of bounds- float minInterval = (getCustomMinInterval(url) != null) ? getCustomMinInterval(url) : MIN_INTERVAL;- float maxInterval = (getCustomMaxInterval(url) != null) ? getCustomMaxInterval(url) : MAX_INTERVAL;if (interval < minInterval) {interval = minInterval;} else if (interval > maxInterval) {diff --git a/src/test/org/apache/nutch/crawl/TestAdaptiveFetchSchedule.java b/src/test/org/apache/nutch/crawl/TestAdaptiveFetchSchedule.javaindex 2ae06ecff..c06ae3007 100644--- a/src/test/org/apache/nutch/crawl/TestAdaptiveFetchSchedule.java+++ b/src/test/org/apache/nutch/crawl/TestAdaptiveFetchSchedule.java@@ -42,8 +42,6 @@ public class TestAdaptiveFetchSchedule {private Configuration conf;private long curTime, lastModified;private int changed, interval, calculateInterval;-- private static final long ONE_DAY = 86400;@BeforeEachpublic void setUp() throws Exception {@@ -130,14 +128,57 @@ public class TestAdaptiveFetchSchedule {* Test https://issues.apache.org/jira/browse/NUTCH-1564*/@Test- public void testSetFetchSchedule() {- conf.set("db.fetch.schedule.class", "org.apache.nutch.crawl.AdaptiveFetchSchedule");- conf.set("db.fetch.schedule.adaptive.sync_delta", "true"); // default- conf.set("db.fetch.schedule.adaptive.sync_delta_rate", "0.3"); // default- conf.set("db.fetch.interval.default", String.valueOf(ONE_DAY * 2)); // 2 days- conf.set("db.fetch.schedule.adaptive.min_interval", String.valueOf(ONE_DAY)); // 1 day- conf.set("db.fetch.schedule.adaptive.max_interval", String.valueOf(ONE_DAY * 7)); // 7 days- conf.set("db.fetch.interval.max", String.valueOf(ONE_DAY * 7)); // 7 days+ public void testSetFetchSchedule1() {+ // db.fetch.schedule.adaptive.sync_delta_rate = 0.3 (default)+ // db.fetch.interval.default = 172800 (2 days)+ // db.fetch.schedule.adaptive.min_interval = 86400 (1 day)+ // db.fetch.schedule.adaptive.max_interval = 604800 (7 days)+ // db.fetch.interval.max = 604800 (7 days)+ // 3-days cycle+ // 30 days since last modified+ doTestSetFetchSchedule(0.3, 2, 1, 7, 7, 3, 30);+ }++ @Test+ public void testSetFetchSchedule2() {+ // db.fetch.schedule.adaptive.sync_delta_rate = 0.3 (default)+ // db.fetch.interval.default = 86400 (1 day)+ // db.fetch.schedule.adaptive.min_interval = 86400 (1 day)+ // db.fetch.schedule.adaptive.max_interval = 172800 (2 days)+ // db.fetch.interval.max = 604800 (7 days)+ // 1-day cycle+ // 10 days since last modified+ doTestSetFetchSchedule(0.3, 1, 1, 2, 7, 1, 10);+ }++ @Test+ public void testSetFetchSchedule3() {+ // db.fetch.schedule.adaptive.sync_delta_rate = 0.3 (default)+ // db.fetch.interval.default = 172800 (2 days)+ // db.fetch.schedule.adaptive.min_interval = 86400 (1 day)+ // db.fetch.schedule.adaptive.max_interval = 864000 (10 days)+ // db.fetch.interval.max = 864000 (10 days)+ // 3-days cycle+ // 180 days since last modified+ doTestSetFetchSchedule(0.3, 2, 1, 10, 10, 3, 180);+ }++ private void doTestSetFetchSchedule(double deltaRate, int intervalDefaultDays,+ int minIntervalDays, int maxIntervalDays, int intervalMaxDays,+ int previousFetchTimeDays, int modifiedTimeDays) {+ // need to properly override defaults+ Properties props = new Properties();+ props.setProperty("db.fetch.schedule.class", "org.apache.nutch.crawl.AdaptiveFetchSchedule");+ props.setProperty("db.fetch.schedule.adaptive.sync_delta", "true"); // default+ props.setProperty("db.fetch.schedule.adaptive.sync_delta_rate", String.valueOf(deltaRate));+ props.setProperty("db.fetch.interval.default", String.valueOf(FetchSchedule.SECONDS_PER_DAY * intervalDefaultDays));+ props.setProperty("db.fetch.schedule.adaptive.min_interval", String.valueOf(FetchSchedule.SECONDS_PER_DAY * minIntervalDays));+ props.setProperty("db.fetch.schedule.adaptive.max_interval", String.valueOf(FetchSchedule.SECONDS_PER_DAY * maxIntervalDays));+ props.setProperty("db.fetch.interval.max", String.valueOf(FetchSchedule.SECONDS_PER_DAY * intervalMaxDays));++ conf = NutchConfiguration.create(true, props);+ inc_rate = conf.getFloat("db.fetch.schedule.adaptive.inc_rate", 0.2f); // default+ dec_rate = conf.getFloat("db.fetch.schedule.adaptive.dec_rate", 0.2f); // default// ignore adaptive-host-specific-intervals.txtText url = new Text("http://www.example2.com");@@ -147,14 +188,12 @@ public class TestAdaptiveFetchSchedule {CrawlDatum datum = prepareCrawlDatum();Date fetchTime = Date.from(Instant.now());- // previous fetch 3 days ago- Date previousFetchTime = Date.from(Instant.now().minus(Duration.ofDays(3)));- // last modified 1 month ago- Date modifiedTime = Date.from(Instant.now().minus(Duration.ofDays(30)));+ Date previousFetchTime = Date.from(Instant.now().minus(Duration.ofDays(previousFetchTimeDays)));+ Date modifiedTime = Date.from(Instant.now().minus(Duration.ofDays(modifiedTimeDays)));datum.setStatus(CrawlDatum.STATUS_FETCH_SUCCESS);datum.setRetriesSinceFetch(0);datum.setModifiedTime(modifiedTime.getTime());- datum.setFetchTime(fetchTime.getTime());+ datum.setFetchTime(fetchTime.getTime());System.out.println("CrawlDatum fetchTime: " + fetchTime + "; modifiedTime: " + modifiedTime);@@ -166,8 +205,9 @@ public class TestAdaptiveFetchSchedule {assertTrue(nextFetchTime.after(fetchTime));// adapt milliseconds to seconds- assertTrue((nextFetchTime.getTime() - fetchTime.getTime()) / 1000 >= ONE_DAY);- assertTrue((nextFetchTime.getTime() - fetchTime.getTime()) / 1000 <= ONE_DAY * 7);+ long fetchTimeDiff = (nextFetchTime.getTime() - fetchTime.getTime()) / 1000L ;+ assertTrue(fetchTimeDiff >= FetchSchedule.SECONDS_PER_DAY * minIntervalDays);+ assertTrue(fetchTimeDiff <= FetchSchedule.SECONDS_PER_DAY * maxIntervalDays);}}