Skip to content

armgit / nutch

Public Imported from GitHub
0

Apache Nutch is an extensible and scalable web crawler

NUTCH-1564: fix AdaptiveFetchSchedule for unmodified pages

Isabelle Giguere58687ec9ec11
diff --git a/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java b/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java
index 38e3162b1..aae385174 100644
--- a/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java
+++ b/src/java/org/apache/nutch/crawl/AdaptiveFetchSchedule.java
@@ -35,6 +35,7 @@ import java.util.HashMap;
import java.lang.invoke.MethodHandles;
import java.net.URI;
import java.net.URISyntaxException;
+import java.time.Duration;
/**
* This class implements an adaptive re-fetch algorithm. This works as follows:
@@ -332,21 +333,29 @@ public class AdaptiveFetchSchedule extends AbstractFetchSchedule {
case FetchSchedule.STATUS_UNKNOWN:
break;
}
+
+ // Ensure the interval does not fall outside of bounds
+ float minInterval = (getCustomMinInterval(url) != null) ? getCustomMinInterval(url) : MIN_INTERVAL;
+ float maxInterval = (getCustomMaxInterval(url) != null) ? getCustomMaxInterval(url) : MAX_INTERVAL;
+
if (SYNC_DELTA) {
// try to synchronize with the time of change
- long delta = (fetchTime - modifiedTime) / 1000L;
- if (delta > interval)
- interval = delta;
- refTime = fetchTime - Math.round(delta * SYNC_DELTA_RATE * 1000);
- // make sure we are not in the past
- if (refTime < fetchTime) {
- refTime = fetchTime;
+ long delta = (fetchTime - modifiedTime);
+ if (delta > (interval * 1000))
+ interval = delta / 1000L;
+ // offset: a fraction (sync_delta_rate) of the difference between the last modification time, and the last fetch time.
+ long offset = Math.round(delta * SYNC_DELTA_RATE);
+ long maxIntervalMillis = (long) maxInterval * 1000L;
+ LOG.trace("delta (days): " + Duration.ofMillis(delta).toDays()
+ + "; offset (days): " + Duration.ofMillis(offset).toDays()
+ + "; maxInterval (days): " + Duration.ofMillis(maxIntervalMillis).toDays());
+ // convert the offset to a ratio of max interval: avoid next fetchTime in the past, and mimic fetches within max interval
+ if (delta > 0 && offset > maxIntervalMillis) {
+ offset = offset / delta * maxIntervalMillis; // ex: 9/30*7 = 2.1
}
+ refTime = fetchTime - offset;
}
- // Ensure the interval does not fall outside of bounds
- float minInterval = (getCustomMinInterval(url) != null) ? getCustomMinInterval(url) : MIN_INTERVAL;
- float maxInterval = (getCustomMaxInterval(url) != null) ? getCustomMaxInterval(url) : MAX_INTERVAL;
if (interval < minInterval) {
interval = minInterval;
} else if (interval > maxInterval) {
diff --git a/src/test/org/apache/nutch/crawl/TestAdaptiveFetchSchedule.java b/src/test/org/apache/nutch/crawl/TestAdaptiveFetchSchedule.java
index 2ae06ecff..c06ae3007 100644
--- a/src/test/org/apache/nutch/crawl/TestAdaptiveFetchSchedule.java
+++ b/src/test/org/apache/nutch/crawl/TestAdaptiveFetchSchedule.java
@@ -42,8 +42,6 @@ public class TestAdaptiveFetchSchedule {
private Configuration conf;
private long curTime, lastModified;
private int changed, interval, calculateInterval;
-
- private static final long ONE_DAY = 86400;
@BeforeEach
public void setUp() throws Exception {
@@ -130,14 +128,57 @@ public class TestAdaptiveFetchSchedule {
* Test https://issues.apache.org/jira/browse/NUTCH-1564
*/
@Test
- public void testSetFetchSchedule() {
- conf.set("db.fetch.schedule.class", "org.apache.nutch.crawl.AdaptiveFetchSchedule");
- conf.set("db.fetch.schedule.adaptive.sync_delta", "true"); // default
- conf.set("db.fetch.schedule.adaptive.sync_delta_rate", "0.3"); // default
- conf.set("db.fetch.interval.default", String.valueOf(ONE_DAY * 2)); // 2 days
- conf.set("db.fetch.schedule.adaptive.min_interval", String.valueOf(ONE_DAY)); // 1 day
- conf.set("db.fetch.schedule.adaptive.max_interval", String.valueOf(ONE_DAY * 7)); // 7 days
- conf.set("db.fetch.interval.max", String.valueOf(ONE_DAY * 7)); // 7 days
+ public void testSetFetchSchedule1() {
+ // db.fetch.schedule.adaptive.sync_delta_rate = 0.3 (default)
+ // db.fetch.interval.default = 172800 (2 days)
+ // db.fetch.schedule.adaptive.min_interval = 86400 (1 day)
+ // db.fetch.schedule.adaptive.max_interval = 604800 (7 days)
+ // db.fetch.interval.max = 604800 (7 days)
+ // 3-days cycle
+ // 30 days since last modified
+ doTestSetFetchSchedule(0.3, 2, 1, 7, 7, 3, 30);
+ }
+
+ @Test
+ public void testSetFetchSchedule2() {
+ // db.fetch.schedule.adaptive.sync_delta_rate = 0.3 (default)
+ // db.fetch.interval.default = 86400 (1 day)
+ // db.fetch.schedule.adaptive.min_interval = 86400 (1 day)
+ // db.fetch.schedule.adaptive.max_interval = 172800 (2 days)
+ // db.fetch.interval.max = 604800 (7 days)
+ // 1-day cycle
+ // 10 days since last modified
+ doTestSetFetchSchedule(0.3, 1, 1, 2, 7, 1, 10);
+ }
+
+ @Test
+ public void testSetFetchSchedule3() {
+ // db.fetch.schedule.adaptive.sync_delta_rate = 0.3 (default)
+ // db.fetch.interval.default = 172800 (2 days)
+ // db.fetch.schedule.adaptive.min_interval = 86400 (1 day)
+ // db.fetch.schedule.adaptive.max_interval = 864000 (10 days)
+ // db.fetch.interval.max = 864000 (10 days)
+ // 3-days cycle
+ // 180 days since last modified
+ doTestSetFetchSchedule(0.3, 2, 1, 10, 10, 3, 180);
+ }
+
+ private void doTestSetFetchSchedule(double deltaRate, int intervalDefaultDays,
+ int minIntervalDays, int maxIntervalDays, int intervalMaxDays,
+ int previousFetchTimeDays, int modifiedTimeDays) {
+ // need to properly override defaults
+ Properties props = new Properties();
+ props.setProperty("db.fetch.schedule.class", "org.apache.nutch.crawl.AdaptiveFetchSchedule");
+ props.setProperty("db.fetch.schedule.adaptive.sync_delta", "true"); // default
+ props.setProperty("db.fetch.schedule.adaptive.sync_delta_rate", String.valueOf(deltaRate));
+ props.setProperty("db.fetch.interval.default", String.valueOf(FetchSchedule.SECONDS_PER_DAY * intervalDefaultDays));
+ props.setProperty("db.fetch.schedule.adaptive.min_interval", String.valueOf(FetchSchedule.SECONDS_PER_DAY * minIntervalDays));
+ props.setProperty("db.fetch.schedule.adaptive.max_interval", String.valueOf(FetchSchedule.SECONDS_PER_DAY * maxIntervalDays));
+ props.setProperty("db.fetch.interval.max", String.valueOf(FetchSchedule.SECONDS_PER_DAY * intervalMaxDays));
+
+ conf = NutchConfiguration.create(true, props);
+ inc_rate = conf.getFloat("db.fetch.schedule.adaptive.inc_rate", 0.2f); // default
+ dec_rate = conf.getFloat("db.fetch.schedule.adaptive.dec_rate", 0.2f); // default
// ignore adaptive-host-specific-intervals.txt
Text url = new Text("http://www.example2.com");
@@ -147,14 +188,12 @@ public class TestAdaptiveFetchSchedule {
CrawlDatum datum = prepareCrawlDatum();
Date fetchTime = Date.from(Instant.now());
- // previous fetch 3 days ago
- Date previousFetchTime = Date.from(Instant.now().minus(Duration.ofDays(3)));
- // last modified 1 month ago
- Date modifiedTime = Date.from(Instant.now().minus(Duration.ofDays(30)));
+ Date previousFetchTime = Date.from(Instant.now().minus(Duration.ofDays(previousFetchTimeDays)));
+ Date modifiedTime = Date.from(Instant.now().minus(Duration.ofDays(modifiedTimeDays)));
datum.setStatus(CrawlDatum.STATUS_FETCH_SUCCESS);
datum.setRetriesSinceFetch(0);
datum.setModifiedTime(modifiedTime.getTime());
- datum.setFetchTime(fetchTime.getTime());
+ datum.setFetchTime(fetchTime.getTime());
System.out.println("CrawlDatum fetchTime: " + fetchTime + "; modifiedTime: " + modifiedTime);
@@ -166,8 +205,9 @@ public class TestAdaptiveFetchSchedule {
assertTrue(nextFetchTime.after(fetchTime));
// adapt milliseconds to seconds
- assertTrue((nextFetchTime.getTime() - fetchTime.getTime()) / 1000 >= ONE_DAY);
- assertTrue((nextFetchTime.getTime() - fetchTime.getTime()) / 1000 <= ONE_DAY * 7);
+ long fetchTimeDiff = (nextFetchTime.getTime() - fetchTime.getTime()) / 1000L ;
+ assertTrue(fetchTimeDiff >= FetchSchedule.SECONDS_PER_DAY * minIntervalDays);
+ assertTrue(fetchTimeDiff <= FetchSchedule.SECONDS_PER_DAY * maxIntervalDays);
}
}