Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
Added a specific default crawl profile for the recrawl job.
lucciomanb712a0671e82
diff --git a/source/net/yacy/crawler/CrawlSwitchboard.java b/source/net/yacy/crawler/CrawlSwitchboard.javaindex 1078605d1..bc030b10f 100644--- a/source/net/yacy/crawler/CrawlSwitchboard.java+++ b/source/net/yacy/crawler/CrawlSwitchboard.java@@ -61,6 +61,7 @@ public final class CrawlSwitchboard {public static final String CRAWL_PROFILE_AUTOCRAWL_DEEP = "autocrawlDeep";public static final String CRAWL_PROFILE_AUTOCRAWL_SHALLOW = "autocrawlShallow";+ public static final String CRAWL_PROFILE_RECRAWL_JOB = "recrawlJob";public static final String CRAWL_PROFILE_PROXY = "proxy";public static final String CRAWL_PROFILE_REMOTE = "remote";public static final String CRAWL_PROFILE_SNIPPET_LOCAL_TEXT = "snippetLocalText";@@ -88,7 +89,13 @@ public final class CrawlSwitchboard {public static final String DBFILE_PASSIVE_CRAWL_PROFILES = "crawlProfilesPassive1.heap";// Default time cycle in minutes before an indexed URL by a given crawl profile can be accepted for recrawl */-++ /**+ * The default recrawl time cycle in minutes for recrawl jobs. The recrawl date+ * limit can be set up by the recrawl job selection query, but a default limit+ * prevent unwanted overload on targets)+ */+ public static final long CRAWL_PROFILE_RECRAWL_JOB_RECRAWL_CYCLE = 60L; // on hourpublic static final long CRAWL_PROFILE_PROXY_RECRAWL_CYCLE = 60L * 24L; // one daypublic static final long CRAWL_PROFILE_SNIPPET_LOCAL_TEXT_RECRAWL_CYCLE = 60L * 24L * 30L; // 30 dayspublic static final long CRAWL_PROFILE_SNIPPET_GLOBAL_TEXT_RECRAWL_CYCLE = 60L * 24L * 30L; // 30 days@@ -104,7 +111,7 @@ public final class CrawlSwitchboard {private final Map<String, RowHandleSet> profilesActiveCrawlsCounter;public CrawlProfile defaultProxyProfile, defaultRemoteProfile, defaultTextSnippetLocalProfile, defaultTextSnippetGlobalProfile;public CrawlProfile defaultTextGreedyLearningProfile, defaultMediaSnippetLocalProfile, defaultMediaSnippetGlobalProfile, defaultSurrogateProfile;- public CrawlProfile defaultAutocrawlDeepProfile, defaultAutocrawlShallowProfile;+ public CrawlProfile defaultAutocrawlDeepProfile, defaultAutocrawlShallowProfile, defaultRecrawlJobProfile;private Map<String, CrawlProfile> defaultPushProfiles; // for each collection one profileprivate final File queuesRoot;private Switchboard switchboard;@@ -466,6 +473,13 @@ public final class CrawlSwitchboard {UTF8.getBytes(this.defaultTextSnippetGlobalProfile.handle()),this.defaultTextSnippetGlobalProfile);this.defaultTextSnippetGlobalProfile.setCacheStrategy(CacheStrategy.IFEXIST);++ // generate new default entry for RecrawlBusyThread+ this.defaultRecrawlJobProfile = RecrawlBusyThread.buildDefaultCrawlProfile();+ this.profilesActiveCrawls.put(+ UTF8.getBytes(this.defaultRecrawlJobProfile.handle()),+ this.defaultRecrawlJobProfile);+// generate new default entry for greedy learningthis.defaultTextGreedyLearningProfile =new CrawlProfile(diff --git a/source/net/yacy/crawler/RecrawlBusyThread.java b/source/net/yacy/crawler/RecrawlBusyThread.javaindex 07d6393c8..bfeef8e8e 100644--- a/source/net/yacy/crawler/RecrawlBusyThread.java+++ b/source/net/yacy/crawler/RecrawlBusyThread.java@@ -34,6 +34,8 @@ import org.apache.solr.common.SolrDocumentList;import net.yacy.cora.document.id.DigestURL;import net.yacy.cora.federate.solr.connector.SolrConnector;+import net.yacy.cora.federate.yacy.CacheStrategy;+import net.yacy.cora.protocol.ClientIdentification;import net.yacy.cora.util.ConcurrentLog;import net.yacy.crawler.data.CrawlProfile;import net.yacy.crawler.data.NoticedURL;@@ -183,7 +185,7 @@ public class RecrawlBusyThread extends AbstractBusyThread {int added = 0;if (!this.urlstack.isEmpty()) {- final CrawlProfile profile = sb.crawler.defaultTextSnippetGlobalProfile;+ final CrawlProfile profile = sb.crawler.defaultRecrawlJobProfile;for (final DigestURL url : this.urlstack) {final Request request = sb.loader.request(url, true, true);@@ -302,6 +304,28 @@ public class RecrawlBusyThread extends AbstractBusyThread {}return true;}++ /**+ * @return a new default CrawlProfile instance to be used for recrawl jobs.+ */+ public static CrawlProfile buildDefaultCrawlProfile() {+ CrawlProfile profile = new CrawlProfile(CrawlSwitchboard.CRAWL_PROFILE_RECRAWL_JOB, CrawlProfile.MATCH_ALL_STRING, // crawlerUrlMustMatch+ CrawlProfile.MATCH_NEVER_STRING, // crawlerUrlMustNotMatch+ CrawlProfile.MATCH_ALL_STRING, // crawlerIpMustMatch+ CrawlProfile.MATCH_NEVER_STRING, // crawlerIpMustNotMatch+ CrawlProfile.MATCH_NEVER_STRING, // crawlerCountryMustMatch+ CrawlProfile.MATCH_NEVER_STRING, // crawlerNoDepthLimitMatch+ CrawlProfile.MATCH_ALL_STRING, // indexUrlMustMatch+ CrawlProfile.MATCH_NEVER_STRING, // indexUrlMustNotMatch+ CrawlProfile.MATCH_ALL_STRING, // indexContentMustMatch+ CrawlProfile.MATCH_NEVER_STRING, // indexContentMustNotMatch+ 0, false, CrawlProfile.getRecrawlDate(CrawlSwitchboard.CRAWL_PROFILE_RECRAWL_JOB_RECRAWL_CYCLE), -1,+ true, true, true, false, // crawlingQ, followFrames, obeyHtmlRobotsNoindex, obeyHtmlRobotsNofollow,+ true, true, true, false, -1, false, true, CrawlProfile.MATCH_NEVER_STRING, CacheStrategy.IFFRESH,+ "robot_" + CrawlSwitchboard.CRAWL_PROFILE_RECRAWL_JOB,+ ClientIdentification.yacyInternetCrawlerAgentName, null, null, 0);+ return profile;+ }@Overridepublic int getJobCount() {