Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
Disabled check for canonical URL This check disables indexing if a canonical tag exists and the current page is not the canonical page. This check often fails after a forwarding of a www.-domain to a non-www.-domain where the canonical tag states that the forwarding page is the canonical page. This makes the page not indexable. We disable the check therefore by default.
Michael Peter Christena4bf2eb2aa1c
diff --git a/source/net/yacy/crawler/CrawlSwitchboard.java b/source/net/yacy/crawler/CrawlSwitchboard.javaindex 153b39539..4f63608ad 100644--- a/source/net/yacy/crawler/CrawlSwitchboard.java+++ b/source/net/yacy/crawler/CrawlSwitchboard.java@@ -294,7 +294,7 @@ public final class CrawlSwitchboard {CrawlProfile.MATCH_NEVER_STRING, //indexUrlMustNotMatchCrawlProfile.MATCH_ALL_STRING, //indexContentMustMatchCrawlProfile.MATCH_NEVER_STRING, //indexContentMustNotMatch- true, //noindexWhenCanonicalUnequalURL+ false, //noindexWhenCanonicalUnequalURLInteger.parseInt(sb.getConfig(SwitchboardConstants.AUTOCRAWL_DEEP_DEPTH, "3")),true,CrawlProfile.getRecrawlDate(Integer.parseInt(sb.getConfig(SwitchboardConstants.AUTOCRAWL_DAYS, "1"))*1440),@@ -329,7 +329,7 @@ public final class CrawlSwitchboard {CrawlProfile.MATCH_NEVER_STRING, //indexUrlMustNotMatchCrawlProfile.MATCH_ALL_STRING, //indexContentMustMatchCrawlProfile.MATCH_NEVER_STRING, //indexContentMustNotMatch- true, //noindexWhenCanonicalUnequalURL+ false, //noindexWhenCanonicalUnequalURLInteger.parseInt(sb.getConfig(SwitchboardConstants.AUTOCRAWL_SHALLOW_DEPTH, "1")),true,CrawlProfile.getRecrawlDate(Integer.parseInt(sb.getConfig(SwitchboardConstants.AUTOCRAWL_DAYS, "1"))*1440),@@ -364,7 +364,7 @@ public final class CrawlSwitchboard {CrawlProfile.MATCH_NEVER_STRING, //indexUrlMustNotMatchCrawlProfile.MATCH_ALL_STRING, //indexContentMustMatchCrawlProfile.MATCH_NEVER_STRING, //indexContentMustNotMatch- true, //noindexWhenCanonicalUnequalURL+ false, //noindexWhenCanonicalUnequalURLInteger.parseInt(sb.getConfig(SwitchboardConstants.PROXY_PREFETCH_DEPTH, "0")),true,CrawlProfile.getRecrawlDate(CRAWL_PROFILE_PROXY_RECRAWL_CYCLE),@@ -398,7 +398,7 @@ public final class CrawlSwitchboard {CrawlProfile.MATCH_NEVER_STRING, //indexUrlMustNotMatchCrawlProfile.MATCH_ALL_STRING, //indexContentMustMatchCrawlProfile.MATCH_NEVER_STRING, //indexContentMustNotMatch- true, //noindexWhenCanonicalUnequalURL+ false, //noindexWhenCanonicalUnequalURL0,false,null,@@ -432,7 +432,7 @@ public final class CrawlSwitchboard {CrawlProfile.MATCH_NEVER_STRING, //indexUrlMustNotMatchCrawlProfile.MATCH_ALL_STRING, //indexContentMustMatchCrawlProfile.MATCH_NEVER_STRING, //indexContentMustNotMatch- true, //noindexWhenCanonicalUnequalURL+ false, //noindexWhenCanonicalUnequalURL0,false,CrawlProfile.getRecrawlDate(CRAWL_PROFILE_SNIPPET_LOCAL_TEXT_RECRAWL_CYCLE),@@ -466,7 +466,7 @@ public final class CrawlSwitchboard {CrawlProfile.MATCH_NEVER_STRING, //indexUrlMustNotMatchCrawlProfile.MATCH_ALL_STRING, //indexContentMustMatchCrawlProfile.MATCH_NEVER_STRING, //indexContentMustNotMatch- true, //noindexWhenCanonicalUnequalURL+ false, //noindexWhenCanonicalUnequalURL0,false,CrawlProfile.getRecrawlDate(CRAWL_PROFILE_SNIPPET_GLOBAL_TEXT_RECRAWL_CYCLE),@@ -508,7 +508,7 @@ public final class CrawlSwitchboard {CrawlProfile.MATCH_NEVER_STRING, //indexUrlMustNotMatchCrawlProfile.MATCH_ALL_STRING, //indexContentMustMatchCrawlProfile.MATCH_NEVER_STRING, //indexContentMustNotMatch- true, //noindexWhenCanonicalUnequalURL+ false, //noindexWhenCanonicalUnequalURL0,false,CrawlProfile.getRecrawlDate(CRAWL_PROFILE_GREEDY_LEARNING_TEXT_RECRAWL_CYCLE),@@ -542,7 +542,7 @@ public final class CrawlSwitchboard {CrawlProfile.MATCH_NEVER_STRING, //indexUrlMustNotMatchCrawlProfile.MATCH_ALL_STRING, //indexContentMustMatchCrawlProfile.MATCH_NEVER_STRING, //indexContentMustNotMatch- true, //noindexWhenCanonicalUnequalURL+ false, //noindexWhenCanonicalUnequalURL0,false,CrawlProfile.getRecrawlDate(CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA_RECRAWL_CYCLE),@@ -576,7 +576,7 @@ public final class CrawlSwitchboard {CrawlProfile.MATCH_NEVER_STRING, //indexUrlMustNotMatchCrawlProfile.MATCH_ALL_STRING, //indexContentMustMatchCrawlProfile.MATCH_NEVER_STRING, //indexContentMustNotMatch- true, //noindexWhenCanonicalUnequalURL+ false, //noindexWhenCanonicalUnequalURL0,false,CrawlProfile.getRecrawlDate(CRAWL_PROFILE_SNIPPET_GLOBAL_MEDIA_RECRAWL_CYCLE),@@ -610,7 +610,7 @@ public final class CrawlSwitchboard {CrawlProfile.MATCH_NEVER_STRING, //indexUrlMustNotMatchCrawlProfile.MATCH_ALL_STRING, //indexContentMustMatchCrawlProfile.MATCH_NEVER_STRING, //indexContentMustNotMatch- true, //noindexWhenCanonicalUnequalURL+ false, //noindexWhenCanonicalUnequalURL0,false,CrawlProfile.getRecrawlDate(CRAWL_PROFILE_PACK_RECRAWL_CYCLE),@@ -647,7 +647,7 @@ public final class CrawlSwitchboard {CrawlProfile.MATCH_NEVER_STRING, //indexUrlMustNotMatchCrawlProfile.MATCH_ALL_STRING, //indexContentMustMatchCrawlProfile.MATCH_NEVER_STRING, //indexContentMustNotMatch- true, //noindexWhenCanonicalUnequalURL+ false, //noindexWhenCanonicalUnequalURL0,false,null,diff --git a/source/net/yacy/crawler/RecrawlBusyThread.java b/source/net/yacy/crawler/RecrawlBusyThread.javaindex 3baa56255..10867d2c0 100644--- a/source/net/yacy/crawler/RecrawlBusyThread.java+++ b/source/net/yacy/crawler/RecrawlBusyThread.java@@ -352,12 +352,12 @@ public class RecrawlBusyThread extends AbstractBusyThread {CrawlProfile.MATCH_NEVER_STRING, // indexUrlMustNotMatchCrawlProfile.MATCH_ALL_STRING, // indexContentMustMatchCrawlProfile.MATCH_NEVER_STRING, // indexContentMustNotMatch- true, //noindexWhenCanonicalUnequalURL+ false, //noindexWhenCanonicalUnequalURL0, false, CrawlProfile.getRecrawlDate(CrawlSwitchboard.CRAWL_PROFILE_RECRAWL_JOB_RECRAWL_CYCLE), -1,true, true, true, false, // crawlingQ, followFrames, obeyHtmlRobotsNoindex, obeyHtmlRobotsNofollow,true, true, true, false, -1, false, true, CrawlProfile.MATCH_NEVER_STRING, CacheStrategy.IFFRESH,"robot_" + CrawlSwitchboard.CRAWL_PROFILE_RECRAWL_JOB,- ClientIdentification.yacyInternetCrawlerAgentName,+ ClientIdentification.yacyInternetCrawlerAgentName,TagValency.EVAL, null, null, 0);return profile;}diff --git a/source/net/yacy/htroot/CrawlStartExpert.java b/source/net/yacy/htroot/CrawlStartExpert.javaindex 56b17d767..b382d3300 100644--- a/source/net/yacy/htroot/CrawlStartExpert.java+++ b/source/net/yacy/htroot/CrawlStartExpert.java@@ -371,7 +371,7 @@ public class CrawlStartExpert {// Check Canonical?if (post == null) {- prop.put("noindexWhenCanonicalUnequalURLChecked", 1);+ prop.put("noindexWhenCanonicalUnequalURLChecked", 0);} else {prop.put("noindexWhenCanonicalUnequalURLChecked",post.getBoolean("noindexWhenCanonicalUnequalURL") ? 1 : 0);@@ -571,7 +571,7 @@ public class CrawlStartExpert {}// ---------- Agent name- final List<String> agentNames = new ArrayList<String>();+ final List<String> agentNames = new ArrayList<>();if (sb.isIntranetMode()) {agentNames.add(ClientIdentification.yacyIntranetCrawlerAgentName);}