Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
added canonical filter attention: this is on by default! (it should do the right thing)
Michael Peter Christen9fcd8f1bdac3
diff --git a/htroot/CrawlProfileEditor_p.xml b/htroot/CrawlProfileEditor_p.xmlindex 0b880ac3f..58036b48e 100644--- a/htroot/CrawlProfileEditor_p.xml+++ b/htroot/CrawlProfileEditor_p.xml@@ -1,48 +1,49 @@<?xml version="1.0" encoding="UTF-8" standalone="yes"?><crawlProfiles>#{crawlProfiles}#- <crawlProfile>- <handle>#[handle]#</handle>- <name>#[name]#</name>- <collections>#[collections]#</collections>- <agentName>#[agentName]#</agentName>- <userAgent>#[userAgent]#</userAgent>- <depth>#[depth]#</depth>- <directDocByURL>#(directDocByURL)#false::true#(/directDocByURL)#</directDocByURL>- <recrawlIfOlder>#[recrawlIfOlder]#</recrawlIfOlder>- <domMaxPages>#[domMaxPages]#</domMaxPages>- <crawlingQ>#(crawlingQ)#false::true#(/crawlingQ)#</crawlingQ>- <followFrames>#(followFrames)#false::true#(/followFrames)#</followFrames>- <obeyHtmlRobotsNoindex>#(obeyHtmlRobotsNoindex)#false::true#(/obeyHtmlRobotsNoindex)#</obeyHtmlRobotsNoindex>- <obeyHtmlRobotsNofollow>#(obeyHtmlRobotsNofollow)#false::true#(/obeyHtmlRobotsNofollow)#</obeyHtmlRobotsNofollow>- <indexText>#(indexText)#false::true#(/indexText)#</indexText>- <indexMedia>#(indexMedia)#false::true#(/indexMedia)#</indexMedia>- <storeHTCache>#(storeHTCache)#false::true#(/storeHTCache)#</storeHTCache>- <remoteIndexing>#(remoteIndexing)#false::true#(/remoteIndexing)#</remoteIndexing>- <cacheStrategy>#[cacheStrategy]#</cacheStrategy>- <crawlerAlwaysCheckMediaType>#(crawlerAlwaysCheckMediaType)#false::true#(/crawlerAlwaysCheckMediaType)#</crawlerAlwaysCheckMediaType>- <crawlerURLMustMatch>#[crawlerURLMustMatch]#</crawlerURLMustMatch>- <crawlerURLMustNotMatch>#[crawlerURLMustNotMatch]#</crawlerURLMustNotMatch>- <crawlerOriginURLMustMatch>#[crawlerOriginURLMustMatch]#</crawlerOriginURLMustMatch>- <crawlerOriginURLMustNotMatch>#[crawlerOriginURLMustNotMatch]#</crawlerOriginURLMustNotMatch>- <crawlerIPMustMatch>#[crawlerIPMustMatch]#</crawlerIPMustMatch>- <crawlerIPMustNotMatch>#[crawlerIPMustNotMatch]#</crawlerIPMustNotMatch>- <crawlerCountryMustMatch>#[crawlerCountryMustMatch]#</crawlerCountryMustMatch>- <crawlerNoLimitURLMustMatch>#[crawlerNoLimitURLMustMatch]#</crawlerNoLimitURLMustMatch>- <indexURLMustMatch>#[indexURLMustMatch]#</indexURLMustMatch>- <indexURLMustNotMatch>#[indexURLMustNotMatch]#</indexURLMustNotMatch>- <indexContentMustMatch>#[indexContentMustMatch]#</indexContentMustMatch>- <indexContentMustNotMatch>#[indexContentMustNotMatch]#</indexContentMustNotMatch>- <indexMediaTypeMustMatch>#[indexMediaTypeMustMatch]#</indexMediaTypeMustMatch>- <indexMediaTypeMustNotMatch>#[indexMediaTypeMustNotMatch]#</indexMediaTypeMustNotMatch>- <indexSolrQueryMustMatch>#[indexSolrQueryMustMatch]#</indexSolrQueryMustMatch>- <indexSolrQueryMustNotMatch>#[indexSolrQueryMustNotMatch]#</indexSolrQueryMustNotMatch>- <status>#(status)#terminated::active::system#(/status)#</status>- <crawlingDomFilterContent>- #{crawlingDomFilterContent}#- <item>#[item]#</item>- #{/crawlingDomFilterContent}#- </crawlingDomFilterContent>- </crawlProfile>+ <crawlProfile>+ <handle>#[handle]#</handle>+ <name>#[name]#</name>+ <collections>#[collections]#</collections>+ <agentName>#[agentName]#</agentName>+ <userAgent>#[userAgent]#</userAgent>+ <depth>#[depth]#</depth>+ <directDocByURL>#(directDocByURL)#false::true#(/directDocByURL)#</directDocByURL>+ <recrawlIfOlder>#[recrawlIfOlder]#</recrawlIfOlder>+ <domMaxPages>#[domMaxPages]#</domMaxPages>+ <crawlingQ>#(crawlingQ)#false::true#(/crawlingQ)#</crawlingQ>+ <followFrames>#(followFrames)#false::true#(/followFrames)#</followFrames>+ <obeyHtmlRobotsNoindex>#(obeyHtmlRobotsNoindex)#false::true#(/obeyHtmlRobotsNoindex)#</obeyHtmlRobotsNoindex>+ <obeyHtmlRobotsNofollow>#(obeyHtmlRobotsNofollow)#false::true#(/obeyHtmlRobotsNofollow)#</obeyHtmlRobotsNofollow>+ <indexText>#(indexText)#false::true#(/indexText)#</indexText>+ <indexMedia>#(indexMedia)#false::true#(/indexMedia)#</indexMedia>+ <storeHTCache>#(storeHTCache)#false::true#(/storeHTCache)#</storeHTCache>+ <remoteIndexing>#(remoteIndexing)#false::true#(/remoteIndexing)#</remoteIndexing>+ <cacheStrategy>#[cacheStrategy]#</cacheStrategy>+ <crawlerAlwaysCheckMediaType>#(crawlerAlwaysCheckMediaType)#false::true#(/crawlerAlwaysCheckMediaType)#</crawlerAlwaysCheckMediaType>+ <crawlerURLMustMatch>#[crawlerURLMustMatch]#</crawlerURLMustMatch>+ <crawlerURLMustNotMatch>#[crawlerURLMustNotMatch]#</crawlerURLMustNotMatch>+ <crawlerOriginURLMustMatch>#[crawlerOriginURLMustMatch]#</crawlerOriginURLMustMatch>+ <crawlerOriginURLMustNotMatch>#[crawlerOriginURLMustNotMatch]#</crawlerOriginURLMustNotMatch>+ <crawlerIPMustMatch>#[crawlerIPMustMatch]#</crawlerIPMustMatch>+ <crawlerIPMustNotMatch>#[crawlerIPMustNotMatch]#</crawlerIPMustNotMatch>+ <crawlerCountryMustMatch>#[crawlerCountryMustMatch]#</crawlerCountryMustMatch>+ <crawlerNoLimitURLMustMatch>#[crawlerNoLimitURLMustMatch]#</crawlerNoLimitURLMustMatch>+ <indexURLMustMatch>#[indexURLMustMatch]#</indexURLMustMatch>+ <indexURLMustNotMatch>#[indexURLMustNotMatch]#</indexURLMustNotMatch>+ <indexContentMustMatch>#[indexContentMustMatch]#</indexContentMustMatch>+ <indexContentMustNotMatch>#[indexContentMustNotMatch]#</indexContentMustNotMatch>+ <indexMediaTypeMustMatch>#[indexMediaTypeMustMatch]#</indexMediaTypeMustMatch>+ <indexMediaTypeMustNotMatch>#[indexMediaTypeMustNotMatch]#</indexMediaTypeMustNotMatch>+ <indexSolrQueryMustMatch>#[indexSolrQueryMustMatch]#</indexSolrQueryMustMatch>+ <indexSolrQueryMustNotMatch>#[indexSolrQueryMustNotMatch]#</indexSolrQueryMustNotMatch>+ <noindexWhenCanonicalUnequalURL>#(noindexWhenCanonicalUnequalURL)#false::true#(/noindexWhenCanonicalUnequalURL)#</noindexWhenCanonicalUnequalURL>+ <status>#(status)#terminated::active::system#(/status)#</status>+ <crawlingDomFilterContent>+ #{crawlingDomFilterContent}#+ <item>#[item]#</item>+ #{/crawlingDomFilterContent}#+ </crawlingDomFilterContent>+ </crawlProfile>#{/crawlProfiles}#</crawlProfiles>diff --git a/htroot/CrawlStartExpert.html b/htroot/CrawlStartExpert.htmlindex c3e60c1b8..9917cd7e1 100644--- a/htroot/CrawlStartExpert.html+++ b/htroot/CrawlStartExpert.html@@ -412,6 +412,9 @@<table style="border-width: 0px"><tr><td style="width:110px"><img src="env/grafics/plus.gif" alt=""> must-match</td><td><input name="indexmustmatch" id="indexmustmatch" type="text" size="55" maxlength="100000" value="#[indexmustmatch]#" onblur="if (this.value=='') this.value='.*';"/> (must not be empty)</td></tr><tr><td><img src="env/grafics/minus.gif" alt=""> must-not-match</td><td><input name="indexmustnotmatch" id="indexmustnotmatch" type="text" size="55" maxlength="100000" value="#[indexmustnotmatch]#" /></td></tr>+ <tr>+ <td colspan="2"><input type="checkbox" name="noindexWhenCanonicalUnequalURL" id="noindexWhenCanonicalUnequalURL" #(noindexWhenCanonicalUnequalURLChecked)#::checked="checked"#(/noindexWhenCanonicalUnequalURLChecked)#/> No Indexing when Canonical present and Canonical != URL</td>+ </tr></table></dd><dt>Filter on Content of Document<br/>(all visible text, including camel-case-tokenized url and title)</dt>@@ -470,7 +473,7 @@<tr><td style="width:110px"><img src="env/grafics/minus.gif" alt=""> must-not-match</td><td>- <input name="indexSolrQueryMustNotMatch" id="indexSolrQueryMustNotMatch" type="text" size="55" maxlength="100000" value="#[indexSolrQueryMustNotMatch]#" aria-describedby="indexSolrQueryInfo" />+ <input name="indexSolrQueryMustNotMatch" id="indexSolrQueryMustNotMatch" type="text" size="55" maxlength="100000" value="#[indexSolrQueryMustNotMatch]#" aria-describedby="indexSolrQueryInfo" enabled="false"/></td></tr>#(/embeddedSolrConnected)#diff --git a/source/net/yacy/crawler/CrawlStacker.java b/source/net/yacy/crawler/CrawlStacker.javaindex 16ffea78b..1b4874a96 100644--- a/source/net/yacy/crawler/CrawlStacker.java+++ b/source/net/yacy/crawler/CrawlStacker.java@@ -488,7 +488,7 @@ public final class CrawlStacker implements WorkflowTask<Request>{// check if ip is local ip addressfinal String urlRejectReason = this.urlInAcceptedDomain(url);if (urlRejectReason != null) {- if (CrawlStacker.log.isFine()) CrawlStacker.log.fine("denied_(" + urlRejectReason + ")");+ if (CrawlStacker.log.isFine()) CrawlStacker.log.fine("URL not in accepted Domain (" + urlRejectReason + ")");return "denied_(" + urlRejectReason + ")";}diff --git a/source/net/yacy/crawler/CrawlSwitchboard.java b/source/net/yacy/crawler/CrawlSwitchboard.javaindex aca7b95cd..3c17c75cf 100644--- a/source/net/yacy/crawler/CrawlSwitchboard.java+++ b/source/net/yacy/crawler/CrawlSwitchboard.java@@ -294,6 +294,7 @@ public final class CrawlSwitchboard {CrawlProfile.MATCH_NEVER_STRING, //indexUrlMustNotMatchCrawlProfile.MATCH_ALL_STRING, //indexContentMustMatchCrawlProfile.MATCH_NEVER_STRING, //indexContentMustNotMatch+ true, //noindexWhenCanonicalUnequalURLInteger.parseInt(sb.getConfig(SwitchboardConstants.AUTOCRAWL_DEEP_DEPTH, "3")),true,CrawlProfile.getRecrawlDate(Integer.parseInt(sb.getConfig(SwitchboardConstants.AUTOCRAWL_DAYS, "1"))*1440),@@ -328,6 +329,7 @@ public final class CrawlSwitchboard {CrawlProfile.MATCH_NEVER_STRING, //indexUrlMustNotMatchCrawlProfile.MATCH_ALL_STRING, //indexContentMustMatchCrawlProfile.MATCH_NEVER_STRING, //indexContentMustNotMatch+ true, //noindexWhenCanonicalUnequalURLInteger.parseInt(sb.getConfig(SwitchboardConstants.AUTOCRAWL_SHALLOW_DEPTH, "1")),true,CrawlProfile.getRecrawlDate(Integer.parseInt(sb.getConfig(SwitchboardConstants.AUTOCRAWL_DAYS, "1"))*1440),@@ -362,6 +364,7 @@ public final class CrawlSwitchboard {CrawlProfile.MATCH_NEVER_STRING, //indexUrlMustNotMatchCrawlProfile.MATCH_ALL_STRING, //indexContentMustMatchCrawlProfile.MATCH_NEVER_STRING, //indexContentMustNotMatch+ true, //noindexWhenCanonicalUnequalURLInteger.parseInt(sb.getConfig(SwitchboardConstants.PROXY_PREFETCH_DEPTH, "0")),true,CrawlProfile.getRecrawlDate(CRAWL_PROFILE_PROXY_RECRAWL_CYCLE),@@ -395,6 +398,7 @@ public final class CrawlSwitchboard {CrawlProfile.MATCH_NEVER_STRING, //indexUrlMustNotMatchCrawlProfile.MATCH_ALL_STRING, //indexContentMustMatchCrawlProfile.MATCH_NEVER_STRING, //indexContentMustNotMatch+ true, //noindexWhenCanonicalUnequalURL0,false,null,@@ -428,6 +432,7 @@ public final class CrawlSwitchboard {CrawlProfile.MATCH_NEVER_STRING, //indexUrlMustNotMatchCrawlProfile.MATCH_ALL_STRING, //indexContentMustMatchCrawlProfile.MATCH_NEVER_STRING, //indexContentMustNotMatch+ true, //noindexWhenCanonicalUnequalURL0,false,CrawlProfile.getRecrawlDate(CRAWL_PROFILE_SNIPPET_LOCAL_TEXT_RECRAWL_CYCLE),@@ -461,6 +466,7 @@ public final class CrawlSwitchboard {CrawlProfile.MATCH_NEVER_STRING, //indexUrlMustNotMatchCrawlProfile.MATCH_ALL_STRING, //indexContentMustMatchCrawlProfile.MATCH_NEVER_STRING, //indexContentMustNotMatch+ true, //noindexWhenCanonicalUnequalURL0,false,CrawlProfile.getRecrawlDate(CRAWL_PROFILE_SNIPPET_GLOBAL_TEXT_RECRAWL_CYCLE),@@ -502,6 +508,7 @@ public final class CrawlSwitchboard {CrawlProfile.MATCH_NEVER_STRING, //indexUrlMustNotMatchCrawlProfile.MATCH_ALL_STRING, //indexContentMustMatchCrawlProfile.MATCH_NEVER_STRING, //indexContentMustNotMatch+ true, //noindexWhenCanonicalUnequalURL0,false,CrawlProfile.getRecrawlDate(CRAWL_PROFILE_GREEDY_LEARNING_TEXT_RECRAWL_CYCLE),@@ -535,6 +542,7 @@ public final class CrawlSwitchboard {CrawlProfile.MATCH_NEVER_STRING, //indexUrlMustNotMatchCrawlProfile.MATCH_ALL_STRING, //indexContentMustMatchCrawlProfile.MATCH_NEVER_STRING, //indexContentMustNotMatch+ true, //noindexWhenCanonicalUnequalURL0,false,CrawlProfile.getRecrawlDate(CRAWL_PROFILE_SNIPPET_LOCAL_MEDIA_RECRAWL_CYCLE),@@ -568,6 +576,7 @@ public final class CrawlSwitchboard {CrawlProfile.MATCH_NEVER_STRING, //indexUrlMustNotMatchCrawlProfile.MATCH_ALL_STRING, //indexContentMustMatchCrawlProfile.MATCH_NEVER_STRING, //indexContentMustNotMatch+ true, //noindexWhenCanonicalUnequalURL0,false,CrawlProfile.getRecrawlDate(CRAWL_PROFILE_SNIPPET_GLOBAL_MEDIA_RECRAWL_CYCLE),@@ -601,6 +610,7 @@ public final class CrawlSwitchboard {CrawlProfile.MATCH_NEVER_STRING, //indexUrlMustNotMatchCrawlProfile.MATCH_ALL_STRING, //indexContentMustMatchCrawlProfile.MATCH_NEVER_STRING, //indexContentMustNotMatch+ true, //noindexWhenCanonicalUnequalURL0,false,CrawlProfile.getRecrawlDate(CRAWL_PROFILE_SURROGATE_RECRAWL_CYCLE),@@ -637,6 +647,7 @@ public final class CrawlSwitchboard {CrawlProfile.MATCH_NEVER_STRING, //indexUrlMustNotMatchCrawlProfile.MATCH_ALL_STRING, //indexContentMustMatchCrawlProfile.MATCH_NEVER_STRING, //indexContentMustNotMatch+ true, //noindexWhenCanonicalUnequalURL0,false,null,diff --git a/source/net/yacy/crawler/RecrawlBusyThread.java b/source/net/yacy/crawler/RecrawlBusyThread.javaindex af32db0cf..11b3d3a98 100644--- a/source/net/yacy/crawler/RecrawlBusyThread.java+++ b/source/net/yacy/crawler/RecrawlBusyThread.java@@ -352,6 +352,7 @@ public class RecrawlBusyThread extends AbstractBusyThread {CrawlProfile.MATCH_NEVER_STRING, // indexUrlMustNotMatchCrawlProfile.MATCH_ALL_STRING, // indexContentMustMatchCrawlProfile.MATCH_NEVER_STRING, // indexContentMustNotMatch+ true, //noindexWhenCanonicalUnequalURL0, false, CrawlProfile.getRecrawlDate(CrawlSwitchboard.CRAWL_PROFILE_RECRAWL_JOB_RECRAWL_CYCLE), -1,true, true, true, false, // crawlingQ, followFrames, obeyHtmlRobotsNoindex, obeyHtmlRobotsNofollow,true, true, true, false, -1, false, true, CrawlProfile.MATCH_NEVER_STRING, CacheStrategy.IFFRESH,diff --git a/source/net/yacy/crawler/data/CrawlProfile.java b/source/net/yacy/crawler/data/CrawlProfile.javaindex 19f36b613..64af0c489 100644--- a/source/net/yacy/crawler/data/CrawlProfile.java+++ b/source/net/yacy/crawler/data/CrawlProfile.java@@ -115,6 +115,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements MINDEXING_MEDIA_TYPE_MUSTNOTMATCH("indexMediaTypeMustNotMatch", false, CrawlAttribute.STRING, "Indexing Media Type (MIME) Must-Not-Match Filter"),INDEXING_SOLR_QUERY_MUSTMATCH("indexSolrQueryMustMatch", false, CrawlAttribute.STRING, "Indexing Solr Query Must-Match Filter"),INDEXING_SOLR_QUERY_MUSTNOTMATCH("indexSolrQueryMustNotMatch", false, CrawlAttribute.STRING, "Indexing Solr Query Must-Not-Match Filter"),+ NOINDEX_WHEN_CANONICAL_UNEQUAL_URL("noindexWhenCanonicalUnequalURL", false, CrawlAttribute.STRING, "No Indexing for Documents with Canonical != URL"),RECRAWL_IF_OLDER ("recrawlIfOlder", false, CrawlAttribute.INTEGER, "Recrawl If Older"),STORE_HTCACHE ("storeHTCache", false, CrawlAttribute.BOOLEAN, "Store in HTCache"),CACHE_STRAGEGY ("cacheStrategy", false, CrawlAttribute.STRING, "Cache Strategy (NOCACHE,IFFRESH,IFEXIST,CACHEONLY)"),@@ -223,6 +224,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements Mfinal String crawlerCountryMustMatch, final String crawlerNoDepthLimitMatch,final String indexUrlMustMatch, final String indexUrlMustNotMatch,final String indexContentMustMatch, final String indexContentMustNotMatch,+ final boolean noindexWhenCanonicalUnequalURL,final int depth,final boolean directDocByURL,final Date recrawlIfOlder /*date*/,@@ -300,6 +302,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements Mput(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTNOTMATCH.key, CrawlProfile.MATCH_NEVER_STRING);put(CrawlAttribute.INDEXING_SOLR_QUERY_MUSTMATCH.key, CrawlProfile.SOLR_MATCH_ALL_QUERY);put(CrawlAttribute.INDEXING_SOLR_QUERY_MUSTNOTMATCH.key, CrawlProfile.SOLR_EMPTY_QUERY);+ put(CrawlAttribute.NOINDEX_WHEN_CANONICAL_UNEQUAL_URL.key, noindexWhenCanonicalUnequalURL);}/**@@ -851,6 +854,12 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements Mreturn (r.equals(Boolean.TRUE.toString()));}+ public boolean noindexWhenCanonicalUnequalURL() {+ final String r = get(CrawlAttribute.NOINDEX_WHEN_CANONICAL_UNEQUAL_URL.key);+ if (r == null) return true;+ return (r.equals(Boolean.TRUE.toString()));+ }+public boolean storeHTCache() {final String r = get(CrawlAttribute.STORE_HTCACHE.key);if (r == null) return false;@@ -997,6 +1006,7 @@ public class CrawlProfile extends ConcurrentHashMap<String, String> implements Mprop.putXML(CRAWL_PROFILE_PREFIX + count + "_indexURLMustNotMatch", this.get(CrawlAttribute.INDEXING_URL_MUSTNOTMATCH.key));prop.putXML(CRAWL_PROFILE_PREFIX + count + "_indexContentMustMatch", this.get(CrawlAttribute.INDEXING_CONTENT_MUSTMATCH.key));prop.putXML(CRAWL_PROFILE_PREFIX + count + "_indexContentMustNotMatch", this.get(CrawlAttribute.INDEXING_CONTENT_MUSTNOTMATCH.key));+ prop.put(CRAWL_PROFILE_PREFIX + count + "_" + CrawlAttribute.NOINDEX_WHEN_CANONICAL_UNEQUAL_URL, noindexWhenCanonicalUnequalURL() ? 1 : 0);prop.putXML(CRAWL_PROFILE_PREFIX + count + "_" + CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTMATCH.key, this.get(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTMATCH.key));prop.putXML(CRAWL_PROFILE_PREFIX + count + "_" + CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTNOTMATCH.key, this.get(CrawlAttribute.INDEXING_MEDIA_TYPE_MUSTNOTMATCH.key));prop.putXML(CRAWL_PROFILE_PREFIX + count + "_" + CrawlAttribute.INDEXING_SOLR_QUERY_MUSTMATCH.key, this.get(CrawlAttribute.INDEXING_SOLR_QUERY_MUSTMATCH.key));diff --git a/source/net/yacy/crawler/retrieval/Response.java b/source/net/yacy/crawler/retrieval/Response.javaindex acf7c6fd0..567685510 100644--- a/source/net/yacy/crawler/retrieval/Response.java+++ b/source/net/yacy/crawler/retrieval/Response.java@@ -175,10 +175,10 @@ public class Response {int p = mime.indexOf('/');if (p < 0) return new String[]{mime};if (doctype == DT_TEXT) return new String[]{"text" + mime.substring(p)};- if (doctype == DT_IMAGE) return new String[]{"image" + mime.substring(p)};- if (doctype == DT_AUDIO) return new String[]{"audio" + mime.substring(p)};- if (doctype == DT_MOVIE) return new String[]{"video" + mime.substring(p)};- return new String[]{mime};+ if (doctype == DT_IMAGE) return new String[]{"image" + mime.substring(p)};+ if (doctype == DT_AUDIO) return new String[]{"audio" + mime.substring(p)};+ if (doctype == DT_MOVIE) return new String[]{"video" + mime.substring(p)};+ return new String[]{mime};}public static final int QUEUE_STATE_FRESH = 0;@@ -235,16 +235,16 @@ public class Response {* @return the original request that produced this response*/public Request getRequest() {- return request;- }+ return request;+ }public ResponseHeader getResponseHeader() {return this.responseHeader;}public RequestHeader getRequestHeader() {- return this.requestHeader;- }+ return this.requestHeader;+ }public boolean fromCache() {return this.fromCache;@@ -260,11 +260,11 @@ public class Response {return this.request.name();}- /**- * @return the requested URL that produced this response. When redirection(s)- * occurred, this is not the initial URL, but the last redirection- * target.- */+ /**+ * @return the requested URL that produced this response. When redirection(s)+ * occurred, this is not the initial URL, but the last redirection+ * target.+ */public DigestURL url() {return this.request.url();}@@ -745,11 +745,11 @@ public class Response {// -ranges in request// we checked that in shallStoreCache- /*- * Eventually check if a parser supports the media yype. Depending on the crawl- * profile, the indexingDocumentProcessor can eventually index only URL metadata- * using the generic parser for unsupported media types- */+ /*+ * Eventually check if a parser supports the media yype. Depending on the crawl+ * profile, the indexingDocumentProcessor can eventually index only URL metadata+ * using the generic parser for unsupported media types+ */if (this.responseHeader != null && !profile().isIndexNonParseableUrls()) {final String mimeType = this.responseHeader.getContentType();final String parserError = TextParser.supportsMime(mimeType);diff --git a/source/net/yacy/document/Document.java b/source/net/yacy/document/Document.javaindex ba3365d5c..34676836a 100644--- a/source/net/yacy/document/Document.java+++ b/source/net/yacy/document/Document.java@@ -91,12 +91,12 @@ public class Document {/** links to icons that belongs to the document (mapped by absolute URL) */private Map<DigestURL, IconEntry> icons;- /**- * URLs of linked data item types/classes referenced by the document (for example in- * HTML with standard annotations such as RDFa, microdata, microformats or- * JSON-LD)- */- private Set<DigestURL> linkedDataTypes;+ /**+ * URLs of linked data item types/classes referenced by the document (for example in+ * HTML with standard annotations such as RDFa, microdata, microformats or+ * JSON-LD)+ */+ private Set<DigestURL> linkedDataTypes;private boolean resorted;private final Set<String> languages;private boolean indexingDenied;@@ -131,13 +131,13 @@ public class Document {this.parserObject = parserObject;this.keywords = new LinkedHashSet<String>();if (keywords != null) {- Collections.addAll(this.keywords, keywords);+ Collections.addAll(this.keywords, keywords);}this.titles = (titles == null) ? new ArrayList<String>(1) : titles;this.creator = (author == null) ? new StringBuilder(0) : new StringBuilder(author);this.sections = new LinkedList<String>() ;if (sections != null) {- Collections.addAll(this.sections, sections);+ Collections.addAll(this.sections, sections);}this.descriptions = (abstrcts == null) ? new ArrayList<String>() : abstrcts;if (lat >= -90.0d && lat <= 90.0d && lon >= -180.0d && lon <= 180.0d) {@@ -216,13 +216,21 @@ public class Document {}this.scraperObject = scraper;}++ public AnchorURL getCanonical() {+ final Object scraper = this.getScraperObject();+ if (!(scraper instanceof ContentScraper)) return null;+ final ContentScraper html = (ContentScraper) scraper;+ AnchorURL canonical = html.getCanonical();+ return canonical;+ }public Set<String> getContentLanguages() {return this.languages;}public String getFileName() {- return this.source.getFileName();+ return this.source.getFileName();}public Map<String, Set<String>> getGenericFacets() {@@ -233,15 +241,15 @@ public class Document {* @return true when this document is the result of a partially parsed resource, for example due to resource content size exceeding a given limit*/public boolean isPartiallyParsed() {- return this.partiallyParsed;- }+ return this.partiallyParsed;+ }/*** @param partiallyParsed set to true to indicates this document is the result of a partially parsed resource, for example due to resource content size exceeding a given limit*/public void setPartiallyParsed(final boolean partiallyParsed) {- this.partiallyParsed = partiallyParsed;- }+ this.partiallyParsed = partiallyParsed;+ }/*** compute a set of languages that this document contains@@ -637,13 +645,13 @@ dc_rights// we add artificial hyperlinks to the hyperlink set// that can be calculated from given hyperlinks and imagelinks- /*- * Should we also include icons ? with- * this.hyperlinks.putAll(allReflinks(this.icons.keySet())); It is- * problematic as allReflinks will modify icons set set, removing those whose URL is- * starting with "/www" but it is not desired for icons such as- * www.wikipedia.org/static/favicon/wikipedia.ico- */+ /*+ * Should we also include icons ? with+ * this.hyperlinks.putAll(allReflinks(this.icons.keySet())); It is+ * problematic as allReflinks will modify icons set set, removing those whose URL is+ * starting with "/www" but it is not desired for icons such as+ * www.wikipedia.org/static/favicon/wikipedia.ico+ */this.hyperlinks.putAll(allReflinks(this.images.values()));this.hyperlinks.putAll(allReflinks(this.audiolinks.keySet()));@@ -804,16 +812,16 @@ dc_rights}InputStream textStream = doc.getTextStream();try {- FileUtils.copy(textStream, (ByteArrayOutputStream) this.text);+ FileUtils.copy(textStream, (ByteArrayOutputStream) this.text);} finally {- try {- if(textStream != null) {- /* textStream can be a FileInputStream : we must close it to ensure releasing system resource */- textStream.close();- }- } catch(IOException e) {- ConcurrentLog.warn("DOCUMENT", "Could not close text input stream");- }+ try {+ if(textStream != null) {+ /* textStream can be a FileInputStream : we must close it to ensure releasing system resource */+ textStream.close();+ }+ } catch(IOException e) {+ ConcurrentLog.warn("DOCUMENT", "Could not close text input stream");+ }}this.anchors.addAll(doc.getAnchors());@@ -826,41 +834,41 @@ dc_rights* @return links to icons that belongs to the document (mapped by absolute URL)*/public Map<DigestURL, IconEntry> getIcons() {- return icons;- }+ return icons;+ }/*** Set links to icons that belongs to the document (mapped by absolute URL)* @param icons*/public void setIcons(final Map<DigestURL, IconEntry> icons) {- /* Better to ensure now icons property will not be null */- if(icons != null) {- this.icons = icons;- } else {- this.icons = new HashMap<>();- }- }+ /* Better to ensure now icons property will not be null */+ if(icons != null) {+ this.icons = icons;+ } else {+ this.icons = new HashMap<>();+ }+ }- /**- * @return URLs of linked data item types/classes referenced by the document (for example in- * HTML with standard annotations such as RDFa, microdata, microformats or- * JSON-LD)- */+ /**+ * @return URLs of linked data item types/classes referenced by the document (for example in+ * HTML with standard annotations such as RDFa, microdata, microformats or+ * JSON-LD)+ */public Set<DigestURL> getLinkedDataTypes() {- return this.linkedDataTypes;- }+ return this.linkedDataTypes;+ }- /**- * @return URLs of linked data item types/classes referenced by the document- */+ /**+ * @return URLs of linked data item types/classes referenced by the document+ */public void setLinkedDataTypes(final Set<DigestURL> linkedDataTypes) {- if(linkedDataTypes != null) {- /* Ensure non null property */- this.linkedDataTypes = linkedDataTypes;- } else {- this.linkedDataTypes.clear();- }+ if(linkedDataTypes != null) {+ /* Ensure non null property */+ this.linkedDataTypes = linkedDataTypes;+ } else {+ this.linkedDataTypes.clear();+ }}@@ -1034,14 +1042,14 @@ dc_rights} catch (final IOException e) {ConcurrentLog.logException(e);} finally {- try {- if(textStream != null) {- /* textStream can be a FileInputStream : we must close it to ensure releasing system resource */- textStream.close();- }- } catch (IOException e) {- ConcurrentLog.warn("DOCUMENT", "Could not close text input stream");- }+ try {+ if(textStream != null) {+ /* textStream can be a FileInputStream : we must close it to ensure releasing system resource */+ textStream.close();+ }+ } catch (IOException e) {+ ConcurrentLog.warn("DOCUMENT", "Could not close text input stream");+ }}}anchors.addAll(doc.getAnchors());@@ -1098,7 +1106,7 @@ dc_rightspublic final static String IFRAME_MARKER = "iframe";public final static String FRAME_MARKER = "frame";public final static String EMBED_MARKER = "embed";-+public static Map<AnchorURL, String> getHyperlinks(final Document[] documents, boolean includeNofollow) {final Map<AnchorURL, String> result = new HashMap<>();for (final Document d: documents) {diff --git a/source/net/yacy/htroot/CrawlStartExpert.java b/source/net/yacy/htroot/CrawlStartExpert.javaindex 9c8d55149..56b17d767 100644--- a/source/net/yacy/htroot/CrawlStartExpert.java+++ b/source/net/yacy/htroot/CrawlStartExpert.java@@ -369,6 +369,13 @@ public class CrawlStartExpert {}}+ // Check Canonical?+ if (post == null) {+ prop.put("noindexWhenCanonicalUnequalURLChecked", 1);+ } else {+ prop.put("noindexWhenCanonicalUnequalURLChecked",+ post.getBoolean("noindexWhenCanonicalUnequalURL") ? 1 : 0);+ }// ---------- Clean-Up before Crawl Start// delete if older settings: number valuediff --git a/source/net/yacy/htroot/Crawler_p.java b/source/net/yacy/htroot/Crawler_p.javaindex 98ca68dc1..d6b19e883 100644--- a/source/net/yacy/htroot/Crawler_p.java+++ b/source/net/yacy/htroot/Crawler_p.java@@ -316,6 +316,7 @@ public class Crawler_p {final String indexUrlMustNotMatch = post.get("indexmustnotmatch", CrawlProfile.MATCH_NEVER_STRING);final String indexContentMustMatch = post.get("indexcontentmustmatch", CrawlProfile.MATCH_ALL_STRING);final String indexContentMustNotMatch = post.get("indexcontentmustnotmatch", CrawlProfile.MATCH_NEVER_STRING);+ final boolean noindexWhenCanonicalUnequalURL = "on".equals(post.get("noindexWhenCanonicalUnequalURL", "off"));final boolean crawlOrder = post.get("crawlOrder", "off").equals("on");env.setConfig("crawlOrder", crawlOrder);@@ -614,6 +615,7 @@ public class Crawler_p {indexUrlMustNotMatch,indexContentMustMatch,indexContentMustNotMatch,+ noindexWhenCanonicalUnequalURL,newcrawlingdepth,directDocByURL,crawlingIfOlder,diff --git a/source/net/yacy/htroot/QuickCrawlLink_p.java b/source/net/yacy/htroot/QuickCrawlLink_p.javaindex 4230b69ea..6c2f432f4 100644--- a/source/net/yacy/htroot/QuickCrawlLink_p.java+++ b/source/net/yacy/htroot/QuickCrawlLink_p.java@@ -150,6 +150,7 @@ public class QuickCrawlLink_p {CrawlProfile.MATCH_NEVER_STRING, //indexUrlMustNotMatchCrawlProfile.MATCH_ALL_STRING, //indexContentMustMatchCrawlProfile.MATCH_NEVER_STRING, //indexContentMustNotMatch+ false,CrawlingDepth,true,CrawlProfile.getRecrawlDate(60 * 24 * 30), // recrawlIfOlder (minutes); here: one monthdiff --git a/source/net/yacy/search/Switchboard.java b/source/net/yacy/search/Switchboard.javaindex 34ebfcc94..50902296e 100644--- a/source/net/yacy/search/Switchboard.java+++ b/source/net/yacy/search/Switchboard.java@@ -3152,28 +3152,73 @@ public final class Switchboard extends serverSwitch {return new IndexingQueueEntry(in.queueEntry, in.documents, null);}}- if (!(profile.indexUrlMustMatchPattern() == CrawlProfile.MATCH_ALL_PATTERN || profile.indexUrlMustMatchPattern().matcher(urls).matches()) ||- (profile.indexUrlMustNotMatchPattern() != CrawlProfile.MATCH_NEVER_PATTERN && profile.indexUrlMustNotMatchPattern().matcher(urls).matches())) {- if (this.log.isInfo()) this.log.info("Not Condensed Resource '" + urls + "': indexing prevented by regular expression on url; indexUrlMustMatchPattern = " + profile.indexUrlMustMatchPattern().pattern() + ", indexUrlMustNotMatchPattern = " + profile.indexUrlMustNotMatchPattern().pattern());++ // check mustmatch pattern+ Pattern mustmatchurl = profile.indexUrlMustMatchPattern();+ if (mustmatchurl != CrawlProfile.MATCH_ALL_PATTERN && !mustmatchurl.matcher(urls).matches()) {+ String info = "Not Condensed Resource '" + urls + "': indexing prevented by regular expression on url; indexUrlMustMatchPattern = " + mustmatchurl.pattern();+ if (this.log.isInfo()) this.log.info(info);+ // create a new errorURL DB entry+ this.crawlQueues.errorURL.push(in.queueEntry.url(), in.queueEntry.depth(), profile, FailCategory.FINAL_PROCESS_CONTEXT, info, -1);+ return new IndexingQueueEntry(in.queueEntry, in.documents, null);+ }++ // check mustnotmatch+ Pattern mustnotmatchurl = profile.indexUrlMustNotMatchPattern();+ if (mustnotmatchurl != CrawlProfile.MATCH_NEVER_PATTERN && mustnotmatchurl.matcher(urls).matches()) {+ String info = "Not Condensed Resource '" + urls + "': indexing prevented by regular expression on url; indexUrlMustNotMatchPattern = " + mustnotmatchurl;+ if (this.log.isInfo()) this.log.info(info);// create a new errorURL DB entry- this.crawlQueues.errorURL.push(in.queueEntry.url(), in.queueEntry.depth(), profile, FailCategory.FINAL_PROCESS_CONTEXT, "indexing prevented by regular expression on url; indexUrlMustMatchPattern = " + profile.indexUrlMustMatchPattern().pattern() + ", indexUrlMustNotMatchPattern = " + profile.indexUrlMustNotMatchPattern().pattern(), -1);+ this.crawlQueues.errorURL.push(in.queueEntry.url(), in.queueEntry.depth(), profile, FailCategory.FINAL_PROCESS_CONTEXT, info, -1);return new IndexingQueueEntry(in.queueEntry, in.documents, null);}// check which files may take part in the indexing processfinal List<Document> doclist = new ArrayList<>();docloop: for (final Document document : in.documents) {++ // check canonical+ if (profile.noindexWhenCanonicalUnequalURL()) {+ AnchorURL canonical = document.getCanonical();+ DigestURL source = document.dc_source();+ if (canonical != null && source != null) {+ String canonical_norm = canonical.toNormalform(true);+ String source_norm = source.toNormalform(true);+ if (!canonical_norm.equals(source_norm)) {+ String info = "Not Condensed Resource '" + urls + "': denied, canonical != source; canonical = " +canonical_norm + "; source = " + source_norm;+ if (this.log.isInfo()) this.log.info(info);+ // create a new errorURL DB entry+ this.crawlQueues.errorURL.push(in.queueEntry.url(), in.queueEntry.depth(), profile, FailCategory.FINAL_PROCESS_CONTEXT, info, -1);+ continue docloop;+ }+ }+ }++ // check indexing denied flagsif (document.indexingDenied() && profile.obeyHtmlRobotsNoindex() && !this.isIntranetMode()) {if (this.log.isInfo()) this.log.info("Not Condensed Resource '" + urls + "': denied by document-attached noindexing rule");// create a new errorURL DB entrythis.crawlQueues.errorURL.push(in.queueEntry.url(), in.queueEntry.depth(), profile, FailCategory.FINAL_PROCESS_CONTEXT, "denied by document-attached noindexing rule", -1);continue docloop;}- if (!(profile.indexContentMustMatchPattern() == CrawlProfile.MATCH_ALL_PATTERN || profile.indexContentMustMatchPattern().matcher(document.getTextString()).matches()) ||- (profile.indexContentMustNotMatchPattern() != CrawlProfile.MATCH_NEVER_PATTERN && profile.indexContentMustNotMatchPattern().matcher(document.getTextString()).matches())) {- if (this.log.isInfo()) this.log.info("Not Condensed Resource '" + urls + "': indexing prevented by regular expression on content; indexContentMustMatchPattern = " + profile.indexContentMustMatchPattern().pattern() + ", indexContentMustNotMatchPattern = " + profile.indexContentMustNotMatchPattern().pattern());++ // check content pattern must-match+ Pattern mustmatchcontent = profile.indexContentMustMatchPattern();+ if (mustmatchcontent != CrawlProfile.MATCH_ALL_PATTERN && !mustmatchcontent.matcher(document.getTextString()).matches()) {+ String info = "Not Condensed Resource '" + urls + "': indexing prevented by regular expression on content; indexContentMustMatchPattern = " + mustmatchcontent.pattern() ;+ if (this.log.isInfo()) this.log.info(info);+ // create a new errorURL DB entry+ this.crawlQueues.errorURL.push(in.queueEntry.url(), in.queueEntry.depth(), profile, FailCategory.FINAL_PROCESS_CONTEXT, info, -1);+ continue docloop;+ }++ // check content pattern must-not-match+ Pattern mustnotmatchcontent = profile.indexContentMustNotMatchPattern();+ if (mustnotmatchcontent != CrawlProfile.MATCH_NEVER_PATTERN && mustnotmatchcontent.matcher(document.getTextString()).matches()) {+ String info = "Not Condensed Resource '" + urls + "': indexing prevented by regular expression on content; indexContentMustNotMatchPattern = " + mustnotmatchcontent.pattern();+ if (this.log.isInfo()) this.log.info(info);// create a new errorURL DB entry- this.crawlQueues.errorURL.push(in.queueEntry.url(), in.queueEntry.depth(), profile, FailCategory.FINAL_PROCESS_CONTEXT, "indexing prevented by regular expression on content; indexContentMustMatchPattern = " + profile.indexContentMustMatchPattern().pattern() + ", indexContentMustNotMatchPattern = " + profile.indexContentMustNotMatchPattern().pattern(), -1);+ this.crawlQueues.errorURL.push(in.queueEntry.url(), in.queueEntry.depth(), profile, FailCategory.FINAL_PROCESS_CONTEXT, info, -1);continue docloop;}