Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
added nav filter
Michael Peter Christen187075b878f9
diff --git a/htroot/AccessTracker_p.java b/htroot/AccessTracker_p.javaindex 6dcb80c71..07f9e422d 100644--- a/htroot/AccessTracker_p.java+++ b/htroot/AccessTracker_p.java@@ -24,12 +24,10 @@// along with this program; if not, write to the Free Software// Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA-import java.util.AbstractMap;import java.util.Collection;import java.util.ConcurrentModificationException;import java.util.Date;import java.util.Iterator;-import java.util.LinkedList;import java.util.List;import java.util.Map;import java.util.Map.Entry;@@ -39,9 +37,7 @@ import java.util.concurrent.LinkedBlockingQueue;import net.yacy.cora.date.GenericFormatter;import net.yacy.cora.protocol.Domains;import net.yacy.cora.protocol.RequestHeader;-import net.yacy.cora.sorting.ConcurrentScoreMap;import net.yacy.cora.sorting.OrderedScoreMap;-import net.yacy.cora.sorting.ScoreMap;import net.yacy.cora.util.CommonPattern;import net.yacy.cora.util.ConcurrentLog;import net.yacy.peers.Seed;diff --git a/htroot/CrawlStartExpert.html b/htroot/CrawlStartExpert.htmlindex b07b26aed..0681684ba 100644--- a/htroot/CrawlStartExpert.html+++ b/htroot/CrawlStartExpert.html@@ -370,10 +370,10 @@<legend>Content Filter</legend><p>These are limitations on parts of a document. The filter will be applied after a web page was loaded.</p><dl>- <dt>Filter div class names</dt>+ <dt>Filter div or nav class names</dt><dd><table style="border-width: 0px">- <tr><td style="width:110px">set of CSS class names</td><td><input name="ignoreclassname" id="ignoreclassname" type="text" size="55" maxlength="100000" value="#[ignoreclassname]#" onblur="if (this.value=='') this.value='';"/></td><td>comma-separated list of <div> element class names which should be filtered out</td></tr>+ <tr><td style="width:110px">set of CSS class names</td><td><input name="ignoreclassname" id="ignoreclassname" type="text" size="55" maxlength="100000" value="#[ignoreclassname]#" onblur="if (this.value=='') this.value='';"/></td><td>comma-separated list of <div> or <nav> element class names which should be filtered out</td></tr></table></dd></dl>diff --git a/source/net/yacy/cora/protocol/Scanner.java b/source/net/yacy/cora/protocol/Scanner.javaindex 6d11388aa..8c84931a9 100644--- a/source/net/yacy/cora/protocol/Scanner.java+++ b/source/net/yacy/cora/protocol/Scanner.java@@ -152,6 +152,7 @@ public class Scanner {if (access != Access.unknown) Scanner.this.services.put(this, access);}} catch (final OutOfMemoryError e) {+ e.printStackTrace();}}public long age() {diff --git a/source/net/yacy/document/parser/html/ContentScraper.java b/source/net/yacy/document/parser/html/ContentScraper.javaindex 1a4d46bab..36fb1e34c 100644--- a/source/net/yacy/document/parser/html/ContentScraper.java+++ b/source/net/yacy/document/parser/html/ContentScraper.java@@ -129,6 +129,7 @@ public class ContentScraper extends AbstractScraper implements Scraper {script(TagType.pair),span(TagType.pair),div(TagType.pair),+ nav(TagType.pair),article(TagType.pair), // html5time(TagType.pair), // html5 <time datetime>// tags used to capture tag content@@ -1020,7 +1021,10 @@ public class ContentScraper extends AbstractScraper implements Scraper {}/* Parent is not marked as ignored : let's check the current tag */- if (!ignore && this.ignoreDivClassNames != null && tag != null && TagName.div.name().equals(tag.name)) {+ if (!ignore &&+ this.ignoreDivClassNames != null &&+ tag != null &&+ (TagName.div.name().equals(tag.name) || TagName.nav.name().equals(tag.name))) {final String classAttr = tag.opts.getProperty("class", EMPTY_STRING);final Set<String> classes = ContentScraper.parseSpaceSeparatedTokens(classAttr);ignore = !Collections.disjoint(this.ignoreDivClassNames, classes);