Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
URL Viewer : apply crawler size limits when adding to local index.
luccioman8100c033a23d
diff --git a/htroot/ViewFile.java b/htroot/ViewFile.javaindex bfa6c4a65..5a8037637 100644--- a/htroot/ViewFile.java+++ b/htroot/ViewFile.java@@ -363,10 +363,15 @@ public class ViewFile {prop.put("showSnippet_teasertext", desc);prop.put("showSnippet", 1);}- // update index with parsed resouce if index entry is older or missing- if (urlEntry == null || urlEntry.loaddate().before(response.lastModified())) {- Switchboard.getSwitchboard().toIndexer(response);- }+ // update index with parsed resource if index entry is older or missing+ final long responseSize = response.size();+ if (urlEntry == null || urlEntry.loaddate().before(response.lastModified())) {+ /* Also check resource size is lower than configured crawler limits */+ if (responseSize >= 0+ && responseSize <= Switchboard.getSwitchboard().loader.protocolMaxFileSize(response.url())) {+ Switchboard.getSwitchboard().toIndexer(response);+ }+ }if (document != null) document.close();}prop.put("error", "0");