Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
eleminate dependency on file-extension in storeDocument but use supported mime-type to also support handling of urls w/o corresponding file-extension. For this refactor use of document.getParserObject() to alway return a Parser (for clean logic) and define/move the scraperObject as local var of AbstractParser. Adjust related calls to getParserObject (where actually a scraperObject is wanted). Addionally skip appending url token to parsed text for dht metadata entries (by default returned as result by rwi index).
reger4c7a77662a90
diff --git a/source/net/yacy/document/AbstractParser.java b/source/net/yacy/document/AbstractParser.javaindex 15f300d7b..daf4a91a2 100644--- a/source/net/yacy/document/AbstractParser.java+++ b/source/net/yacy/document/AbstractParser.java@@ -39,7 +39,8 @@ public abstract class AbstractParser implements Parser {protected final Set<String> SUPPORTED_MIME_TYPES = new LinkedHashSet<String>();protected final Set<String> SUPPORTED_EXTENSIONS = new HashSet<String>();private final String name;-+ protected Object scraperObject; // used scraper or source object if any, otherwise null+/*** initialize a parser with a name* @param namediff --git a/source/net/yacy/document/Document.java b/source/net/yacy/document/Document.javaindex 75b56606f..419dcb625 100644--- a/source/net/yacy/document/Document.java+++ b/source/net/yacy/document/Document.java@@ -92,13 +92,13 @@ public class Document {private final Set<String> languages;private boolean indexingDenied;private final double lon, lat;- private final Object parserObject; // the source object that was used to create the Document+ private final Parser parserObject; // the source object that was used to create the Documentprivate final Map<String, Set<String>> generic_facets; // a map from vocabulary names to the set of tags for that vocabulary which apply for this documentprivate final Date lastModified;private int crawldepth;public Document(final DigestURL location, final String mimeType, final String charset,- final Object parserObject,+ final Parser parserObject,final Set<String> languages,final String[] keywords,final List<String> titles,@@ -160,11 +160,29 @@ public class Document {if (contentDomain != ContentDomain.ALL) return contentDomain;return this.dc_source().getContentDomainFromExt();}-- public Object getParserObject() {++ /**+ * The parser used to generate the document+ * @return Parser+ */+ public Parser getParserObject() {return this.parserObject;}+ /**+ * Confinient call to get the source/scraper object of the underlaying parser+ * if the parser uses a scraper, like htmlParser+ * @return scraper object typically of type ContentScraper but may also of type DCEntry+ */+ public Object getScraperObject() {+ if (this.parserObject instanceof AbstractParser) {+ if (((AbstractParser) this.parserObject).scraperObject != null) {+ return ((AbstractParser) this.parserObject).scraperObject;+ }+ }+ return null;+ }+public Set<String> getContentLanguages() {return this.languages;}@@ -931,9 +949,9 @@ dc_rights// clean up parser datafor (final Document doc: docs) {- Object parserObject = doc.getParserObject();- if (parserObject instanceof ContentScraper) {- final ContentScraper html = (ContentScraper) parserObject;+ Object scraper = doc.getScraperObject();+ if (scraper instanceof ContentScraper) {+ final ContentScraper html = (ContentScraper) scraper;html.close();}}@@ -979,9 +997,9 @@ dc_rightsif (!entry.getKey().attachedNofollow()) result.put(entry.getKey(), entry.getValue());}}- final Object parser = d.getParserObject();- if (parser instanceof ContentScraper) {- final ContentScraper html = (ContentScraper) parser;+ final Object scraper = d.getScraperObject();+ if (scraper instanceof ContentScraper) {+ final ContentScraper html = (ContentScraper) scraper;String refresh = html.getRefreshPath();if (refresh != null && refresh.length() > 0) try {result.put(new AnchorURL(refresh), "refresh");} catch (final MalformedURLException e) {}AnchorURL canonical = html.getCanonical();diff --git a/source/net/yacy/document/content/DCEntry.java b/source/net/yacy/document/content/DCEntry.javaindex f60af57d3..8d54f93c3 100644--- a/source/net/yacy/document/content/DCEntry.java+++ b/source/net/yacy/document/content/DCEntry.java@@ -45,6 +45,8 @@ import net.yacy.cora.document.id.DigestURL;import net.yacy.cora.util.CommonPattern;import net.yacy.cora.util.ConcurrentLog;import net.yacy.document.Document;+import net.yacy.document.parser.genericParser;+import net.yacy.search.schema.CollectionSchema;public class DCEntry extends MultiMapSolrParams {@@ -330,11 +332,15 @@ public class DCEntry extends MultiMapSolrParams {languages.add(getLanguage());List<String> t = new ArrayList<String>(1);t.add(getTitle());++ // for processing during indexing, embed entry as source scraperObject in a standard parserobj object+ genericParser parserobj = new genericParser(this); // init the simplest parser with DCEntry as source/scraperObject used during indexing+return new Document(getIdentifier(true),"text/html",StandardCharsets.UTF_8.name(),- this,+ parserobj,languages,getSubject(), // might be nullt,@@ -343,7 +349,7 @@ public class DCEntry extends MultiMapSolrParams {null,getDescriptions(),getLon(), getLat(),- get("text_t", ""),+ get(CollectionSchema.text_t.name(), ""),null,null,null,diff --git a/source/net/yacy/document/parser/genericParser.java b/source/net/yacy/document/parser/genericParser.javaindex 0d6d64d6b..1d9f2d1c8 100644--- a/source/net/yacy/document/parser/genericParser.java+++ b/source/net/yacy/document/parser/genericParser.java@@ -44,6 +44,17 @@ public class genericParser extends AbstractParser implements Parser {// this parser is used if no other fits. This parser fits all}+ /**+ * Constructor to allow to set a scraperObject+ * because it is desired to keep the scraper/source object protected+ * This is used for surrogate parsers to set a other source/scraper then ContentScraper+ * @param scraper+ */+ public genericParser(Object scraper) {+ super("Generic Parser");+ this.scraperObject = scraper;+ }+@Overridepublic Document[] parse(final DigestURL location,diff --git a/source/net/yacy/document/parser/htmlParser.java b/source/net/yacy/document/parser/htmlParser.javaindex f668d3001..b293404bb 100644--- a/source/net/yacy/document/parser/htmlParser.java+++ b/source/net/yacy/document/parser/htmlParser.java@@ -34,9 +34,7 @@ import java.nio.charset.Charset;import java.nio.charset.IllegalCharsetNameException;import java.nio.charset.StandardCharsets;import java.nio.charset.UnsupportedCharsetException;-import java.util.HashSet;import java.util.LinkedHashMap;-import java.util.Set;import net.yacy.cora.document.encoding.UTF8;import net.yacy.cora.document.id.DigestURL;@@ -60,21 +58,29 @@ public class htmlParser extends AbstractParser implements Parser {private static final int maxLinks = 10000;- public final static String[] htmlExtensions = new String[]{- "htm","html","shtml","shtm","stm","xhtml","phtml","phtm",- "tpl","php","php2","php3","php4","php5","cfm","asp","aspx","tex","txt","msg"- };-- public final static Set<String> htmlExtensionsSet;-- static {- htmlExtensionsSet = new HashSet<>(htmlExtensions.length);- for (String ext: htmlExtensions) htmlExtensionsSet.add(ext);- }-public htmlParser() {super("Streaming HTML Parser");- this.SUPPORTED_EXTENSIONS.addAll(htmlExtensionsSet);+ this.SUPPORTED_EXTENSIONS.add("htm");+ this.SUPPORTED_EXTENSIONS.add("html");+ this.SUPPORTED_EXTENSIONS.add("shtml");+ this.SUPPORTED_EXTENSIONS.add("shtm");+ this.SUPPORTED_EXTENSIONS.add("stm");+ this.SUPPORTED_EXTENSIONS.add("xhtml");+ this.SUPPORTED_EXTENSIONS.add("phtml");+ this.SUPPORTED_EXTENSIONS.add("phtm");+ this.SUPPORTED_EXTENSIONS.add("tpl");+ this.SUPPORTED_EXTENSIONS.add("php");+ this.SUPPORTED_EXTENSIONS.add("php2");+ this.SUPPORTED_EXTENSIONS.add("php3");+ this.SUPPORTED_EXTENSIONS.add("php4");+ this.SUPPORTED_EXTENSIONS.add("php5");+ this.SUPPORTED_EXTENSIONS.add("cfm");+ this.SUPPORTED_EXTENSIONS.add("asp");+ this.SUPPORTED_EXTENSIONS.add("aspx");+ this.SUPPORTED_EXTENSIONS.add("tex");+ this.SUPPORTED_EXTENSIONS.add("txt");+ this.SUPPORTED_EXTENSIONS.add("msg");+this.SUPPORTED_MIME_TYPES.add("text/html");this.SUPPORTED_MIME_TYPES.add("text/xhtml+xml");this.SUPPORTED_MIME_TYPES.add("application/xhtml+xml");@@ -97,7 +103,8 @@ public class htmlParser extends AbstractParser implements Parser {try {// first get a document from the parsed htmlCharset[] detectedcharsetcontainer = new Charset[]{null};- final ContentScraper scraper = parseToScraper(location, documentCharset, vocscraper, detectedcharsetcontainer, timezoneOffset, sourceStream, maxLinks);+ scraperObject = parseToScraper(location, documentCharset, vocscraper, detectedcharsetcontainer, timezoneOffset, sourceStream, maxLinks);+ ContentScraper scraper = (ContentScraper)scraperObject; // shortcut to access ContentScraper methodes// parseToScraper also detects/corrects/sets charset from html content tagfinal Document document = transformScraper(location, mimeType, detectedcharsetcontainer[0].name(), scraper);Document documentSnapshot = null;@@ -130,7 +137,7 @@ public class htmlParser extends AbstractParser implements Parser {* @param scraper* @return*/- private static Document transformScraper(final DigestURL location, final String mimeType, final String charSet, final ContentScraper scraper) {+ private Document transformScraper(final DigestURL location, final String mimeType, final String charSet, final ContentScraper scraper) {final String[] sections = new String[scraper.getHeadlines(1).length +scraper.getHeadlines(2).length +@@ -150,7 +157,7 @@ public class htmlParser extends AbstractParser implements Parser {location,mimeType,charSet,- scraper,+ this,scraper.getContentLanguages(),scraper.getKeywords(),scraper.getTitles(),@@ -178,7 +185,7 @@ public class htmlParser extends AbstractParser implements Parser {} catch (UnsupportedEncodingException e) {sourceStream = new ByteArrayInputStream(UTF8.getBytes(input));}- ContentScraper scraper;+ ContentScraper scraper; // for this static methode no need to init local this.scraperObjecttry {scraper = parseToScraper(location, documentCharset, vocabularyScraper, detectedcharsetcontainer, timezoneOffset, sourceStream, maxLinks);} catch (Failure e) {@@ -242,6 +249,7 @@ public class htmlParser extends AbstractParser implements Parser {}// parsing the content+ // for this static methode no need to init local this.scraperObject herefinal ContentScraper scraper = new ContentScraper(location, maxLinks, vocabularyScraper, timezoneOffset);final TransformerWriter writer = new TransformerWriter(null,null,scraper,null,false, Math.max(64, Math.min(4096, sourceStream.available())));try {diff --git a/source/net/yacy/document/parser/swfParser.java b/source/net/yacy/document/parser/swfParser.javaindex 81bd0473d..f0c7a163f 100644--- a/source/net/yacy/document/parser/swfParser.java+++ b/source/net/yacy/document/parser/swfParser.java@@ -68,10 +68,9 @@ public class swfParser extends AbstractParser implements Parser {try {final SWF2HTML swf2html = new SWF2HTML();String contents = "";- ContentScraper htmlscraper=null;try {- contents = swf2html.convertSWFToHTML(source);- htmlscraper = htmlParser.parseToScraper(location, charset, scraper, timezoneOffset, contents, 100);+ contents = swf2html.convertSWFToHTML(source);+ scraperObject = htmlParser.parseToScraper(location, charset, scraper, timezoneOffset, contents, 100);} catch (final NegativeArraySizeException e) {throw new Parser.Failure(e.getMessage(), location);} catch (final IOException e) {@@ -79,29 +78,9 @@ public class swfParser extends AbstractParser implements Parser {} catch (final Exception e) {throw new Parser.Failure(e.getMessage(), location);}- /*- String url = null;- String urlnr = null;- final String linebreak = System.getProperty("line.separator");- final List<AnchorURL> anchors = new ArrayList<AnchorURL>();- int urls = 0;- int urlStart = -1;- int urlEnd = 0;- int p0 = 0;- //extracting urls- while ((urlStart = contents.indexOf("http://",urlEnd)) >= 0){- urlEnd = contents.indexOf(linebreak,urlStart);- url = contents.substring(urlStart,urlEnd);- urlnr = Integer.toString(++urls);- AnchorURL u = new AnchorURL(url);- u.setNameProperty(urlnr);- anchors.add(u);- contents = contents.substring(0,urlStart)+contents.substring(urlEnd);- }- */-- // As the result of parsing this function must return a plasmaParserDocument object+ // As the result of parsing this function must return a plasmaParserDocument object+ ContentScraper htmlscraper = (ContentScraper) this.scraperObject; // shortcut to access ContentScraper methodesreturn new Document[]{new Document(location, // url of the source documentmimeType, // the documents mime typediff --git a/source/net/yacy/search/index/Segment.java b/source/net/yacy/search/index/Segment.javaindex 6814f8737..be50a96bc 100644--- a/source/net/yacy/search/index/Segment.java+++ b/source/net/yacy/search/index/Segment.java@@ -601,11 +601,14 @@ public class Segment {crawlProfile != null && document.getDepth() <= crawlProfile.snapshotMaxdepth() &&!crawlProfile.snapshotsMustnotmatch().matcher(urlNormalform).matches()) {// load pdf in case that is wanted. This can later be used to compute a web page preview in the search results- String ext = MultiProtocolURL.getFileExtension(url.getFile()).toLowerCase();- if (ext.length() == 0 || url.getFile().length() <= 1 || htmlParser.htmlExtensionsSet.contains(ext)) {+ Parser p = document.getParserObject();+ boolean mimesupported = false;+ if (p instanceof htmlParser)+ mimesupported = ((htmlParser)p).supportedMimeTypes().contains(document.dc_format());++ if (mimesupported)// STORE IMAGE AND METADATATransactions.store(vector, true, crawlProfile.snapshotLoadImage(), crawlProfile.snapshotReplaceold(), proxy, acceptLanguage);- }}// STORE TO SOLRdiff --git a/source/net/yacy/search/schema/CollectionConfiguration.java b/source/net/yacy/search/schema/CollectionConfiguration.javaindex e621e22f4..7fda7067c 100644--- a/source/net/yacy/search/schema/CollectionConfiguration.java+++ b/source/net/yacy/search/schema/CollectionConfiguration.java@@ -335,29 +335,10 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seriif (allAttr || contains(CollectionSchema.audiolinkscount_i)) add(doc, CollectionSchema.audiolinkscount_i, md.laudio());if (allAttr || contains(CollectionSchema.videolinkscount_i)) add(doc, CollectionSchema.videolinkscount_i, md.lvideo());if (allAttr || contains(CollectionSchema.applinkscount_i)) add(doc, CollectionSchema.applinkscount_i, md.lapp());- if (allAttr || contains(CollectionSchema.text_t)) {- // construct the text from other metadata parts.- // This is necessary here since that is used to search the link when no other data (parsed text body) is available- StringBuilder sb = new StringBuilder(120);- // accText(sb, md.dc_title()); // default search field via getQueryFields(), not needed for snippet (always displayed)- // accText(sb, md.dc_creator()); // author is in Default ranking/getQueryFields- // accText(sb, md.dc_publisher()); // has it's own metadata field publisher_t (not part of default queryfields) and mostly N/A- // accText(sb, md.snippet()); // above added to description_txt, default search field via getQueryFields(), description_txt incl. in snippet calculation- accText(sb, md.url().toTokens());- // accText(sb, keywords); // default search field via getQueryFields(), keywords not incl. in snippet calculation- add(doc, CollectionSchema.text_t, sb.toString());- }return doc;}- private static void accText(final StringBuilder sb, String text) {- if (text == null || text.length() == 0) return;- if (sb.length() != 0) sb.append(' ');- text = text.trim();- if (!text.isEmpty() && text.charAt(text.length() - 1) == '.') sb.append(text); else sb.append(text).append('.');- }-public static class Subgraph {public final ArrayList<String>[] urlProtocols, urlStubs, urlAnchorTexts;@SuppressWarnings("unchecked")@@ -541,11 +522,11 @@ public class CollectionConfiguration extends SchemaConfiguration implements SeriSubgraph subgraph = new Subgraph(inboundLinks.size(), outboundLinks.size());int c = 0;- final Object parser = document.getParserObject();+ final Object scraper = document.getScraperObject();boolean containsCanonical = false;DigestURL canonical = null;- if (parser instanceof ContentScraper) {- final ContentScraper html = (ContentScraper) parser;+ if (scraper instanceof ContentScraper) {+ final ContentScraper html = (ContentScraper) scraper;List<ImageEntry> images = html.getImages();// header tags@@ -885,9 +866,9 @@ public class CollectionConfiguration extends SchemaConfiguration implements Seri}}- if (parser instanceof DCEntry) {+ if (scraper instanceof DCEntry) {// the document was created with a surrogate parsing; overwrite all md: -entries to Solr- DCEntry dcentry = (DCEntry) parser;+ DCEntry dcentry = (DCEntry) scraper;for (Map.Entry<String, String[]> entry: dcentry.getMap().entrySet()) {String tag = entry.getKey();if (!tag.startsWith("md:") || tag.length() < 4) continue;