Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
Added HTML microdata typed items parsing capability.
luccioman58b98347296d
diff --git a/source/net/yacy/document/parser/html/ContentScraper.java b/source/net/yacy/document/parser/html/ContentScraper.javaindex b9bbd78cb..76981ffc2 100644--- a/source/net/yacy/document/parser/html/ContentScraper.java+++ b/source/net/yacy/document/parser/html/ContentScraper.java@@ -191,6 +191,13 @@ public class ContentScraper extends AbstractScraper implements Scraper {private final SizeLimitedMap<AnchorURL, EmbedEntry> embeds; // urlhash/embed relationprivate final List<ImageEntry> images;private final SizeLimitedSet<AnchorURL> script, frames, iframes;++ /**+ * URLs of linked data item types referenced from HTML content with standard+ * annotations such as RDFa, microdata, microformats or JSON-LD+ */+ private final SizeLimitedSet<DigestURL> linkedDataTypes;+private final SizeLimitedMap<String, String> metas;private final SizeLimitedMap<String, DigestURL> hreflang, navigation;private LinkedHashSet<String> titles;@@ -260,6 +267,7 @@ public class ContentScraper extends AbstractScraper implements Scraper {this.embeds = new SizeLimitedMap<AnchorURL, EmbedEntry>(maxLinks);this.frames = new SizeLimitedSet<AnchorURL>(maxLinks);this.iframes = new SizeLimitedSet<AnchorURL>(maxLinks);+ this.linkedDataTypes = new SizeLimitedSet<>(maxLinks);this.metas = new SizeLimitedMap<String, String>(maxLinks);this.hreflang = new SizeLimitedMap<String, DigestURL>(maxLinks);this.navigation = new SizeLimitedMap<String, DigestURL>(maxLinks);@@ -543,12 +551,49 @@ public class ContentScraper extends AbstractScraper implements Scraper {}}- private void checkOpts(Tag tag) {+ /**+ * Parse the eventual microdata itemtype attribute of a tag and extract its+ * valid URL tokens when the itemscope attribute is present.+ *+ * @param tagAttributes parsed HTML tag attributes.+ * @return a set of URLs eventually empty when no itemtype attribute is present+ * or when its value is not valid+ * @see <a href="https://www.w3.org/TR/microdata/#dfn-itemtype">itemtype+ * definition at W3C</a>+ * @see <a href=+ * "https://html.spec.whatwg.org/multipage/microdata.html#attr-itemtype">itemtype+ * definition at WHATWG</a>+ */+ private Set<DigestURL> parseMicrodataItemType(final Properties tagAttributes) {+ final Set<DigestURL> types = new HashSet<>();+ if (tagAttributes != null) {+ /*+ * The itemtype attribute must not be specified on elements that do not have an+ * itemscope attribute specified. So we lazily check here for itemscope boolean+ * attribute presence (strictly conforming parsing would also check it has no+ * value or the value is the empty string or "itemscope")+ */+ if (tagAttributes.getProperty("itemscope") != null) {+ final Set<String> itemTypes = parseSpaceSeparatedTokens(tagAttributes.getProperty("itemtype"));++ for (final String itemType : itemTypes) {+ try {+ types.add(new DigestURL(itemType));+ } catch (final MalformedURLException ignored) {+ /* Each itemtype space-separated token must be a valid absolute URL */+ }+ }+ }+ }+ return types;+ }++ private void checkOpts(final Tag tag) {// vocabulary classesfinal String classprop = tag.opts.getProperty("class", EMPTY_STRING);this.vocabularyScraper.check(this.root, classprop, tag.content);- // itemprop (schema.org)+ // itemprop microdata property (standard definition at https://www.w3.org/TR/microdata/#dfn-attr-itemprop)String itemprop = tag.opts.getProperty("itemprop");if (itemprop != null) {String propval = tag.opts.getProperty("content"); // value for <meta itemprop="" content=""> see https://html.spec.whatwg.org/multipage/microdata.html#values@@ -620,7 +665,7 @@ public class ContentScraper extends AbstractScraper implements Scraper {* attribute string, may be null* @return a set of tokens eventually empty*/- public static Set<String> parseSpaceSeparatedTokens(String attr) {+ public static Set<String> parseSpaceSeparatedTokens(final String attr) {Set<String> tokens = new HashSet<>();/* Check attr string is not empty to avoid adding a single empty string* in result */@@ -923,6 +968,22 @@ public class ContentScraper extends AbstractScraper implements Scraper {this.fireScrapeTag1(tag.name, tag.opts, tag.content.getChars());}+ /**+ * Scraping operation applied to any kind of tag opening, being either singleton+ * or paired tag, not restricted to tags listed in+ * {@link ContentScraper#linkTags0} and {@link ContentScraper#linkTags1}.+ */+ @Override+ public void scrapeAnyTagOpening(final String tagName, final Properties tagAttributes) {+ if (tagAttributes != null) {+ /*+ * HTML microdata can be annotated on any kind of tag, so we don't restrict this+ * scraping to the limited sets in linkTags0 and linkTags1+ */+ this.linkedDataTypes.addAll(parseMicrodataItemType(tagAttributes));+ }+ }+/*** Add an anchor to the anchors list, and trigger any eventual listener* @param anchor anchor to add. Must not be null.@@ -1092,6 +1153,14 @@ public class ContentScraper extends AbstractScraper implements Scraper {// returns a url (String) / name (String) relationreturn this.iframes;}++ /**+ * @return URLs of linked data item types referenced from HTML content with standard+ * annotations such as RDFa, microdata, microformats or JSON-LD+ */+ public SizeLimitedSet<DigestURL> getLinkedDataTypes() {+ return this.linkedDataTypes;+ }public Set<AnchorURL> getScript() {return this.script;@@ -1164,7 +1233,7 @@ public class ContentScraper extends AbstractScraper implements Scraper {return this.contentSizeLimitExceeded || this.maxAnchorsExceeded || this.css.isLimitExceeded()|| this.rss.isLimitExceeded() || this.embeds.isLimitExceeded() || this.metas.isLimitExceeded()|| this.hreflang.isLimitExceeded() || this.navigation.isLimitExceeded() || this.script.isLimitExceeded()- || this.frames.isLimitExceeded() || this.iframes.isLimitExceeded();+ || this.frames.isLimitExceeded() || this.iframes.isLimitExceeded() || this.linkedDataTypes.isLimitExceeded();}/*@@ -1384,6 +1453,7 @@ public class ContentScraper extends AbstractScraper implements Scraper {this.script.clear();this.frames.clear();this.iframes.clear();+ this.linkedDataTypes.clear();this.embeds.clear();this.images.clear();this.icons.clear();diff --git a/source/net/yacy/document/parser/html/Scraper.java b/source/net/yacy/document/parser/html/Scraper.javaindex dc8e3e964..b483d5a8b 100644--- a/source/net/yacy/document/parser/html/Scraper.java+++ b/source/net/yacy/document/parser/html/Scraper.java@@ -24,17 +24,52 @@package net.yacy.document.parser.html;+import java.util.Properties;+public interface Scraper {+ /**+ * @param tag+ * a tag name+ * @return true when the tag name belongs to the first category of tags+ * according to the Scraper implementation, and is therefore candidate+ * for processing by+ * {@link #scrapeTag0(net.yacy.document.parser.html.ContentScraper.Tag)}+ * implementation+ */public boolean isTag0(String tag);+ /**+ * @param tag+ * a tag name+ * @return true when the tag name belongs to the second category of tags+ * according to the Scraper implementation, and is therefore candidate+ * for processing by+ * {@link #scrapeTag0(net.yacy.document.parser.html.ContentScraper.Tag)}+ * implementation+ */public boolean isTag1(String tag);public void scrapeText(char[] text, String insideTag);+ /**+ * Process a tag belonging to the first category of tags according to the Scraper implementation+ * @param tag a parsed tag+ */public void scrapeTag0(ContentScraper.Tag tag);+ /**+ * Process a tag belonging to the second category of tags according to the Scraper implementation+ * @param tag a parsed tag+ */public void scrapeTag1(ContentScraper.Tag tag);++ /**+ * Processing applied to any kind of tag opening.+ * @param tagName the tag name+ * @param tagAttributes the atttributes of the tag+ */+ public void scrapeAnyTagOpening(String tagName, Properties tagAttributes);public void scrapeComment(final char[] comment);diff --git a/source/net/yacy/document/parser/html/TransformerWriter.java b/source/net/yacy/document/parser/html/TransformerWriter.javaindex 202c6186d..eb246a997 100644--- a/source/net/yacy/document/parser/html/TransformerWriter.java+++ b/source/net/yacy/document/parser/html/TransformerWriter.java@@ -292,6 +292,10 @@ public final class TransformerWriter extends Writer {final CharBuffer charBuffer = new CharBuffer(ContentScraper.MAX_DOCSIZE, content);ContentScraper.Tag tag = new ContentScraper.Tag(tagname, charBuffer.propParser());charBuffer.close();++ /* Apply processing relevant for any kind of tag opening */+ this.scraper.scrapeAnyTagOpening(tag.name, tag.opts);+if (this.scraper != null && this.scraper.isTag0(tagname)) {// this single tag is collected at once herethis.scraper.scrapeTag0(tag);diff --git a/test/java/net/yacy/document/parser/html/ContentScraperTest.java b/test/java/net/yacy/document/parser/html/ContentScraperTest.javaindex d5ab17b1a..b430f58f3 100644--- a/test/java/net/yacy/document/parser/html/ContentScraperTest.java+++ b/test/java/net/yacy/document/parser/html/ContentScraperTest.java@@ -29,8 +29,11 @@ import java.util.ArrayList;import java.util.Calendar;import java.util.Collection;import java.util.Date;+import java.util.HashMap;import java.util.HashSet;import java.util.List;+import java.util.Map;+import java.util.Map.Entry;import java.util.Set;import org.junit.Assert;@@ -338,5 +341,80 @@ public class ContentScraperTest {Assert.assertEquals("{abc}{def}", ContentScraper.removeUnpairedBrackets("{abc}{def}", '{', '}'));Assert.assertEquals("{{abc}{def}}", ContentScraper.removeUnpairedBrackets("{{abc}{def}}", '{', '}'));}++ /**+ * Test microdata itemtype attribute parsing+ * @throws IOException+ */+ @Test+ public void testParseMicroDataItemType() throws IOException {+ final String htmlHeader = "<!DOCTYPE html><head><title>Test document</title></head>";+ final DigestURL docUrl = new DigestURL("http://example.org/microdata.html");+++ final Map<String, String[]> html2Results = new HashMap<>();+ /* Basic microdata syntax example with no item type */+ String html = htmlHeader + "<div itemscope><p>My name is <span itemprop=\"name\">Elizabeth</span>.</p></div>";+ String[] expectedUrls = {};+ html2Results.put(html, expectedUrls);++ /* Nested items with no item type */+ html = "<div itemscope>\n" ++ " <p>Name: <span itemprop=\"name\">Amanda</span></p>\n" ++ " <p>Band: <span itemprop=\"band\" itemscope> <span itemprop=\"name\">Jazz Band</span> (<span itemprop=\"size\">12</span> players)</span></p>\n" ++ "</div>";+ expectedUrls = new String[0];+ html2Results.put(html, expectedUrls);++ /* One typed item */+ html = htmlHeader + "<div itemscope itemtype=\"https://schema.org/LocalBusiness\"><img itemprop=\"logo\" src=\"our-logo.png\" alt=\"Our Company\"></div>";+ expectedUrls = new String[]{"https://schema.org/LocalBusiness"};+ html2Results.put(html, expectedUrls);++ /* more than one type per item */+ html = htmlHeader + "<dl itemscope itemtype=\"https://md.example.com/loco https://md.example.com/lighting\">" ++ " <dt>Name:\n" ++ " <dd itemprop=\"name\">Tank Locomotive (DB 80)\n" ++ " <dt>Product code:\n" ++ " <dd itemprop=\"product-code\">33041\n" ++ " <dt>Scale:\n" ++ " <dd itemprop=\"scale\">HO\n" ++ " <dt>Digital:\n" ++ " <dd itemprop=\"digital\">Delta\n" ++ "</dl>";+ expectedUrls = new String[]{"https://md.example.com/loco", "https://md.example.com/lighting"};+ html2Results.put(html, expectedUrls);++ /* Nested typed items */+ html = htmlHeader + "<div itemscope itemtype=\"http://schema.org/Product\">\n" ++ " <span itemprop=\"name\">Panasonic White 60L Refrigerator</span>\n" ++ " <img src=\"panasonic-fridge-60l-white.jpg\" alt=\"\">\n" ++ " <div itemprop=\"aggregateRating\"\n" ++ " itemscope itemtype=\"http://schema.org/AggregateRating\">\n" ++ " <meter itemprop=\"ratingValue\" min=0 value=3.5 max=5>Rated 3.5/5</meter>\n" ++ " (based on <span itemprop=\"reviewCount\">11</span> customer reviews)\n" ++ " </div>\n" ++ "</div>";+ expectedUrls = new String[]{"http://schema.org/Product", "http://schema.org/AggregateRating"};+ html2Results.put(html, expectedUrls);+++ for (final Entry<String, String[]> html2Result : html2Results.entrySet()) {+ ContentScraper scraper = new ContentScraper(docUrl, 10, new HashSet<String>(), new VocabularyScraper(), 0);+ try (final Writer writer = new TransformerWriter(null, null, scraper, null, false)) {+ FileUtils.copy(new StringReader(html2Result.getKey()), writer);++ final Set<DigestURL> expected = new HashSet<>();+ for (final String url : html2Result.getValue()) {+ expected.add(new DigestURL(url));+ }++ Assert.assertEquals(expected.size(), scraper.getLinkedDataTypes().size());+ Assert.assertTrue(expected.containsAll(scraper.getLinkedDataTypes()));+ } finally {+ scraper.close();+ }+ }+ }}