Distributed Peer-to-Peer Web Search Engine and Intranet Search Appliance
Java decentralizedintranet-searchintranet-search-enginep2pp2p-networkprivacyprivacy-protectionsearch-enginesearchenginewebweb-search-enginewebsearchwebseearch-engine
fixed parsing of x-data-elements in <div>
Michael Peter Christencb3366c3e95d
diff --git a/.project b/.projectindex 7bc830b00..5f27c43c7 100644--- a/.project+++ b/.project@@ -25,4 +25,15 @@<nature>org.eclipse.jdt.core.javanature</nature><nature>org.apache.ivyde.eclipse.ivynature</nature></natures>+ <filteredResources>+ <filter>+ <id>1771186756556</id>+ <name></name>+ <type>30</type>+ <matcher>+ <id>org.eclipse.core.resources.regexFilterMatcher</id>+ <arguments>node_modules|\.git|__CREATED_BY_JAVA_LANGUAGE_SERVER__</arguments>+ </matcher>+ </filter>+ </filteredResources></projectDescription>diff --git a/source/net/yacy/document/parser/html/TransformerWriter.java b/source/net/yacy/document/parser/html/TransformerWriter.javaindex 177d0584e..6477869ff 100644--- a/source/net/yacy/document/parser/html/TransformerWriter.java+++ b/source/net/yacy/document/parser/html/TransformerWriter.java@@ -395,27 +395,9 @@ public final class TransformerWriter extends Writer {if (this.inSingleQuote) {this.buffer.append(c);if (c == singlequote) this.inSingleQuote = false;- // check error cases- if ((c == rb) && (this.buffer.length() > 0 && this.buffer.charAt(0) == lb)) {- this.inSingleQuote = false;- // the tag ends here. after filtering: pass on- filtered = this.tokenProcessor(this.buffer.getChars(), singlequote);- if (this.out != null) { this.out.write(filtered); }- // this.buffer = new serverByteBuffer();- this.buffer.reset();- }} else if (this.inDoubleQuote) {this.buffer.append(c);if (c == doublequote) this.inDoubleQuote = false;- // check error cases- if (c == rb && this.buffer.length() > 0 && this.buffer.charAt(0) == lb) {- this.inDoubleQuote = false;- // the tag ends here. after filtering: pass on- filtered = this.tokenProcessor(this.buffer.getChars(), doublequote);- if (this.out != null) this.out.write(filtered);- // this.buffer = new serverByteBuffer();- this.buffer.reset();- }} else if (this.inComment) {this.buffer.append(c);if (c == rb &&@@ -554,4 +536,4 @@ public final class TransformerWriter extends Writer {return !this.binaryUnsuspect;}-}\ No newline at end of file+}diff --git a/test/java/net/yacy/document/parser/htmlParserTest.java b/test/java/net/yacy/document/parser/htmlParserTest.javaindex 760326ee1..5b87fdf88 100644--- a/test/java/net/yacy/document/parser/htmlParserTest.java+++ b/test/java/net/yacy/document/parser/htmlParserTest.java@@ -470,4 +470,82 @@ public class htmlParserTest {System.out.println("ScraperScriptTagTest: [" + textSource + "] = [" + txt + "]");assertEquals(txt, textSource);}++ /**+ * Test for parseToScraper of class htmlParser with quoted attribute content+ * containing ">" characters (for example Alpine.js x-data arrow functions).+ * Attribute values must never leak as plain text.+ */+ @Test+ public void testParseToScraper_QuotedAttributeWithGt() throws MalformedURLException, IOException {+ final AnchorURL url = new AnchorURL("http://localhost/");+ final String charset = StandardCharsets.UTF_8.name();+ final String textSource = "Visible text";+ final String testhtml = "<html><body>"+ + "<div class=\"relative flex items-start justify-between fluid-container-full\""+ + " x-data=\"{"+ + "mainMenuOpen: false,"+ + "isDesktop: false,"+ + "mobileMainNavHeight: 384,"+ + "init() {"+ + "this.checkIsDesktop();"+ + "},"+ + "checkIsDesktop() {"+ + "this.isDesktop = window.matchMedia('screen and (min-width:1024px)').matches;"+ + "},"+ + "closeMainMenu() {"+ + "this.mainMenuOpen = false;"+ + "},"+ + "openMainMenu() {"+ + "this.mainMenuOpen = true;"+ + "},"+ + "onSubMenuIntersect(el) {"+ + "this.$nextTick(() => {"+ + "this.mobileMainNavHeight = el.getBoundingClientRect().height;"+ + "});"+ + "}"+ + "}\">"+ + "<p>" + textSource + "</p>"+ + "</div>"+ + "</body></html>";++ final ContentScraper scraper = parseToScraper(url, charset, TagValency.IGNORE, new HashSet<String>(),+ new VocabularyScraper(), 0, testhtml, 10, 10);++ final String txt = scraper.getText();+ assertEquals(textSource + ".", txt);+ assertFalse(txt.contains("getBoundingClientRect"));+ assertFalse(txt.contains("mobileMainNavHeight"));+ }++ /**+ * Test for parseToScraper of class htmlParser with quoted attribute content+ * containing a comparison using ">" and additional Alpine-style attributes.+ * Attribute names/values must never leak as plain text.+ */+ @Test+ public void testParseToScraper_QuotedAttributeComparisonGt() throws MalformedURLException, IOException {+ final AnchorURL url = new AnchorURL("http://localhost/");+ final String charset = StandardCharsets.UTF_8.name();+ final String textSource = "Next image";+ final String testhtml = "<html><body>"+ + "<button "+ + "type=\"button\" "+ + "class=\"splide__arrow splide__arrow--prev absolute left-1 top-1/2 z-10 -translate-y-1/2 lg:left-2.5 [@media(pointer:coarse)]:hidden\" "+ + "x-show=\"imgModalItems.length > 1\" "+ + "@click=\"lightboxPrev()\""+ + ">"+ + textSource+ + "</button>"+ + "</body></html>";++ final ContentScraper scraper = parseToScraper(url, charset, TagValency.IGNORE, new HashSet<String>(),+ new VocabularyScraper(), 0, testhtml, 10, 10);++ final String txt = scraper.getText();+ assertTrue(txt.contains(textSource));+ assertFalse(txt.contains("@click"));+ assertFalse(txt.contains("lightboxPrev"));+ assertFalse(txt.contains("imgModalItems.length"));+ }}