Apache Nutch is an extensible and scalable web crawler
NUTCH-3110 Upgrade to Tika 3.2.3
lewismc713835b73bfd
diff --cc ivy/ivy.xmlindex 98713b0f0,70614f8fd..44914368e--- a/ivy/ivy.xml+++ b/ivy/ivy.xml@@@ -68,13 -69,12 +68,13 @@@<exclude org="ch.qos.reload4j" name="*"/><exclude org="org.slf4j" name="*" /></dependency>- <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-jobclient" rev="3.3.6" conf="*->default">+ <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-jobclient" rev="3.4.2" conf="*->default"><exclude org="ch.qos.reload4j" name="*"/><exclude org="org.slf4j" name="*" />- </dependency><!-- End of Hadoop Dependencies -->+ </dependency>+ <!-- End of Hadoop Dependencies -->- <dependency org="org.tallison.tika" name="tika-core-shaded" rev="2.9.1.0" conf="*->default" transitive="false"/>- <dependency org="org.tallison.tika" name="tika-core-shaded" rev="3.1.0.0" conf="*->default" transitive="false"/>++ <dependency org="org.apache.tika" name="tika-core" rev="3.2.3" conf="*->default"/><dependency org="xml-apis" name="xml-apis" rev="1.4.01" /><!-- force this version as it is required by Tika --><dependency org="xerces" name="xercesImpl" rev="2.12.2" />diff --cc src/plugin/language-identifier/ivy.xmlindex 60dbbefd1,149d6c147..448619ac1--- a/src/plugin/language-identifier/ivy.xml+++ b/src/plugin/language-identifier/ivy.xml@@@ -37,13 -37,7 +37,13 @@@</publications><dependencies>- <dependency org="org.apache.tika" name="tika-langdetect-optimaize" rev="2.9.0" conf="*->default">- <dependency org="org.tallison.tika" name="tika-langdetect-optimaize-shaded" rev="3.1.0.0" conf="*->default" transitive="false"/>++ <dependency org="org.apache.tika" name="tika-langdetect-optimaize" rev="3.2.3" conf="*->default">+ <!-- exclusions of dependencies provided in Nutch core (ivy/ivy.xml) -->+ <exclude org="org.apache.tika" name="tika-core" />+ <exclude org="com.google.guava" name="guava" />+ <exclude org="org.slf4j" name="slf4j-api" />+ <!-- exclusions of dependencies provided in Nutch core (ivy/ivy.xml) -->+ </dependency></dependencies></ivy-module>diff --cc src/plugin/language-identifier/plugin.xmlindex a65ca78cc,a4625d060..7fc3d555c--- a/src/plugin/language-identifier/plugin.xml+++ b/src/plugin/language-identifier/plugin.xml@@@ -26,16 -26,7 +26,15 @@@<export name="*"/></library><!-- dependencies of Tika's Optimaize language detector (tika-langdetect-optimaize) -->- <library name="annotations-12.0.jar"/>- <library name="checker-qual-3.33.0.jar"/>- <library name="error_prone_annotations-2.18.0.jar"/>- <library name="failureaccess-1.0.1.jar"/>- <library name="j2objc-annotations-2.8.jar"/>- <library name="tika-langdetect-optimaize-shaded-3.1.0.0.jar"/>++ <library name="annotations-26.0.2.jar"/>++ <library name="error_prone_annotations-2.41.0.jar"/>++ <library name="failureaccess-1.0.3.jar"/>++ <library name="j2objc-annotations-3.0.0.jar"/>+ <library name="jsonic-1.2.11.jar"/>- <library name="jsr305-3.0.2.jar"/>++ <library name="jspecify-1.0.0.jar"/>+ <library name="language-detector-0.6.jar"/>+ <library name="listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar"/>- <library name="tika-langdetect-optimaize-2.9.0.jar"/>++ <library name="tika-langdetect-optimaize-3.2.3.jar"/></runtime><requires>diff --cc src/plugin/parse-tika/howto_upgrade_tika.mdindex 8ed6c3f3c,8ed6c3f3c..3c81a607a--- a/src/plugin/parse-tika/howto_upgrade_tika.md+++ b/src/plugin/parse-tika/howto_upgrade_tika.md@@@ -15,31 -15,31 +15,17 @@@limitations under the License.-->--We are currently using a shim (https://github.com/tballison/hadoop-safe-tika--because of binary conflicts in commons-io versions between what Hadoop supports and the more--modern features that Apache Tika and Apache POI were using in commons-io.++# How to Upgrade Apache Tika in Nutch--For now, all you have to do is update the fat jar dependencies:++Nutch uses official Apache Tika artifacts for document parsing and language detection.--1. tika-core-shaded in ivy/ivy.xml----2. tika-parsers-standard-package-shaded in src/plugin/parse-tika/ivy.xml----3. The library name version for tika-parsers-standard-package-shaded in src/plugin/parse-tika/plugin.xml----4. Repeat steps 2 and 3 for the language-identifier----5. Build Nutch and run all unit tests:---- $ cd ../../../-- $ ant clean runtime test----The following directions are what we used to do with thin jars. Hopefully, we'll--be able to get back to these directions once we have version harmony with Hadoop and Tika/POI.++## Upgrade Steps1. Upgrade Tika dependency (tika-core) in ivy/ivy.xml--2. Upgrade Tika dependency in src/plugin/parse-tika/ivy.xml++2. Upgrade Tika dependency (tika-parsers-standard-package) in src/plugin/parse-tika/ivy.xml++ Note: The tika-handler-boilerpipe module provides Boilerpipe support for text extraction++ (tika.extractor=boilerpipe configuration). This was moved from tika-core in TIKA-4138.3. Upgrade Tika's own dependencies in src/plugin/parse-tika/plugin.xml@@@ -49,9 -49,9 +35,9 @@@$ ls lib | sed 's/^/ <library name="/g' | sed 's/$/"\/>/g'In the plugin.xml replace all lines between-- <!-- dependencies of Tika (tika-parsers) -->++ <!-- dependencies of Tika (tika-parsers-standard-package) -->and-- <!-- end of dependencies of Tika (tika-parsers) -->++ <!-- end of dependencies of Tika (tika-parsers-standard-package) -->with the output of the command above.4. (Optionally) remove overlapping dependencies between parse-tika and Nutch core dependencies:@@@ -61,7 -61,7 +47,7 @@@build/plugins/parse-tika/(eventually with different versions)- duplicated libs can be added to the exclusions of transitive dependencies in-- build/plugins/parse-tika/ivy.xml++ src/plugin/parse-tika/ivy.xml- but the library versions in ivy/ivy.xml MUST correspond to those required by Tika5. Remove the locally "installed" dependencies in src/plugin/parse-tika/lib/:@@@ -72,6 -72,6 +58,11 @@@$ cd ../language-identifier/++ Update tika-langdetect-optimaize in src/plugin/language-identifier/ivy.xml++ Then regenerate the library list in plugin.xml:++ $ ant -f ./build-ivy.xml++ $ ls lib | sed 's/^/ <library name="/g' | sed 's/$/"\/>/g'++7. Build Nutch and run all unit tests:$ cd ../../../diff --cc src/plugin/parse-tika/ivy.xmlindex 6d96ed3cd,f4aa049a9..23af88eec--- a/src/plugin/parse-tika/ivy.xml+++ b/src/plugin/parse-tika/ivy.xml@@@ -37,7 -37,7 +37,15 @@@</publications><dependencies>- <dependency org="org.tallison.tika" name="tika-parsers-standard-package-shaded" rev="2.9.1.0" conf="*->default" transitive="false"/>- <dependency org="org.tallison.tika" name="tika-parsers-standard-package-shaded" rev="3.1.0.0" conf="*->default" transitive="false"/>++ <dependency org="org.apache.tika" name="tika-parsers-standard-package" rev="3.2.3" conf="*->default">++ <!-- exclusions of dependencies provided in Nutch core (ivy/ivy.xml) -->++ <exclude org="org.apache.tika" name="tika-core"/>++ <exclude org="org.slf4j" name="slf4j-api"/>++ </dependency>++ <!-- Boilerpipe for text extraction (tika.extractor=boilerpipe) - TIKA-4138 -->++ <dependency org="org.apache.tika" name="tika-handler-boilerpipe" rev="3.2.3" conf="*->default">++ <exclude org="org.slf4j" name="slf4j-api"/>++ </dependency></dependencies></ivy-module>diff --cc src/plugin/parse-tika/plugin.xmlindex 04afb9fac,876174893..1b9252cc6--- a/src/plugin/parse-tika/plugin.xml+++ b/src/plugin/parse-tika/plugin.xml@@@ -25,7 -25,7 +25,97 @@@<library name="parse-tika.jar"><export name="*"/></library>- <library name="tika-parsers-standard-package-shaded-2.9.1.0.jar"/>- <library name="tika-parsers-standard-package-shaded-3.1.0.0.jar"/>++ <!-- dependencies of Tika (tika-parsers-standard-package) -->++ <library name="SparseBitSet-1.3.jar"/>++ <library name="angus-activation-2.0.2.jar"/>++ <library name="apache-mime4j-core-0.8.13.jar"/>++ <library name="apache-mime4j-dom-0.8.13.jar"/>++ <library name="asm-9.8.jar"/>++ <library name="bcjmail-jdk18on-1.81.jar"/>++ <library name="bcpkix-jdk18on-1.81.jar"/>++ <library name="bcprov-jdk18on-1.81.jar"/>++ <library name="bcutil-jdk18on-1.81.jar"/>++ <library name="boilerpipe-1.1.0.jar"/>++ <library name="commons-codec-1.19.0.jar"/>++ <library name="commons-collections4-4.5.0.jar"/>++ <library name="commons-compress-1.28.0.jar"/>++ <library name="commons-csv-1.14.1.jar"/>++ <library name="commons-exec-1.5.0.jar"/>++ <library name="commons-io-2.20.0.jar"/>++ <library name="commons-lang3-3.18.0.jar"/>++ <library name="commons-logging-1.3.5.jar"/>++ <library name="commons-math3-3.6.1.jar"/>++ <library name="curvesapi-1.08.jar"/>++ <library name="dd-plist-1.28.jar"/>++ <library name="dec-0.1.2.jar"/>++ <library name="fontbox-3.0.5.jar"/>++ <library name="istack-commons-runtime-4.1.2.jar"/>++ <library name="jackcess-4.0.8.jar"/>++ <library name="jackcess-encrypt-4.0.3.jar"/>++ <library name="jackson-annotations-2.20.jar"/>++ <library name="jackson-core-2.20.0.jar"/>++ <library name="jackson-databind-2.20.0.jar"/>++ <library name="jai-imageio-core-1.4.0.jar"/>++ <library name="jakarta.activation-api-2.1.3.jar"/>++ <library name="jakarta.xml.bind-api-4.0.2.jar"/>++ <library name="java-libpst-0.9.3.jar"/>++ <library name="jaxb-core-4.0.5.jar"/>++ <library name="jaxb-runtime-4.0.5.jar"/>++ <library name="jbig2-imageio-3.0.4.jar"/>++ <library name="jcl-over-slf4j-2.0.17.jar"/>++ <library name="jdom2-2.0.6.1.jar"/>++ <library name="jempbox-1.8.17.jar"/>++ <library name="jhighlight-1.1.0.jar"/>++ <library name="jmatio-1.5.jar"/>++ <library name="jsoup-1.21.2.jar"/>++ <library name="juniversalchardet-2.5.0.jar"/>++ <library name="junrar-7.5.5.jar"/>++ <library name="jwarc-0.32.0.jar"/>++ <library name="log4j-api-2.25.1.jar"/>++ <library name="metadata-extractor-2.19.0.jar"/>++ <library name="parso-2.0.14.jar"/>++ <library name="pdfbox-3.0.5.jar"/>++ <library name="pdfbox-io-3.0.5.jar"/>++ <library name="pdfbox-tools-3.0.5.jar"/>++ <library name="picocli-4.7.6.jar"/>++ <library name="poi-5.4.1.jar"/>++ <library name="poi-ooxml-5.4.1.jar"/>++ <library name="poi-ooxml-lite-5.4.1.jar"/>++ <library name="poi-scratchpad-5.4.1.jar"/>++ <library name="rome-2.1.0.jar"/>++ <library name="rome-utils-2.1.0.jar"/>++ <library name="tika-parser-apple-module-3.2.3.jar"/>++ <library name="tika-parser-audiovideo-module-3.2.3.jar"/>++ <library name="tika-parser-cad-module-3.2.3.jar"/>++ <library name="tika-parser-code-module-3.2.3.jar"/>++ <library name="tika-parser-crypto-module-3.2.3.jar"/>++ <library name="tika-parser-digest-commons-3.2.3.jar"/>++ <library name="tika-parser-font-module-3.2.3.jar"/>++ <library name="tika-parser-html-module-3.2.3.jar"/>++ <library name="tika-parser-image-module-3.2.3.jar"/>++ <library name="tika-parser-mail-commons-3.2.3.jar"/>++ <library name="tika-parser-mail-module-3.2.3.jar"/>++ <library name="tika-parser-microsoft-module-3.2.3.jar"/>++ <library name="tika-parser-miscoffice-module-3.2.3.jar"/>++ <library name="tika-parser-news-module-3.2.3.jar"/>++ <library name="tika-parser-ocr-module-3.2.3.jar"/>++ <library name="tika-parser-pdf-module-3.2.3.jar"/>++ <library name="tika-parser-pkg-module-3.2.3.jar"/>++ <library name="tika-parser-text-module-3.2.3.jar"/>++ <library name="tika-parser-webarchive-module-3.2.3.jar"/>++ <library name="tika-parser-xml-module-3.2.3.jar"/>++ <library name="tika-parser-xmp-commons-3.2.3.jar"/>++ <library name="tika-parser-zip-commons-3.2.3.jar"/>++ <library name="tika-parsers-standard-package-3.2.3.jar"/>++ <library name="tika-handler-boilerpipe-3.2.3.jar"/>++ <library name="txw2-4.0.5.jar"/>++ <library name="vorbis-java-core-0.8.jar"/>++ <library name="vorbis-java-tika-0.8.jar"/>++ <library name="xmlbeans-5.3.0.jar"/>++ <library name="xmpbox-3.0.5.jar"/>++ <library name="xmpcore-6.1.11.jar"/>++ <library name="xz-1.10.jar"/>++ <!-- end of dependencies of Tika (tika-parsers-standard-package) --></runtime><requires>diff --cc src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/DOMContentUtils.javaindex 81ef6c523,81ef6c523..048afd711--- a/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/DOMContentUtils.java+++ b/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/DOMContentUtils.java@@@ -22,6 -22,6 +22,7 @@@ import java.util.ArrayListimport java.util.Collection;import java.util.HashMap;import java.util.HashSet;++import java.util.LinkedHashMap;import java.util.List;import java.util.Set;import java.util.regex.Pattern;@@@ -394,6 -394,6 +395,8 @@@ public class DOMContentUtils* @param node a {@link Node} under which to discover anchors*/public void getOutlinks(URL base, ArrayList<Outlink> outlinks, Node node) {++ // Use a LinkedHashMap to deduplicate outlinks by URL while preserving insertion order++ LinkedHashMap<String, Outlink> outlinkMap = new LinkedHashMap<>();NodeWalker walker = new NodeWalker(node);while (walker.hasNext()) {@@@ -435,16 -435,16 +438,30 @@@try {URL url = URLUtil.resolveURL(base, target);-- Outlink outlink = new Outlink(url.toString(), linkText-- .toString().trim());-- outlinks.add(outlink);---- // NUTCH-2433 - Keep the node name where the URL was found into-- // the outlink metadata-- if (keepNodenames) {-- MapWritable metadata = new MapWritable();-- metadata.put(new Text(srcTagMetaName), new Text(nodeName));-- outlink.setMetadata(metadata);++ String urlStr = url.toString();++ String anchor = linkText.toString().trim();++++ // Only add if URL not seen, or replace if current has better anchor++ Outlink existing = outlinkMap.get(urlStr);++ if (existing == null) {++ Outlink outlink = new Outlink(urlStr, anchor);++ // NUTCH-2433 - Keep the node name where the URL was found into++ // the outlink metadata++ if (keepNodenames) {++ MapWritable metadata = new MapWritable();++ metadata.put(new Text(srcTagMetaName), new Text(nodeName));++ outlink.setMetadata(metadata);++ }++ outlinkMap.put(urlStr, outlink);++ } else if (existing.getAnchor().isEmpty() && !anchor.isEmpty()) {++ // Replace empty anchor with non-empty one++ Outlink outlink = new Outlink(urlStr, anchor);++ if (keepNodenames) {++ MapWritable metadata = new MapWritable();++ metadata.put(new Text(srcTagMetaName), new Text(nodeName));++ outlink.setMetadata(metadata);++ }++ outlinkMap.put(urlStr, outlink);}} catch (MalformedURLException e) {// don't care@@@ -456,6 -456,6 +473,8 @@@}}}++++ outlinks.addAll(outlinkMap.values());}// This one is used by NUTCH-1918@@@ -464,6 -464,6 +483,9 @@@String target = null;String anchor = null;boolean noFollow = false;++++ // Use a LinkedHashMap to deduplicate outlinks by URL while preserving insertion order++ LinkedHashMap<String, Outlink> outlinkMap = new LinkedHashMap<>();for (Link link : tikaExtractedOutlinks) {target = link.getUri();@@@ -475,17 -475,17 +497,27 @@@if (target != null && !noFollow) {try {URL url = URLUtil.resolveURL(base, target);++ String urlStr = url.toString();// clean the anchoranchor = anchor.replaceAll("\\s+", " ");anchor = anchor.trim();-- outlinks.add(new Outlink(url.toString(), anchor));++ // Only add if URL not seen, or replace if current has better anchor++ Outlink existing = outlinkMap.get(urlStr);++ if (existing == null) {++ outlinkMap.put(urlStr, new Outlink(urlStr, anchor));++ } else if (existing.getAnchor().isEmpty() && !anchor.isEmpty()) {++ // Replace empty anchor with non-empty one++ outlinkMap.put(urlStr, new Outlink(urlStr, anchor));++ }} catch (MalformedURLException e) {// don't care}}}}++++ outlinks.addAll(outlinkMap.values());}}diff --cc src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaParser.javaindex e75f70aef,85cd1fae0..2c110309e--- a/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaParser.java+++ b/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaParser.java@@@ -22,14 -22,14 +22,16 @@@ import java.net.MalformedURLExceptionimport java.net.URL;import java.util.ArrayList;import java.util.Arrays;--import java.util.HashSet;import java.util.List;import java.util.Map;--import java.util.Set;import org.apache.commons.lang3.StringUtils;import org.apache.hadoop.conf.Configuration;import org.apache.html.dom.HTMLDocumentImpl;++++import de.l3s.boilerpipe.BoilerpipeExtractor;++import org.apache.tika.sax.boilerpipe.BoilerpipeContentHandler;++import org.apache.nutch.metadata.Nutch;import org.apache.nutch.parse.HTMLMetaTags;import org.apache.nutch.parse.HtmlParseFilters;@@@ -45,7 -45,7 +47,6 @@@ import org.apache.tika.config.TikaConfiimport org.apache.tika.metadata.Metadata;import org.apache.tika.metadata.TikaCoreProperties;import org.apache.tika.mime.MediaType;--import org.apache.tika.sax.boilerpipe.BoilerpipeContentHandler;import org.apache.tika.parser.AutoDetectParser;import org.apache.tika.parser.CompositeParser;import org.apache.tika.parser.ParseContext;@@@ -77,9 -77,9 +78,11 @@@ public class TikaParser implements org.private HtmlMapper HTMLMapper;private boolean parseEmbedded = true;private boolean upperCaseElementNames = true;++++ // Boilerpipe configurationprivate boolean useBoilerpipe;private String boilerpipeExtractorName;-- private Set<String> boilerpipeMimeTypes;++ private String[] boilerpipeMimeTypes;@Overridepublic ParseResult getParse(Content content) {@@@ -118,21 -118,21 +121,10 @@@byte[] raw = content.getContent();Metadata tikamd = new Metadata();-- ContentHandler domHandler;---- // Check whether to use Tika's BoilerplateContentHandler-- if (useBoilerpipe && boilerpipeMimeTypes.contains(mimeType)) {-- BoilerpipeContentHandler bpHandler = new BoilerpipeContentHandler(-- (ContentHandler) new DOMBuilder(doc, root),-- BoilerpipeExtractorRepository.getExtractor(boilerpipeExtractorName));-- bpHandler.setIncludeMarkup(true);-- domHandler = (ContentHandler) bpHandler;-- } else {-- DOMBuilder domBuilder = new DOMBuilder(doc, root);-- domBuilder.setUpperCaseElementNames(upperCaseElementNames);-- domBuilder.setDefaultNamespaceURI(XHTMLContentHandler.XHTML);-- domHandler = (ContentHandler) domBuilder;-- }++ DOMBuilder domBuilder = new DOMBuilder(doc, root);++ domBuilder.setUpperCaseElementNames(upperCaseElementNames);++ domBuilder.setDefaultNamespaceURI(XHTMLContentHandler.XHTML);++ ContentHandler domHandler = (ContentHandler) domBuilder;LinkContentHandler linkContentHandler = new LinkContentHandler();@@@ -141,17 -141,17 +133,28 @@@context.set(Parser.class, new AutoDetectParser(tikaConfig));}-- TeeContentHandler teeContentHandler = new TeeContentHandler(domHandler,-- linkContentHandler);++ ContentHandler contentHandler;++ // Check if boilerpipe should be used for this content++ if (useBoilerpipe && boilerpipeExtractorName != null++ && Arrays.asList(boilerpipeMimeTypes).contains(mimeType)) {++ BoilerpipeExtractor extractor = BoilerpipeExtractorRepository++ .getExtractor(boilerpipeExtractorName);++ BoilerpipeContentHandler boilerpipeHandler = new BoilerpipeContentHandler(++ domHandler, extractor);++ contentHandler = new TeeContentHandler(boilerpipeHandler,++ linkContentHandler);++ } else {++ contentHandler = new TeeContentHandler(domHandler, linkContentHandler);++ }if (HTMLMapper != null)context.set(HtmlMapper.class, HTMLMapper);tikamd.set(Metadata.CONTENT_TYPE, mimeType);try {-- parser.parse(new ByteArrayInputStream(raw),-- (ContentHandler) teeContentHandler, tikamd, context);++ parser.parse(new ByteArrayInputStream(raw), contentHandler, tikamd,++ context);} catch (Exception e) {- LOG.error("Error parsing " + content.getUrl(), e);+ LOG.error("Error parsing {}", content.getUrl(), e);return new ParseStatus(ParseStatus.FAILED, e.getMessage()).getEmptyParseResult(content.getUrl(), getConf());}@@@ -317,13 -317,13 +320,14 @@@Nutch.CACHING_FORBIDDEN_CONTENT);upperCaseElementNames = conf.getBoolean("tika.uppercase.element.names",true);++ parseEmbedded = conf.getBoolean("tika.parse.embedded", true);++++ // Boilerpipe configurationuseBoilerpipe = conf.get("tika.extractor", "none").equals("boilerpipe");boilerpipeExtractorName = conf.get("tika.extractor.boilerpipe.algorithm","ArticleExtractor");-- boilerpipeMimeTypes = new HashSet<>(Arrays-- .asList(conf.getTrimmedStrings("tika.extractor.boilerpipe.mime.types",-- "text/html", "application/xhtml+xml")));-- parseEmbedded = conf.getBoolean("tika.parse.embedded", true);++ boilerpipeMimeTypes = conf.getStrings("tika.extractor.boilerpipe.mime.types",++ "text/html", "application/xhtml+xml");}@Overridediff --cc src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestBoilerpipeExtraction.javaindex 000000000,000000000..f5b480e3fnew file mode 100644--- /dev/null+++ b/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestBoilerpipeExtraction.java@@@ -1,0 -1,0 +1,112 @@@++/*++ * Licensed to the Apache Software Foundation (ASF) under one or more++ * contributor license agreements. See the NOTICE file distributed with++ * this work for additional information regarding copyright ownership.++ * The ASF licenses this file to You under the Apache License, Version 2.0++ * (the "License"); you may not use this file except in compliance with++ * the License. You may obtain a copy of the License at++ *++ * http://www.apache.org/licenses/LICENSE-2.0++ *++ * Unless required by applicable law or agreed to in writing, software++ * distributed under the License is distributed on an "AS IS" BASIS,++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.++ * See the License for the specific language governing permissions and++ * limitations under the License.++ */++package org.apache.nutch.parse.tika;++++import org.apache.hadoop.conf.Configuration;++import org.apache.nutch.metadata.Metadata;++import org.apache.nutch.parse.Parse;++import org.apache.nutch.protocol.Content;++import org.apache.nutch.util.NutchConfiguration;++import org.junit.jupiter.api.Test;++++import static org.junit.jupiter.api.Assertions.*;++++/**++ * Tests for Boilerpipe content extraction integration with Tika.++ * These tests help ensure boilerpipe support works correctly across Tika upgrades.++ */++public class TestBoilerpipeExtraction {++++ private static final String HTML_WITH_BOILERPLATE =++ "<!DOCTYPE html><html><head><title>Article Title</title></head><body>"++ + "<div id='header'>Navigation | Menu | Links</div>"++ + "<div id='content'>"++ + "<h1>Main Article Heading</h1>"++ + "<p>This is the main article content that should be extracted. "++ + "It contains important information about the topic.</p>"++ + "<p>Another paragraph with meaningful content that adds value.</p>"++ + "</div>"++ + "<div id='footer'>Copyright 2024 | Terms | Privacy</div>"++ + "</body></html>";++++ @Test++ public void testBoilerpipeExtraction() {++ Configuration conf = NutchConfiguration.create();++ conf.set("plugin.includes", "parse-tika");++ conf.set("tika.extractor", "boilerpipe");++ conf.set("tika.extractor.boilerpipe.algorithm", "ArticleExtractor");++++ TikaParser parser = new TikaParser();++ parser.setConf(conf);++++ String url = "http://example.com/article.html";++ Content content = new Content(url, url,++ HTML_WITH_BOILERPLATE.getBytes(), "text/html", new Metadata(), conf);++++ Parse parse = parser.getParse(content).get(url);++ String text = parse.getText();++++ // Boilerpipe should extract main content++ assertTrue(text.contains("Main Article Heading"),++ "Should contain article heading");++ assertTrue(text.contains("main article content"),++ "Should contain main content");++ }++++ @Test++ public void testBoilerpipeDisabled() {++ Configuration conf = NutchConfiguration.create();++ conf.set("plugin.includes", "parse-tika");++ conf.set("tika.extractor", "none"); // Default - boilerpipe disabled++++ TikaParser parser = new TikaParser();++ parser.setConf(conf);++++ String url = "http://example.com/article.html";++ Content content = new Content(url, url,++ HTML_WITH_BOILERPLATE.getBytes(), "text/html", new Metadata(), conf);++++ Parse parse = parser.getParse(content).get(url);++ String text = parse.getText();++++ // Without boilerpipe, all text should be extracted including boilerplate++ assertTrue(text.contains("Navigation"), "Should contain navigation text");++ assertTrue(text.contains("Copyright"), "Should contain footer text");++ assertTrue(text.contains("Main Article Heading"), "Should contain main content");++ }++++ @Test++ public void testBoilerpipeWithNonHtmlMimeType() {++ Configuration conf = NutchConfiguration.create();++ conf.set("plugin.includes", "parse-tika");++ conf.set("tika.extractor", "boilerpipe");++ conf.set("tika.extractor.boilerpipe.algorithm", "ArticleExtractor");++ // Only apply to HTML mime types++ conf.setStrings("tika.extractor.boilerpipe.mime.types", "text/html");++++ TikaParser parser = new TikaParser();++ parser.setConf(conf);++++ // Using XHTML mime type which is not in the configured list++ String url = "http://example.com/article.xhtml";++ Content content = new Content(url, url,++ HTML_WITH_BOILERPLATE.getBytes(), "application/xhtml+xml", new Metadata(), conf);++++ Parse parse = parser.getParse(content).get(url);++ assertNotNull(parse, "Should parse successfully even when boilerpipe doesn't apply");++ }++}diff --cc src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestEncodingDetection.javaindex 000000000,000000000..347d76a13new file mode 100644--- /dev/null+++ b/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestEncodingDetection.java@@@ -1,0 -1,0 +1,193 @@@++/*++ * Licensed to the Apache Software Foundation (ASF) under one or more++ * contributor license agreements. See the NOTICE file distributed with++ * this work for additional information regarding copyright ownership.++ * The ASF licenses this file to You under the Apache License, Version 2.0++ * (the "License"); you may not use this file except in compliance with++ * the License. You may obtain a copy of the License at++ *++ * http://www.apache.org/licenses/LICENSE-2.0++ *++ * Unless required by applicable law or agreed to in writing, software++ * distributed under the License is distributed on an "AS IS" BASIS,++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.++ * See the License for the specific language governing permissions and++ * limitations under the License.++ */++package org.apache.nutch.parse.tika;++++import org.apache.hadoop.conf.Configuration;++import org.apache.nutch.metadata.Metadata;++import org.apache.nutch.parse.Parse;++import org.apache.nutch.protocol.Content;++import org.apache.nutch.util.NutchConfiguration;++import org.junit.jupiter.api.BeforeEach;++import org.junit.jupiter.api.Test;++++import java.nio.charset.Charset;++import java.nio.charset.StandardCharsets;++++import static org.junit.jupiter.api.Assertions.*;++++/**++ * Tests for encoding/charset detection.++ * These tests help ensure charset detection works correctly across Tika upgrades.++ */++public class TestEncodingDetection {++++ private Configuration conf;++ private TikaParser parser;++++ @BeforeEach++ public void setup() {++ conf = NutchConfiguration.create();++ conf.set("plugin.includes", "parse-tika");++ parser = new TikaParser();++ parser.setConf(conf);++ }++++ @Test++ public void testUtf8WithMetaCharset() {++ String html = "<!DOCTYPE html><html><head>"++ + "<meta charset=\"utf-8\">"++ + "<title>UTF-8 Test</title></head>"++ + "<body>äöü ñ 中文</body></html>";++++ byte[] bytes = html.getBytes(StandardCharsets.UTF_8);++ Parse parse = doParse(bytes);++++ assertEquals("UTF-8 Test", parse.getData().getTitle());++ assertTrue(parse.getText().contains("äöü"));++ assertTrue(parse.getText().contains("中文"));++ }++++ @Test++ public void testUtf8WithHttpEquiv() {++ String html = "<!DOCTYPE html><html><head>"++ + "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">"++ + "<title>UTF-8 HTTP-Equiv</title></head>"++ + "<body>Special chars: äöü</body></html>";++++ byte[] bytes = html.getBytes(StandardCharsets.UTF_8);++ Parse parse = doParse(bytes);++++ assertEquals("UTF-8 HTTP-Equiv", parse.getData().getTitle());++ assertTrue(parse.getText().contains("äöü"));++ }++++ @Test++ public void testUtf8BOM() {++ String html = "<!DOCTYPE html><html><head>"++ + "<title>BOM Test</title></head>"++ + "<body>Content with BOM</body></html>";++++ // UTF-8 BOM++ byte[] bom = new byte[]{(byte)0xEF, (byte)0xBB, (byte)0xBF};++ byte[] htmlBytes = html.getBytes(StandardCharsets.UTF_8);++ byte[] withBom = new byte[bom.length + htmlBytes.length];++ System.arraycopy(bom, 0, withBom, 0, bom.length);++ System.arraycopy(htmlBytes, 0, withBom, bom.length, htmlBytes.length);++++ Parse parse = doParse(withBom);++++ assertEquals("BOM Test", parse.getData().getTitle());++ assertTrue(parse.getText().contains("Content with BOM"));++ }++++ @Test++ public void testIso88591() {++ String html = "<!DOCTYPE html><html><head>"++ + "<meta charset=\"iso-8859-1\">"++ + "<title>ISO-8859-1 Test</title></head>"++ + "<body>German: \u00e4\u00f6\u00fc</body></html>";++++ try {++ Charset charset = Charset.forName("ISO-8859-1");++ byte[] bytes = html.getBytes(charset);++ Parse parse = doParse(bytes);++++ assertEquals("ISO-8859-1 Test", parse.getData().getTitle());++ } catch (Exception e) {++ // ISO-8859-1 might not be available on all systems++ // Skip test in that case++ }++ }++++ @Test++ public void testUtf16BE() {++ String html = "<!DOCTYPE html><html><head>"++ + "<title>UTF-16BE Test</title></head>"++ + "<body>Content</body></html>";++++ // UTF-16BE doesn't add BOM automatically++ byte[] bytes = html.getBytes(StandardCharsets.UTF_16BE);++++ // Add BOM manually for UTF-16BE++ byte[] bom = new byte[]{(byte)0xFE, (byte)0xFF};++ byte[] withBom = new byte[bom.length + bytes.length];++ System.arraycopy(bom, 0, withBom, 0, bom.length);++ System.arraycopy(bytes, 0, withBom, bom.length, bytes.length);++++ Parse parse = doParse(withBom);++++ assertNotNull(parse.getData().getTitle());++ }++++ @Test++ public void testUtf16LE() {++ String html = "<!DOCTYPE html><html><head>"++ + "<title>UTF-16LE Test</title></head>"++ + "<body>Content</body></html>";++++ // UTF-16LE doesn't add BOM automatically++ byte[] bytes = html.getBytes(StandardCharsets.UTF_16LE);++++ // Add BOM manually for UTF-16LE++ byte[] bom = new byte[]{(byte)0xFF, (byte)0xFE};++ byte[] withBom = new byte[bom.length + bytes.length];++ System.arraycopy(bom, 0, withBom, 0, bom.length);++ System.arraycopy(bytes, 0, withBom, bom.length, bytes.length);++++ Parse parse = doParse(withBom);++++ assertNotNull(parse.getData().getTitle());++ }++++ @Test++ public void testNoEncodingDeclaration() {++ // HTML without any encoding declaration - should default to something reasonable++ String html = "<!DOCTYPE html><html><head>"++ + "<title>No Encoding</title></head>"++ + "<body>Plain ASCII content</body></html>";++++ byte[] bytes = html.getBytes(StandardCharsets.US_ASCII);++ Parse parse = doParse(bytes);++++ assertEquals("No Encoding", parse.getData().getTitle());++ assertTrue(parse.getText().contains("Plain ASCII content"));++ }++++ @Test++ public void testXmlDeclarationEncoding() {++ String html = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"++ + "<!DOCTYPE html><html xmlns=\"http://www.w3.org/1999/xhtml\">"++ + "<head><title>XHTML Test</title></head>"++ + "<body>Special: äöü</body></html>";++++ byte[] bytes = html.getBytes(StandardCharsets.UTF_8);++++ String url = "http://example.com/";++ Content content = new Content(url, url, bytes,++ "application/xhtml+xml", new Metadata(), conf);++ Parse parse = parser.getParse(content).get(url);++++ assertNotNull(parse);++ // Title extraction may vary for XHTML++ }++++ private Parse doParse(byte[] bytes) {++ String url = "http://example.com/";++ Content content = new Content(url, url, bytes, "text/html", new Metadata(), conf);++ return parser.getParse(content).get(url);++ }++}diff --cc src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestHtmlParser.javaindex 17d244447,781e8910a..0460cce70--- a/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestHtmlParser.java+++ b/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestHtmlParser.java@@@ -76,8 -75,8 +76,10 @@@ public class TestHtmlParser+ encodingTestContent },{ "HTML5, utf-8, BOM", "utf-8","\ufeff<!DOCTYPE html>\n<html>\n<head>\n" + encodingTestContent },++ // Note: UTF-16 encoder adds BOM automatically, so don't include explicit BOM++ // to avoid double BOM (FE FF FE FF) which confuses parsers{ "HTML5, utf-16, BOM", "utf-16",-- "\ufeff<!DOCTYPE html>\n<html>\n<head>\n" + encodingTestContent } };++ "<!DOCTYPE html>\n<html>\n<head>\n" + encodingTestContent } };private static final String resolveBaseUrlTestContent = //"<html>\n<head>\n" + //diff --cc src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestLinkExtractionEdgeCases.javaindex 000000000,000000000..fb3a04d2enew file mode 100644--- /dev/null+++ b/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestLinkExtractionEdgeCases.java@@@ -1,0 -1,0 +1,221 @@@++/*++ * Licensed to the Apache Software Foundation (ASF) under one or more++ * contributor license agreements. See the NOTICE file distributed with++ * this work for additional information regarding copyright ownership.++ * The ASF licenses this file to You under the Apache License, Version 2.0++ * (the "License"); you may not use this file except in compliance with++ * the License. You may obtain a copy of the License at++ *++ * http://www.apache.org/licenses/LICENSE-2.0++ *++ * Unless required by applicable law or agreed to in writing, software++ * distributed under the License is distributed on an "AS IS" BASIS,++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.++ * See the License for the specific language governing permissions and++ * limitations under the License.++ */++package org.apache.nutch.parse.tika;++++import org.apache.hadoop.conf.Configuration;++import org.apache.nutch.metadata.Metadata;++import org.apache.nutch.parse.Outlink;++import org.apache.nutch.parse.Parse;++import org.apache.nutch.protocol.Content;++import org.apache.nutch.util.NutchConfiguration;++import org.junit.jupiter.api.BeforeEach;++import org.junit.jupiter.api.Test;++++import java.util.Arrays;++import java.util.Set;++import java.util.stream.Collectors;++++import static org.junit.jupiter.api.Assertions.*;++++/**++ * Tests for link extraction edge cases.++ * These tests help catch behavior changes in Tika's link extraction across versions.++ */++public class TestLinkExtractionEdgeCases {++++ private Configuration conf;++ private TikaParser parser;++++ @BeforeEach++ public void setup() {++ conf = NutchConfiguration.create();++ conf.set("plugin.includes", "parse-tika");++ parser = new TikaParser();++ parser.setConf(conf);++ }++++ @Test++ public void testDuplicateLinkDeduplication() {++ // Same URL appearing multiple times should be deduplicated++ String html = "<html><body>"++ + "<a href='http://example.com/page'>Link 1</a>"++ + "<a href='http://example.com/page'>Link 2</a>"++ + "<a href='http://example.com/page'>Link 3</a>"++ + "</body></html>";++++ Outlink[] outlinks = parseOutlinks(html);++++ // Should deduplicate to single link++ assertEquals(1, outlinks.length, "Duplicate URLs should be deduplicated");++ assertEquals("http://example.com/page", outlinks[0].getToUrl());++ }++++ @Test++ public void testLinkOrderPreserved() {++ String html = "<html><body>"++ + "<a href='http://example.com/first'>First</a>"++ + "<a href='http://example.com/second'>Second</a>"++ + "<a href='http://example.com/third'>Third</a>"++ + "</body></html>";++++ Outlink[] outlinks = parseOutlinks(html);++++ assertEquals(3, outlinks.length, "Should extract all 3 links");++ assertEquals("http://example.com/first", outlinks[0].getToUrl(),++ "First link should be first");++ assertEquals("http://example.com/second", outlinks[1].getToUrl(),++ "Second link should be second");++ assertEquals("http://example.com/third", outlinks[2].getToUrl(),++ "Third link should be third");++ }++++ @Test++ public void testNestedAnchors() {++ // Malformed HTML with nested anchors++ String html = "<html><body>"++ + "<a href='/outer'><a href='/inner'>Nested</a></a>"++ + "</body></html>";++++ Outlink[] outlinks = parseOutlinks(html);++++ // At minimum, inner link should be extracted++ Set<String> urls = Arrays.stream(outlinks)++ .map(Outlink::getToUrl)++ .collect(Collectors.toSet());++ assertTrue(urls.contains("http://example.com/inner"),++ "Inner link should be extracted");++ }++++ @Test++ public void testRelativeLinks() {++ String html = "<html><body>"++ + "<a href='page.html'>Relative</a>"++ + "<a href='./page.html'>Dot Relative</a>"++ + "<a href='../parent.html'>Parent</a>"++ + "<a href='/absolute.html'>Absolute Path</a>"++ + "</body></html>";++++ Outlink[] outlinks = parseOutlinks(html, "http://example.com/dir/index.html");++++ Set<String> urls = Arrays.stream(outlinks)++ .map(Outlink::getToUrl)++ .collect(Collectors.toSet());++++ assertTrue(urls.contains("http://example.com/dir/page.html") ||++ urls.contains("http://example.com/page.html"),++ "Should resolve relative link");++ assertTrue(urls.contains("http://example.com/absolute.html"),++ "Should resolve absolute path");++ }++++ @Test++ public void testAnchorTextExtraction() {++ String html = "<html><body>"++ + "<a href='http://example.com/page'> Anchor Text </a>"++ + "</body></html>";++++ Outlink[] outlinks = parseOutlinks(html);++++ assertEquals(1, outlinks.length);++ // Anchor text should be trimmed and whitespace normalized++ assertEquals("Anchor Text", outlinks[0].getAnchor().trim().replaceAll("\\s+", " "));++ }++++ @Test++ public void testEmptyAnchorText() {++ String html = "<html><body>"++ + "<a href='http://example.com/page'></a>"++ + "<a href='http://example.com/page2'> </a>"++ + "</body></html>";++++ Outlink[] outlinks = parseOutlinks(html);++++ // Links should still be extracted even with empty anchor text++ assertTrue(outlinks.length >= 1, "Should extract links with empty anchors");++ }++++ @Test++ public void testImageLinks() {++ String html = "<html><body>"++ + "<a href='http://example.com/page'><img src='image.jpg' alt='Image Alt'></a>"++ + "</body></html>";++++ Outlink[] outlinks = parseOutlinks(html);++++ // Should extract the anchor link (Tika may also extract image src as separate link)++ Set<String> urls = Arrays.stream(outlinks)++ .map(Outlink::getToUrl)++ .collect(Collectors.toSet());++ assertTrue(urls.contains("http://example.com/page"),++ "Should extract the anchor link");++ }++++ @Test++ public void testFormActionNotExtracted() {++ conf.setBoolean("parser.html.form.use_action", false);++ parser.setConf(conf);++++ String html = "<html><body>"++ + "<form action='http://example.com/submit' method='post'>"++ + "<input type='submit'/>"++ + "</form>"++ + "<a href='http://example.com/link'>Real Link</a>"++ + "</body></html>";++++ Outlink[] outlinks = parseOutlinks(html);++++ // Form action should not be extracted as outlink++ Set<String> urls = Arrays.stream(outlinks)++ .map(Outlink::getToUrl)++ .collect(Collectors.toSet());++ assertFalse(urls.contains("http://example.com/submit"),++ "Form action should not be extracted when disabled");++ assertTrue(urls.contains("http://example.com/link"),++ "Regular links should still be extracted");++ }++++ @Test++ public void testFragmentLinks() {++ String html = "<html><body>"++ + "<a href='#section'>Fragment Only</a>"++ + "<a href='page.html#section'>Page with Fragment</a>"++ + "</body></html>";++++ Outlink[] outlinks = parseOutlinks(html);++++ // Fragment-only links typically not extracted, page links should be++ Set<String> urls = Arrays.stream(outlinks)++ .map(Outlink::getToUrl)++ .collect(Collectors.toSet());++++ // The page link should be extracted (with or without fragment)++ boolean hasPageLink = urls.stream()++ .anyMatch(u -> u.contains("page.html"));++ assertTrue(hasPageLink, "Page links should be extracted");++ }++++ private Outlink[] parseOutlinks(String html) {++ return parseOutlinks(html, "http://example.com/");++ }++++ private Outlink[] parseOutlinks(String html, String baseUrl) {++ Content content = new Content(baseUrl, baseUrl, html.getBytes(),++ "text/html", new Metadata(), conf);++ Parse parse = parser.getParse(content).get(baseUrl);++ return parse.getData().getOutlinks();++ }++}diff --cc src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestMetadataExtraction.javaindex 000000000,000000000..04c0ace6cnew file mode 100644--- /dev/null+++ b/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestMetadataExtraction.java@@@ -1,0 -1,0 +1,223 @@@++/*++ * Licensed to the Apache Software Foundation (ASF) under one or more++ * contributor license agreements. See the NOTICE file distributed with++ * this work for additional information regarding copyright ownership.++ * The ASF licenses this file to You under the Apache License, Version 2.0++ * (the "License"); you may not use this file except in compliance with++ * the License. You may obtain a copy of the License at++ *++ * http://www.apache.org/licenses/LICENSE-2.0++ *++ * Unless required by applicable law or agreed to in writing, software++ * distributed under the License is distributed on an "AS IS" BASIS,++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.++ * See the License for the specific language governing permissions and++ * limitations under the License.++ */++package org.apache.nutch.parse.tika;++++import org.apache.hadoop.conf.Configuration;++import org.apache.nutch.metadata.Metadata;++import org.apache.nutch.parse.Parse;++import org.apache.nutch.protocol.Content;++import org.apache.nutch.util.NutchConfiguration;++import org.junit.jupiter.api.BeforeEach;++import org.junit.jupiter.api.Test;++++import static org.junit.jupiter.api.Assertions.*;++++/**++ * Tests for metadata extraction from HTML documents.++ * These tests help ensure metadata extraction works correctly across Tika upgrades.++ */++public class TestMetadataExtraction {++++ private Configuration conf;++ private TikaParser parser;++++ @BeforeEach++ public void setup() {++ conf = NutchConfiguration.create();++ conf.set("plugin.includes", "parse-tika");++ parser = new TikaParser();++ parser.setConf(conf);++ }++++ @Test++ public void testBasicMetaTags() {++ String html = "<!DOCTYPE html><html><head>"++ + "<title>Page Title</title>"++ + "<meta name='description' content='Page description here'>"++ + "<meta name='keywords' content='keyword1, keyword2, keyword3'>"++ + "<meta name='author' content='Test Author'>"++ + "</head><body>Content</body></html>";++++ Parse parse = doParse(html);++ org.apache.nutch.metadata.Metadata parseMeta = parse.getData().getParseMeta();++++ assertEquals("Page Title", parse.getData().getTitle());++ assertEquals("Page description here", parseMeta.get("description"));++ assertEquals("keyword1, keyword2, keyword3", parseMeta.get("keywords"));++ assertEquals("Test Author", parseMeta.get("author"));++ }++++ @Test++ public void testOpenGraphMetaTags() {++ String html = "<!DOCTYPE html><html><head>"++ + "<title>Regular Title</title>"++ + "<meta property='og:title' content='OG Title'>"++ + "<meta property='og:description' content='OG Description'>"++ + "<meta property='og:type' content='article'>"++ + "<meta property='og:url' content='http://example.com/page'>"++ + "</head><body>Content</body></html>";++++ Parse parse = doParse(html);++ org.apache.nutch.metadata.Metadata parseMeta = parse.getData().getParseMeta();++++ assertEquals("Regular Title", parse.getData().getTitle());++ // OG tags should be in metadata++ assertNotNull(parseMeta.get("og:title"));++ }++++ @Test++ public void testTwitterCardMetaTags() {++ String html = "<!DOCTYPE html><html><head>"++ + "<title>Page Title</title>"++ + "<meta name='twitter:card' content='summary'>"++ + "<meta name='twitter:title' content='Twitter Title'>"++ + "<meta name='twitter:description' content='Twitter Description'>"++ + "</head><body>Content</body></html>";++++ Parse parse = doParse(html);++ org.apache.nutch.metadata.Metadata parseMeta = parse.getData().getParseMeta();++++ assertEquals("Page Title", parse.getData().getTitle());++ // Twitter tags should be in metadata++ assertNotNull(parseMeta.get("twitter:card"));++ }++++ @Test++ public void testRobotsMetaNoIndex() {++ String html = "<!DOCTYPE html><html><head>"++ + "<meta name='robots' content='noindex'>"++ + "<title>No Index Page</title>"++ + "</head><body>This should not be indexed</body></html>";++++ Parse parse = doParse(html);++++ // With noindex, text should be empty++ assertTrue(parse.getText().isEmpty() || parse.getText().isBlank(),++ "noindex should prevent text extraction");++ }++++ @Test++ public void testRobotsMetaNoFollow() {++ String html = "<!DOCTYPE html><html><head>"++ + "<meta name='robots' content='nofollow'>"++ + "</head><body>"++ + "<a href='http://example.com/link1'>Link 1</a>"++ + "<a href='http://example.com/link2'>Link 2</a>"++ + "</body></html>";++++ Parse parse = doParse(html);++++ // With nofollow, outlinks should be empty++ assertEquals(0, parse.getData().getOutlinks().length,++ "nofollow should prevent outlink extraction");++ }++++ @Test++ public void testRobotsMetaNoIndexNoFollow() {++ String html = "<!DOCTYPE html><html><head>"++ + "<meta name='robots' content='noindex, nofollow'>"++ + "<title>Blocked Page</title>"++ + "</head><body>"++ + "<p>Content here</p>"++ + "<a href='http://example.com/link'>Link</a>"++ + "</body></html>";++++ Parse parse = doParse(html);++++ // Both should be blocked++ assertTrue(parse.getText().isEmpty() || parse.getText().isBlank(),++ "noindex should prevent text extraction");++ assertEquals(0, parse.getData().getOutlinks().length,++ "nofollow should prevent outlink extraction");++ }++++ @Test++ public void testRefreshMetaTag() {++ String html = "<!DOCTYPE html><html><head>"++ + "<meta http-equiv='refresh' content='5;url=http://example.com/newpage'>"++ + "<title>Redirect Page</title>"++ + "</head><body>Redirecting...</body></html>";++++ Parse parse = doParse(html);++++ // Parse should still succeed++ assertNotNull(parse);++ assertEquals("Redirect Page", parse.getData().getTitle());++ }++++ @Test++ public void testBaseHref() {++ String html = "<!DOCTYPE html><html><head>"++ + "<base href='http://other.example.com/'>"++ + "<title>Base HREF Test</title>"++ + "</head><body>"++ + "<a href='page.html'>Relative Link</a>"++ + "</body></html>";++++ String url = "http://example.com/";++ Content content = new Content(url, url, html.getBytes(),++ "text/html", new Metadata(), conf);++ Parse parse = parser.getParse(content).get(url);++++ // Link should be resolved relative to base href++ if (parse.getData().getOutlinks().length > 0) {++ String linkUrl = parse.getData().getOutlinks()[0].getToUrl();++ assertTrue(linkUrl.contains("other.example.com") || linkUrl.contains("example.com"),++ "Link should be resolved using base href or original URL");++ }++ }++++ @Test++ public void testMultipleMetaTagsWithSameName() {++ String html = "<!DOCTYPE html><html><head>"++ + "<title>Multi Meta</title>"++ + "<meta name='keywords' content='first'>"++ + "<meta name='keywords' content='second'>"++ + "</head><body>Content</body></html>";++++ Parse parse = doParse(html);++ org.apache.nutch.metadata.Metadata parseMeta = parse.getData().getParseMeta();++++ // Should handle multiple meta tags with same name++ String keywords = parseMeta.get("keywords");++ assertNotNull(keywords, "Keywords should be extracted");++ }++++ @Test++ public void testEmptyMetaTags() {++ String html = "<!DOCTYPE html><html><head>"++ + "<title></title>"++ + "<meta name='description' content=''>"++ + "<meta name='keywords'>"++ + "</head><body>Content</body></html>";++++ Parse parse = doParse(html);++++ // Should handle empty meta tags gracefully++ assertNotNull(parse);++ // Empty title should result in empty string++ assertTrue(parse.getData().getTitle() == null ||++ parse.getData().getTitle().isEmpty());++ }++++ private Parse doParse(String html) {++ String url = "http://example.com/";++ Content content = new Content(url, url, html.getBytes(),++ "text/html", new Metadata(), conf);++ return parser.getParse(content).get(url);++ }++}diff --cc src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestParserFailureHandling.javaindex 000000000,000000000..2245db7a5new file mode 100644--- /dev/null+++ b/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestParserFailureHandling.java@@@ -1,0 -1,0 +1,221 @@@++/*++ * Licensed to the Apache Software Foundation (ASF) under one or more++ * contributor license agreements. See the NOTICE file distributed with++ * this work for additional information regarding copyright ownership.++ * The ASF licenses this file to You under the Apache License, Version 2.0++ * (the "License"); you may not use this file except in compliance with++ * the License. You may obtain a copy of the License at++ *++ * http://www.apache.org/licenses/LICENSE-2.0++ *++ * Unless required by applicable law or agreed to in writing, software++ * distributed under the License is distributed on an "AS IS" BASIS,++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.++ * See the License for the specific language governing permissions and++ * limitations under the License.++ */++package org.apache.nutch.parse.tika;++++import org.apache.hadoop.conf.Configuration;++import org.apache.nutch.metadata.Metadata;++import org.apache.nutch.parse.ParseResult;++import org.apache.nutch.protocol.Content;++import org.apache.nutch.util.NutchConfiguration;++import org.junit.jupiter.api.BeforeEach;++import org.junit.jupiter.api.Test;++++import static org.junit.jupiter.api.Assertions.*;++++/**++ * Tests for parser failure handling and graceful degradation.++ * These tests ensure the parser handles edge cases without throwing exceptions.++ */++public class TestParserFailureHandling {++++ private Configuration conf;++ private TikaParser parser;++++ @BeforeEach++ public void setup() {++ conf = NutchConfiguration.create();++ conf.set("plugin.includes", "parse-tika");++ parser = new TikaParser();++ parser.setConf(conf);++ }++++ @Test++ public void testMalformedHtml() {++ // Severely malformed HTML++ String html = "<<<>>>not really html at all{{{";++++ ParseResult result = doParse(html, "text/html");++++ // Should return a result even for malformed input (not throw exception)++ assertNotNull(result, "Should return a result even for malformed input");++ }++++ @Test++ public void testEmptyContent() {++ ParseResult result = doParse("", "text/html");++++ // Should handle empty content gracefully++ assertNotNull(result, "Should handle empty content");++ }++++ @Test++ public void testNullBytes() {++ String url = "http://example.com/";++ Content content = new Content(url, url, new byte[0],++ "text/html", new Metadata(), conf);++++ ParseResult result = parser.getParse(content);++++ // Should handle zero-length byte array++ assertNotNull(result, "Should handle zero-length content");++ }++++ @Test++ public void testBinaryContentAsHtml() {++ // Random binary content declared as HTML++ byte[] binaryContent = new byte[]{++ 0x00, 0x01, 0x02, (byte)0xFF, (byte)0xFE, 0x50, 0x4B, 0x03, 0x04++ };++++ String url = "http://example.com/";++ Content content = new Content(url, url, binaryContent,++ "text/html", new Metadata(), conf);++++ // Should not throw exception++ ParseResult result = parser.getParse(content);++ assertNotNull(result, "Should handle binary content declared as HTML");++ }++++ @Test++ public void testUnknownMimeType() {++ String url = "http://example.com/file.xyz";++ Content content = new Content(url, url, "some content".getBytes(),++ "application/x-unknown-type", new Metadata(), conf);++++ ParseResult result = parser.getParse(content);++++ // Should handle gracefully (may return failed status but not throw)++ assertNotNull(result, "Should handle unknown MIME type");++ }++++ @Test++ public void testVeryLongTitle() {++ StringBuilder sb = new StringBuilder();++ sb.append("<!DOCTYPE html><html><head><title>");++ for (int i = 0; i < 10000; i++) {++ sb.append("VeryLongTitle");++ }++ sb.append("</title></head><body>Content</body></html>");++++ ParseResult result = doParse(sb.toString(), "text/html");++++ assertNotNull(result, "Should handle very long titles");++ }++++ @Test++ public void testDeeplyNestedHtml() {++ StringBuilder sb = new StringBuilder();++ sb.append("<!DOCTYPE html><html><body>");++ // Create deeply nested divs++ for (int i = 0; i < 500; i++) {++ sb.append("<div>");++ }++ sb.append("Deep content");++ for (int i = 0; i < 500; i++) {++ sb.append("</div>");++ }++ sb.append("</body></html>");++++ ParseResult result = doParse(sb.toString(), "text/html");++++ assertNotNull(result, "Should handle deeply nested HTML");++ }++++ @Test++ public void testManyLinks() {++ StringBuilder sb = new StringBuilder();++ sb.append("<!DOCTYPE html><html><body>");++ // Create many links++ for (int i = 0; i < 1000; i++) {++ sb.append("<a href='http://example.com/page").append(i).append("'>Link ").append(i).append("</a>");++ }++ sb.append("</body></html>");++++ ParseResult result = doParse(sb.toString(), "text/html");++++ assertNotNull(result, "Should handle many links");++ }++++ @Test++ public void testSpecialCharactersInUrls() {++ String html = "<!DOCTYPE html><html><body>"++ + "<a href='http://example.com/path?q=hello world&foo=bar'>Link 1</a>"++ + "<a href='http://example.com/path?q=hello%20world'>Link 2</a>"++ + "<a href='http://example.com/path#section'>Link 3</a>"++ + "<a href='http://example.com/路径'>Link 4</a>"++ + "</body></html>";++++ ParseResult result = doParse(html, "text/html");++++ assertNotNull(result, "Should handle special characters in URLs");++ }++++ @Test++ public void testNullCharactersInContent() {++ // HTML with null characters embedded++ byte[] htmlBytes = "<!DOCTYPE html><html><body>Before\0After</body></html>".getBytes();++++ String url = "http://example.com/";++ Content content = new Content(url, url, htmlBytes,++ "text/html", new Metadata(), conf);++++ ParseResult result = parser.getParse(content);++++ assertNotNull(result, "Should handle null characters in content");++ }++++ @Test++ public void testIncompleteHtml() {++ // HTML that cuts off mid-tag++ String html = "<!DOCTYPE html><html><head><title>Test</title></head><body><div>Content<a href='http://";++++ ParseResult result = doParse(html, "text/html");++++ assertNotNull(result, "Should handle incomplete HTML");++ }++++ @Test++ public void testHtmlWithScriptErrors() {++ String html = "<!DOCTYPE html><html><head>"++ + "<script>this is { not valid javascript</script>"++ + "<title>Script Error</title>"++ + "</head><body>Content</body></html>";++++ ParseResult result = doParse(html, "text/html");++++ assertNotNull(result, "Should handle invalid JavaScript in script tags");++ }++++ @Test++ public void testHtmlWithCssErrors() {++ String html = "<!DOCTYPE html><html><head>"++ + "<style>this is { not: valid; css }</style>"++ + "<title>CSS Error</title>"++ + "</head><body>Content</body></html>";++++ ParseResult result = doParse(html, "text/html");++++ assertNotNull(result, "Should handle invalid CSS in style tags");++ }++++ private ParseResult doParse(String content, String mimeType) {++ String url = "http://example.com/";++ Content c = new Content(url, url, content.getBytes(),++ mimeType, new Metadata(), conf);++ return parser.getParse(c);++ }++}