Skip to content

armgit / nutch

Public Imported from GitHub
0

Apache Nutch is an extensible and scalable web crawler

NUTCH-3110 Upgrade to Tika 3.2.3

lewismc713835b73bfd
diff --cc ivy/ivy.xml
index 98713b0f0,70614f8fd..44914368e
--- a/ivy/ivy.xml
+++ b/ivy/ivy.xml
@@@ -68,13 -69,12 +68,13 @@@
<exclude org="ch.qos.reload4j" name="*"/>
<exclude org="org.slf4j" name="*" />
</dependency>
- <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-jobclient" rev="3.3.6" conf="*->default">
+ <dependency org="org.apache.hadoop" name="hadoop-mapreduce-client-jobclient" rev="3.4.2" conf="*->default">
<exclude org="ch.qos.reload4j" name="*"/>
<exclude org="org.slf4j" name="*" />
- </dependency><!-- End of Hadoop Dependencies -->
+ </dependency>
+ <!-- End of Hadoop Dependencies -->
- <dependency org="org.tallison.tika" name="tika-core-shaded" rev="2.9.1.0" conf="*->default" transitive="false"/>
- <dependency org="org.tallison.tika" name="tika-core-shaded" rev="3.1.0.0" conf="*->default" transitive="false"/>
++ <dependency org="org.apache.tika" name="tika-core" rev="3.2.3" conf="*->default"/>
<dependency org="xml-apis" name="xml-apis" rev="1.4.01" /><!-- force this version as it is required by Tika -->
<dependency org="xerces" name="xercesImpl" rev="2.12.2" />
diff --cc src/plugin/language-identifier/ivy.xml
index 60dbbefd1,149d6c147..448619ac1
--- a/src/plugin/language-identifier/ivy.xml
+++ b/src/plugin/language-identifier/ivy.xml
@@@ -37,13 -37,7 +37,13 @@@
</publications>
<dependencies>
- <dependency org="org.apache.tika" name="tika-langdetect-optimaize" rev="2.9.0" conf="*->default">
- <dependency org="org.tallison.tika" name="tika-langdetect-optimaize-shaded" rev="3.1.0.0" conf="*->default" transitive="false"/>
++ <dependency org="org.apache.tika" name="tika-langdetect-optimaize" rev="3.2.3" conf="*->default">
+ <!-- exclusions of dependencies provided in Nutch core (ivy/ivy.xml) -->
+ <exclude org="org.apache.tika" name="tika-core" />
+ <exclude org="com.google.guava" name="guava" />
+ <exclude org="org.slf4j" name="slf4j-api" />
+ <!-- exclusions of dependencies provided in Nutch core (ivy/ivy.xml) -->
+ </dependency>
</dependencies>
</ivy-module>
diff --cc src/plugin/language-identifier/plugin.xml
index a65ca78cc,a4625d060..7fc3d555c
--- a/src/plugin/language-identifier/plugin.xml
+++ b/src/plugin/language-identifier/plugin.xml
@@@ -26,16 -26,7 +26,15 @@@
<export name="*"/>
</library>
<!-- dependencies of Tika's Optimaize language detector (tika-langdetect-optimaize) -->
- <library name="annotations-12.0.jar"/>
- <library name="checker-qual-3.33.0.jar"/>
- <library name="error_prone_annotations-2.18.0.jar"/>
- <library name="failureaccess-1.0.1.jar"/>
- <library name="j2objc-annotations-2.8.jar"/>
- <library name="tika-langdetect-optimaize-shaded-3.1.0.0.jar"/>
++ <library name="annotations-26.0.2.jar"/>
++ <library name="error_prone_annotations-2.41.0.jar"/>
++ <library name="failureaccess-1.0.3.jar"/>
++ <library name="j2objc-annotations-3.0.0.jar"/>
+ <library name="jsonic-1.2.11.jar"/>
- <library name="jsr305-3.0.2.jar"/>
++ <library name="jspecify-1.0.0.jar"/>
+ <library name="language-detector-0.6.jar"/>
+ <library name="listenablefuture-9999.0-empty-to-avoid-conflict-with-guava.jar"/>
- <library name="tika-langdetect-optimaize-2.9.0.jar"/>
++ <library name="tika-langdetect-optimaize-3.2.3.jar"/>
</runtime>
<requires>
diff --cc src/plugin/parse-tika/howto_upgrade_tika.md
index 8ed6c3f3c,8ed6c3f3c..3c81a607a
--- a/src/plugin/parse-tika/howto_upgrade_tika.md
+++ b/src/plugin/parse-tika/howto_upgrade_tika.md
@@@ -15,31 -15,31 +15,17 @@@
limitations under the License.
-->
--We are currently using a shim (https://github.com/tballison/hadoop-safe-tika
--because of binary conflicts in commons-io versions between what Hadoop supports and the more
--modern features that Apache Tika and Apache POI were using in commons-io.
++# How to Upgrade Apache Tika in Nutch
--For now, all you have to do is update the fat jar dependencies:
++Nutch uses official Apache Tika artifacts for document parsing and language detection.
--1. tika-core-shaded in ivy/ivy.xml
--
--2. tika-parsers-standard-package-shaded in src/plugin/parse-tika/ivy.xml
--
--3. The library name version for tika-parsers-standard-package-shaded in src/plugin/parse-tika/plugin.xml
--
--4. Repeat steps 2 and 3 for the language-identifier
--
--5. Build Nutch and run all unit tests:
--
-- $ cd ../../../
-- $ ant clean runtime test
--
--The following directions are what we used to do with thin jars. Hopefully, we'll
--be able to get back to these directions once we have version harmony with Hadoop and Tika/POI.
++## Upgrade Steps
1. Upgrade Tika dependency (tika-core) in ivy/ivy.xml
--2. Upgrade Tika dependency in src/plugin/parse-tika/ivy.xml
++2. Upgrade Tika dependency (tika-parsers-standard-package) in src/plugin/parse-tika/ivy.xml
++ Note: The tika-handler-boilerpipe module provides Boilerpipe support for text extraction
++ (tika.extractor=boilerpipe configuration). This was moved from tika-core in TIKA-4138.
3. Upgrade Tika's own dependencies in src/plugin/parse-tika/plugin.xml
@@@ -49,9 -49,9 +35,9 @@@
$ ls lib | sed 's/^/ <library name="/g' | sed 's/$/"\/>/g'
In the plugin.xml replace all lines between
-- <!-- dependencies of Tika (tika-parsers) -->
++ <!-- dependencies of Tika (tika-parsers-standard-package) -->
and
-- <!-- end of dependencies of Tika (tika-parsers) -->
++ <!-- end of dependencies of Tika (tika-parsers-standard-package) -->
with the output of the command above.
4. (Optionally) remove overlapping dependencies between parse-tika and Nutch core dependencies:
@@@ -61,7 -61,7 +47,7 @@@
build/plugins/parse-tika/
(eventually with different versions)
- duplicated libs can be added to the exclusions of transitive dependencies in
-- build/plugins/parse-tika/ivy.xml
++ src/plugin/parse-tika/ivy.xml
- but the library versions in ivy/ivy.xml MUST correspond to those required by Tika
5. Remove the locally "installed" dependencies in src/plugin/parse-tika/lib/:
@@@ -72,6 -72,6 +58,11 @@@
$ cd ../language-identifier/
++ Update tika-langdetect-optimaize in src/plugin/language-identifier/ivy.xml
++ Then regenerate the library list in plugin.xml:
++ $ ant -f ./build-ivy.xml
++ $ ls lib | sed 's/^/ <library name="/g' | sed 's/$/"\/>/g'
++
7. Build Nutch and run all unit tests:
$ cd ../../../
diff --cc src/plugin/parse-tika/ivy.xml
index 6d96ed3cd,f4aa049a9..23af88eec
--- a/src/plugin/parse-tika/ivy.xml
+++ b/src/plugin/parse-tika/ivy.xml
@@@ -37,7 -37,7 +37,15 @@@
</publications>
<dependencies>
- <dependency org="org.tallison.tika" name="tika-parsers-standard-package-shaded" rev="2.9.1.0" conf="*->default" transitive="false"/>
- <dependency org="org.tallison.tika" name="tika-parsers-standard-package-shaded" rev="3.1.0.0" conf="*->default" transitive="false"/>
++ <dependency org="org.apache.tika" name="tika-parsers-standard-package" rev="3.2.3" conf="*->default">
++ <!-- exclusions of dependencies provided in Nutch core (ivy/ivy.xml) -->
++ <exclude org="org.apache.tika" name="tika-core"/>
++ <exclude org="org.slf4j" name="slf4j-api"/>
++ </dependency>
++ <!-- Boilerpipe for text extraction (tika.extractor=boilerpipe) - TIKA-4138 -->
++ <dependency org="org.apache.tika" name="tika-handler-boilerpipe" rev="3.2.3" conf="*->default">
++ <exclude org="org.slf4j" name="slf4j-api"/>
++ </dependency>
</dependencies>
</ivy-module>
diff --cc src/plugin/parse-tika/plugin.xml
index 04afb9fac,876174893..1b9252cc6
--- a/src/plugin/parse-tika/plugin.xml
+++ b/src/plugin/parse-tika/plugin.xml
@@@ -25,7 -25,7 +25,97 @@@
<library name="parse-tika.jar">
<export name="*"/>
</library>
- <library name="tika-parsers-standard-package-shaded-2.9.1.0.jar"/>
- <library name="tika-parsers-standard-package-shaded-3.1.0.0.jar"/>
++ <!-- dependencies of Tika (tika-parsers-standard-package) -->
++ <library name="SparseBitSet-1.3.jar"/>
++ <library name="angus-activation-2.0.2.jar"/>
++ <library name="apache-mime4j-core-0.8.13.jar"/>
++ <library name="apache-mime4j-dom-0.8.13.jar"/>
++ <library name="asm-9.8.jar"/>
++ <library name="bcjmail-jdk18on-1.81.jar"/>
++ <library name="bcpkix-jdk18on-1.81.jar"/>
++ <library name="bcprov-jdk18on-1.81.jar"/>
++ <library name="bcutil-jdk18on-1.81.jar"/>
++ <library name="boilerpipe-1.1.0.jar"/>
++ <library name="commons-codec-1.19.0.jar"/>
++ <library name="commons-collections4-4.5.0.jar"/>
++ <library name="commons-compress-1.28.0.jar"/>
++ <library name="commons-csv-1.14.1.jar"/>
++ <library name="commons-exec-1.5.0.jar"/>
++ <library name="commons-io-2.20.0.jar"/>
++ <library name="commons-lang3-3.18.0.jar"/>
++ <library name="commons-logging-1.3.5.jar"/>
++ <library name="commons-math3-3.6.1.jar"/>
++ <library name="curvesapi-1.08.jar"/>
++ <library name="dd-plist-1.28.jar"/>
++ <library name="dec-0.1.2.jar"/>
++ <library name="fontbox-3.0.5.jar"/>
++ <library name="istack-commons-runtime-4.1.2.jar"/>
++ <library name="jackcess-4.0.8.jar"/>
++ <library name="jackcess-encrypt-4.0.3.jar"/>
++ <library name="jackson-annotations-2.20.jar"/>
++ <library name="jackson-core-2.20.0.jar"/>
++ <library name="jackson-databind-2.20.0.jar"/>
++ <library name="jai-imageio-core-1.4.0.jar"/>
++ <library name="jakarta.activation-api-2.1.3.jar"/>
++ <library name="jakarta.xml.bind-api-4.0.2.jar"/>
++ <library name="java-libpst-0.9.3.jar"/>
++ <library name="jaxb-core-4.0.5.jar"/>
++ <library name="jaxb-runtime-4.0.5.jar"/>
++ <library name="jbig2-imageio-3.0.4.jar"/>
++ <library name="jcl-over-slf4j-2.0.17.jar"/>
++ <library name="jdom2-2.0.6.1.jar"/>
++ <library name="jempbox-1.8.17.jar"/>
++ <library name="jhighlight-1.1.0.jar"/>
++ <library name="jmatio-1.5.jar"/>
++ <library name="jsoup-1.21.2.jar"/>
++ <library name="juniversalchardet-2.5.0.jar"/>
++ <library name="junrar-7.5.5.jar"/>
++ <library name="jwarc-0.32.0.jar"/>
++ <library name="log4j-api-2.25.1.jar"/>
++ <library name="metadata-extractor-2.19.0.jar"/>
++ <library name="parso-2.0.14.jar"/>
++ <library name="pdfbox-3.0.5.jar"/>
++ <library name="pdfbox-io-3.0.5.jar"/>
++ <library name="pdfbox-tools-3.0.5.jar"/>
++ <library name="picocli-4.7.6.jar"/>
++ <library name="poi-5.4.1.jar"/>
++ <library name="poi-ooxml-5.4.1.jar"/>
++ <library name="poi-ooxml-lite-5.4.1.jar"/>
++ <library name="poi-scratchpad-5.4.1.jar"/>
++ <library name="rome-2.1.0.jar"/>
++ <library name="rome-utils-2.1.0.jar"/>
++ <library name="tika-parser-apple-module-3.2.3.jar"/>
++ <library name="tika-parser-audiovideo-module-3.2.3.jar"/>
++ <library name="tika-parser-cad-module-3.2.3.jar"/>
++ <library name="tika-parser-code-module-3.2.3.jar"/>
++ <library name="tika-parser-crypto-module-3.2.3.jar"/>
++ <library name="tika-parser-digest-commons-3.2.3.jar"/>
++ <library name="tika-parser-font-module-3.2.3.jar"/>
++ <library name="tika-parser-html-module-3.2.3.jar"/>
++ <library name="tika-parser-image-module-3.2.3.jar"/>
++ <library name="tika-parser-mail-commons-3.2.3.jar"/>
++ <library name="tika-parser-mail-module-3.2.3.jar"/>
++ <library name="tika-parser-microsoft-module-3.2.3.jar"/>
++ <library name="tika-parser-miscoffice-module-3.2.3.jar"/>
++ <library name="tika-parser-news-module-3.2.3.jar"/>
++ <library name="tika-parser-ocr-module-3.2.3.jar"/>
++ <library name="tika-parser-pdf-module-3.2.3.jar"/>
++ <library name="tika-parser-pkg-module-3.2.3.jar"/>
++ <library name="tika-parser-text-module-3.2.3.jar"/>
++ <library name="tika-parser-webarchive-module-3.2.3.jar"/>
++ <library name="tika-parser-xml-module-3.2.3.jar"/>
++ <library name="tika-parser-xmp-commons-3.2.3.jar"/>
++ <library name="tika-parser-zip-commons-3.2.3.jar"/>
++ <library name="tika-parsers-standard-package-3.2.3.jar"/>
++ <library name="tika-handler-boilerpipe-3.2.3.jar"/>
++ <library name="txw2-4.0.5.jar"/>
++ <library name="vorbis-java-core-0.8.jar"/>
++ <library name="vorbis-java-tika-0.8.jar"/>
++ <library name="xmlbeans-5.3.0.jar"/>
++ <library name="xmpbox-3.0.5.jar"/>
++ <library name="xmpcore-6.1.11.jar"/>
++ <library name="xz-1.10.jar"/>
++ <!-- end of dependencies of Tika (tika-parsers-standard-package) -->
</runtime>
<requires>
diff --cc src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/DOMContentUtils.java
index 81ef6c523,81ef6c523..048afd711
--- a/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/DOMContentUtils.java
+++ b/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/DOMContentUtils.java
@@@ -22,6 -22,6 +22,7 @@@ import java.util.ArrayList
import java.util.Collection;
import java.util.HashMap;
import java.util.HashSet;
++import java.util.LinkedHashMap;
import java.util.List;
import java.util.Set;
import java.util.regex.Pattern;
@@@ -394,6 -394,6 +395,8 @@@ public class DOMContentUtils
* @param node a {@link Node} under which to discover anchors
*/
public void getOutlinks(URL base, ArrayList<Outlink> outlinks, Node node) {
++ // Use a LinkedHashMap to deduplicate outlinks by URL while preserving insertion order
++ LinkedHashMap<String, Outlink> outlinkMap = new LinkedHashMap<>();
NodeWalker walker = new NodeWalker(node);
while (walker.hasNext()) {
@@@ -435,16 -435,16 +438,30 @@@
try {
URL url = URLUtil.resolveURL(base, target);
-- Outlink outlink = new Outlink(url.toString(), linkText
-- .toString().trim());
-- outlinks.add(outlink);
--
-- // NUTCH-2433 - Keep the node name where the URL was found into
-- // the outlink metadata
-- if (keepNodenames) {
-- MapWritable metadata = new MapWritable();
-- metadata.put(new Text(srcTagMetaName), new Text(nodeName));
-- outlink.setMetadata(metadata);
++ String urlStr = url.toString();
++ String anchor = linkText.toString().trim();
++
++ // Only add if URL not seen, or replace if current has better anchor
++ Outlink existing = outlinkMap.get(urlStr);
++ if (existing == null) {
++ Outlink outlink = new Outlink(urlStr, anchor);
++ // NUTCH-2433 - Keep the node name where the URL was found into
++ // the outlink metadata
++ if (keepNodenames) {
++ MapWritable metadata = new MapWritable();
++ metadata.put(new Text(srcTagMetaName), new Text(nodeName));
++ outlink.setMetadata(metadata);
++ }
++ outlinkMap.put(urlStr, outlink);
++ } else if (existing.getAnchor().isEmpty() && !anchor.isEmpty()) {
++ // Replace empty anchor with non-empty one
++ Outlink outlink = new Outlink(urlStr, anchor);
++ if (keepNodenames) {
++ MapWritable metadata = new MapWritable();
++ metadata.put(new Text(srcTagMetaName), new Text(nodeName));
++ outlink.setMetadata(metadata);
++ }
++ outlinkMap.put(urlStr, outlink);
}
} catch (MalformedURLException e) {
// don't care
@@@ -456,6 -456,6 +473,8 @@@
}
}
}
++
++ outlinks.addAll(outlinkMap.values());
}
// This one is used by NUTCH-1918
@@@ -464,6 -464,6 +483,9 @@@
String target = null;
String anchor = null;
boolean noFollow = false;
++
++ // Use a LinkedHashMap to deduplicate outlinks by URL while preserving insertion order
++ LinkedHashMap<String, Outlink> outlinkMap = new LinkedHashMap<>();
for (Link link : tikaExtractedOutlinks) {
target = link.getUri();
@@@ -475,17 -475,17 +497,27 @@@
if (target != null && !noFollow) {
try {
URL url = URLUtil.resolveURL(base, target);
++ String urlStr = url.toString();
// clean the anchor
anchor = anchor.replaceAll("\\s+", " ");
anchor = anchor.trim();
-- outlinks.add(new Outlink(url.toString(), anchor));
++ // Only add if URL not seen, or replace if current has better anchor
++ Outlink existing = outlinkMap.get(urlStr);
++ if (existing == null) {
++ outlinkMap.put(urlStr, new Outlink(urlStr, anchor));
++ } else if (existing.getAnchor().isEmpty() && !anchor.isEmpty()) {
++ // Replace empty anchor with non-empty one
++ outlinkMap.put(urlStr, new Outlink(urlStr, anchor));
++ }
} catch (MalformedURLException e) {
// don't care
}
}
}
}
++
++ outlinks.addAll(outlinkMap.values());
}
}
diff --cc src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaParser.java
index e75f70aef,85cd1fae0..2c110309e
--- a/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaParser.java
+++ b/src/plugin/parse-tika/src/java/org/apache/nutch/parse/tika/TikaParser.java
@@@ -22,14 -22,14 +22,16 @@@ import java.net.MalformedURLException
import java.net.URL;
import java.util.ArrayList;
import java.util.Arrays;
--import java.util.HashSet;
import java.util.List;
import java.util.Map;
--import java.util.Set;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.html.dom.HTMLDocumentImpl;
++
++import de.l3s.boilerpipe.BoilerpipeExtractor;
++import org.apache.tika.sax.boilerpipe.BoilerpipeContentHandler;
++
import org.apache.nutch.metadata.Nutch;
import org.apache.nutch.parse.HTMLMetaTags;
import org.apache.nutch.parse.HtmlParseFilters;
@@@ -45,7 -45,7 +47,6 @@@ import org.apache.tika.config.TikaConfi
import org.apache.tika.metadata.Metadata;
import org.apache.tika.metadata.TikaCoreProperties;
import org.apache.tika.mime.MediaType;
--import org.apache.tika.sax.boilerpipe.BoilerpipeContentHandler;
import org.apache.tika.parser.AutoDetectParser;
import org.apache.tika.parser.CompositeParser;
import org.apache.tika.parser.ParseContext;
@@@ -77,9 -77,9 +78,11 @@@ public class TikaParser implements org.
private HtmlMapper HTMLMapper;
private boolean parseEmbedded = true;
private boolean upperCaseElementNames = true;
++
++ // Boilerpipe configuration
private boolean useBoilerpipe;
private String boilerpipeExtractorName;
-- private Set<String> boilerpipeMimeTypes;
++ private String[] boilerpipeMimeTypes;
@Override
public ParseResult getParse(Content content) {
@@@ -118,21 -118,21 +121,10 @@@
byte[] raw = content.getContent();
Metadata tikamd = new Metadata();
-- ContentHandler domHandler;
--
-- // Check whether to use Tika's BoilerplateContentHandler
-- if (useBoilerpipe && boilerpipeMimeTypes.contains(mimeType)) {
-- BoilerpipeContentHandler bpHandler = new BoilerpipeContentHandler(
-- (ContentHandler) new DOMBuilder(doc, root),
-- BoilerpipeExtractorRepository.getExtractor(boilerpipeExtractorName));
-- bpHandler.setIncludeMarkup(true);
-- domHandler = (ContentHandler) bpHandler;
-- } else {
-- DOMBuilder domBuilder = new DOMBuilder(doc, root);
-- domBuilder.setUpperCaseElementNames(upperCaseElementNames);
-- domBuilder.setDefaultNamespaceURI(XHTMLContentHandler.XHTML);
-- domHandler = (ContentHandler) domBuilder;
-- }
++ DOMBuilder domBuilder = new DOMBuilder(doc, root);
++ domBuilder.setUpperCaseElementNames(upperCaseElementNames);
++ domBuilder.setDefaultNamespaceURI(XHTMLContentHandler.XHTML);
++ ContentHandler domHandler = (ContentHandler) domBuilder;
LinkContentHandler linkContentHandler = new LinkContentHandler();
@@@ -141,17 -141,17 +133,28 @@@
context.set(Parser.class, new AutoDetectParser(tikaConfig));
}
-- TeeContentHandler teeContentHandler = new TeeContentHandler(domHandler,
-- linkContentHandler);
++ ContentHandler contentHandler;
++ // Check if boilerpipe should be used for this content
++ if (useBoilerpipe && boilerpipeExtractorName != null
++ && Arrays.asList(boilerpipeMimeTypes).contains(mimeType)) {
++ BoilerpipeExtractor extractor = BoilerpipeExtractorRepository
++ .getExtractor(boilerpipeExtractorName);
++ BoilerpipeContentHandler boilerpipeHandler = new BoilerpipeContentHandler(
++ domHandler, extractor);
++ contentHandler = new TeeContentHandler(boilerpipeHandler,
++ linkContentHandler);
++ } else {
++ contentHandler = new TeeContentHandler(domHandler, linkContentHandler);
++ }
if (HTMLMapper != null)
context.set(HtmlMapper.class, HTMLMapper);
tikamd.set(Metadata.CONTENT_TYPE, mimeType);
try {
-- parser.parse(new ByteArrayInputStream(raw),
-- (ContentHandler) teeContentHandler, tikamd, context);
++ parser.parse(new ByteArrayInputStream(raw), contentHandler, tikamd,
++ context);
} catch (Exception e) {
- LOG.error("Error parsing " + content.getUrl(), e);
+ LOG.error("Error parsing {}", content.getUrl(), e);
return new ParseStatus(ParseStatus.FAILED, e.getMessage())
.getEmptyParseResult(content.getUrl(), getConf());
}
@@@ -317,13 -317,13 +320,14 @@@
Nutch.CACHING_FORBIDDEN_CONTENT);
upperCaseElementNames = conf.getBoolean("tika.uppercase.element.names",
true);
++ parseEmbedded = conf.getBoolean("tika.parse.embedded", true);
++
++ // Boilerpipe configuration
useBoilerpipe = conf.get("tika.extractor", "none").equals("boilerpipe");
boilerpipeExtractorName = conf.get("tika.extractor.boilerpipe.algorithm",
"ArticleExtractor");
-- boilerpipeMimeTypes = new HashSet<>(Arrays
-- .asList(conf.getTrimmedStrings("tika.extractor.boilerpipe.mime.types",
-- "text/html", "application/xhtml+xml")));
-- parseEmbedded = conf.getBoolean("tika.parse.embedded", true);
++ boilerpipeMimeTypes = conf.getStrings("tika.extractor.boilerpipe.mime.types",
++ "text/html", "application/xhtml+xml");
}
@Override
diff --cc src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestBoilerpipeExtraction.java
index 000000000,000000000..f5b480e3f
new file mode 100644
--- /dev/null
+++ b/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestBoilerpipeExtraction.java
@@@ -1,0 -1,0 +1,112 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one or more
++ * contributor license agreements. See the NOTICE file distributed with
++ * this work for additional information regarding copyright ownership.
++ * The ASF licenses this file to You under the Apache License, Version 2.0
++ * (the "License"); you may not use this file except in compliance with
++ * the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++package org.apache.nutch.parse.tika;
++
++import org.apache.hadoop.conf.Configuration;
++import org.apache.nutch.metadata.Metadata;
++import org.apache.nutch.parse.Parse;
++import org.apache.nutch.protocol.Content;
++import org.apache.nutch.util.NutchConfiguration;
++import org.junit.jupiter.api.Test;
++
++import static org.junit.jupiter.api.Assertions.*;
++
++/**
++ * Tests for Boilerpipe content extraction integration with Tika.
++ * These tests help ensure boilerpipe support works correctly across Tika upgrades.
++ */
++public class TestBoilerpipeExtraction {
++
++ private static final String HTML_WITH_BOILERPLATE =
++ "<!DOCTYPE html><html><head><title>Article Title</title></head><body>"
++ + "<div id='header'>Navigation | Menu | Links</div>"
++ + "<div id='content'>"
++ + "<h1>Main Article Heading</h1>"
++ + "<p>This is the main article content that should be extracted. "
++ + "It contains important information about the topic.</p>"
++ + "<p>Another paragraph with meaningful content that adds value.</p>"
++ + "</div>"
++ + "<div id='footer'>Copyright 2024 | Terms | Privacy</div>"
++ + "</body></html>";
++
++ @Test
++ public void testBoilerpipeExtraction() {
++ Configuration conf = NutchConfiguration.create();
++ conf.set("plugin.includes", "parse-tika");
++ conf.set("tika.extractor", "boilerpipe");
++ conf.set("tika.extractor.boilerpipe.algorithm", "ArticleExtractor");
++
++ TikaParser parser = new TikaParser();
++ parser.setConf(conf);
++
++ String url = "http://example.com/article.html";
++ Content content = new Content(url, url,
++ HTML_WITH_BOILERPLATE.getBytes(), "text/html", new Metadata(), conf);
++
++ Parse parse = parser.getParse(content).get(url);
++ String text = parse.getText();
++
++ // Boilerpipe should extract main content
++ assertTrue(text.contains("Main Article Heading"),
++ "Should contain article heading");
++ assertTrue(text.contains("main article content"),
++ "Should contain main content");
++ }
++
++ @Test
++ public void testBoilerpipeDisabled() {
++ Configuration conf = NutchConfiguration.create();
++ conf.set("plugin.includes", "parse-tika");
++ conf.set("tika.extractor", "none"); // Default - boilerpipe disabled
++
++ TikaParser parser = new TikaParser();
++ parser.setConf(conf);
++
++ String url = "http://example.com/article.html";
++ Content content = new Content(url, url,
++ HTML_WITH_BOILERPLATE.getBytes(), "text/html", new Metadata(), conf);
++
++ Parse parse = parser.getParse(content).get(url);
++ String text = parse.getText();
++
++ // Without boilerpipe, all text should be extracted including boilerplate
++ assertTrue(text.contains("Navigation"), "Should contain navigation text");
++ assertTrue(text.contains("Copyright"), "Should contain footer text");
++ assertTrue(text.contains("Main Article Heading"), "Should contain main content");
++ }
++
++ @Test
++ public void testBoilerpipeWithNonHtmlMimeType() {
++ Configuration conf = NutchConfiguration.create();
++ conf.set("plugin.includes", "parse-tika");
++ conf.set("tika.extractor", "boilerpipe");
++ conf.set("tika.extractor.boilerpipe.algorithm", "ArticleExtractor");
++ // Only apply to HTML mime types
++ conf.setStrings("tika.extractor.boilerpipe.mime.types", "text/html");
++
++ TikaParser parser = new TikaParser();
++ parser.setConf(conf);
++
++ // Using XHTML mime type which is not in the configured list
++ String url = "http://example.com/article.xhtml";
++ Content content = new Content(url, url,
++ HTML_WITH_BOILERPLATE.getBytes(), "application/xhtml+xml", new Metadata(), conf);
++
++ Parse parse = parser.getParse(content).get(url);
++ assertNotNull(parse, "Should parse successfully even when boilerpipe doesn't apply");
++ }
++}
diff --cc src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestEncodingDetection.java
index 000000000,000000000..347d76a13
new file mode 100644
--- /dev/null
+++ b/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestEncodingDetection.java
@@@ -1,0 -1,0 +1,193 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one or more
++ * contributor license agreements. See the NOTICE file distributed with
++ * this work for additional information regarding copyright ownership.
++ * The ASF licenses this file to You under the Apache License, Version 2.0
++ * (the "License"); you may not use this file except in compliance with
++ * the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++package org.apache.nutch.parse.tika;
++
++import org.apache.hadoop.conf.Configuration;
++import org.apache.nutch.metadata.Metadata;
++import org.apache.nutch.parse.Parse;
++import org.apache.nutch.protocol.Content;
++import org.apache.nutch.util.NutchConfiguration;
++import org.junit.jupiter.api.BeforeEach;
++import org.junit.jupiter.api.Test;
++
++import java.nio.charset.Charset;
++import java.nio.charset.StandardCharsets;
++
++import static org.junit.jupiter.api.Assertions.*;
++
++/**
++ * Tests for encoding/charset detection.
++ * These tests help ensure charset detection works correctly across Tika upgrades.
++ */
++public class TestEncodingDetection {
++
++ private Configuration conf;
++ private TikaParser parser;
++
++ @BeforeEach
++ public void setup() {
++ conf = NutchConfiguration.create();
++ conf.set("plugin.includes", "parse-tika");
++ parser = new TikaParser();
++ parser.setConf(conf);
++ }
++
++ @Test
++ public void testUtf8WithMetaCharset() {
++ String html = "<!DOCTYPE html><html><head>"
++ + "<meta charset=\"utf-8\">"
++ + "<title>UTF-8 Test</title></head>"
++ + "<body>äöü ñ 中文</body></html>";
++
++ byte[] bytes = html.getBytes(StandardCharsets.UTF_8);
++ Parse parse = doParse(bytes);
++
++ assertEquals("UTF-8 Test", parse.getData().getTitle());
++ assertTrue(parse.getText().contains("äöü"));
++ assertTrue(parse.getText().contains("中文"));
++ }
++
++ @Test
++ public void testUtf8WithHttpEquiv() {
++ String html = "<!DOCTYPE html><html><head>"
++ + "<meta http-equiv=\"Content-Type\" content=\"text/html; charset=utf-8\">"
++ + "<title>UTF-8 HTTP-Equiv</title></head>"
++ + "<body>Special chars: äöü</body></html>";
++
++ byte[] bytes = html.getBytes(StandardCharsets.UTF_8);
++ Parse parse = doParse(bytes);
++
++ assertEquals("UTF-8 HTTP-Equiv", parse.getData().getTitle());
++ assertTrue(parse.getText().contains("äöü"));
++ }
++
++ @Test
++ public void testUtf8BOM() {
++ String html = "<!DOCTYPE html><html><head>"
++ + "<title>BOM Test</title></head>"
++ + "<body>Content with BOM</body></html>";
++
++ // UTF-8 BOM
++ byte[] bom = new byte[]{(byte)0xEF, (byte)0xBB, (byte)0xBF};
++ byte[] htmlBytes = html.getBytes(StandardCharsets.UTF_8);
++ byte[] withBom = new byte[bom.length + htmlBytes.length];
++ System.arraycopy(bom, 0, withBom, 0, bom.length);
++ System.arraycopy(htmlBytes, 0, withBom, bom.length, htmlBytes.length);
++
++ Parse parse = doParse(withBom);
++
++ assertEquals("BOM Test", parse.getData().getTitle());
++ assertTrue(parse.getText().contains("Content with BOM"));
++ }
++
++ @Test
++ public void testIso88591() {
++ String html = "<!DOCTYPE html><html><head>"
++ + "<meta charset=\"iso-8859-1\">"
++ + "<title>ISO-8859-1 Test</title></head>"
++ + "<body>German: \u00e4\u00f6\u00fc</body></html>";
++
++ try {
++ Charset charset = Charset.forName("ISO-8859-1");
++ byte[] bytes = html.getBytes(charset);
++ Parse parse = doParse(bytes);
++
++ assertEquals("ISO-8859-1 Test", parse.getData().getTitle());
++ } catch (Exception e) {
++ // ISO-8859-1 might not be available on all systems
++ // Skip test in that case
++ }
++ }
++
++ @Test
++ public void testUtf16BE() {
++ String html = "<!DOCTYPE html><html><head>"
++ + "<title>UTF-16BE Test</title></head>"
++ + "<body>Content</body></html>";
++
++ // UTF-16BE doesn't add BOM automatically
++ byte[] bytes = html.getBytes(StandardCharsets.UTF_16BE);
++
++ // Add BOM manually for UTF-16BE
++ byte[] bom = new byte[]{(byte)0xFE, (byte)0xFF};
++ byte[] withBom = new byte[bom.length + bytes.length];
++ System.arraycopy(bom, 0, withBom, 0, bom.length);
++ System.arraycopy(bytes, 0, withBom, bom.length, bytes.length);
++
++ Parse parse = doParse(withBom);
++
++ assertNotNull(parse.getData().getTitle());
++ }
++
++ @Test
++ public void testUtf16LE() {
++ String html = "<!DOCTYPE html><html><head>"
++ + "<title>UTF-16LE Test</title></head>"
++ + "<body>Content</body></html>";
++
++ // UTF-16LE doesn't add BOM automatically
++ byte[] bytes = html.getBytes(StandardCharsets.UTF_16LE);
++
++ // Add BOM manually for UTF-16LE
++ byte[] bom = new byte[]{(byte)0xFF, (byte)0xFE};
++ byte[] withBom = new byte[bom.length + bytes.length];
++ System.arraycopy(bom, 0, withBom, 0, bom.length);
++ System.arraycopy(bytes, 0, withBom, bom.length, bytes.length);
++
++ Parse parse = doParse(withBom);
++
++ assertNotNull(parse.getData().getTitle());
++ }
++
++ @Test
++ public void testNoEncodingDeclaration() {
++ // HTML without any encoding declaration - should default to something reasonable
++ String html = "<!DOCTYPE html><html><head>"
++ + "<title>No Encoding</title></head>"
++ + "<body>Plain ASCII content</body></html>";
++
++ byte[] bytes = html.getBytes(StandardCharsets.US_ASCII);
++ Parse parse = doParse(bytes);
++
++ assertEquals("No Encoding", parse.getData().getTitle());
++ assertTrue(parse.getText().contains("Plain ASCII content"));
++ }
++
++ @Test
++ public void testXmlDeclarationEncoding() {
++ String html = "<?xml version=\"1.0\" encoding=\"UTF-8\"?>"
++ + "<!DOCTYPE html><html xmlns=\"http://www.w3.org/1999/xhtml\">"
++ + "<head><title>XHTML Test</title></head>"
++ + "<body>Special: äöü</body></html>";
++
++ byte[] bytes = html.getBytes(StandardCharsets.UTF_8);
++
++ String url = "http://example.com/";
++ Content content = new Content(url, url, bytes,
++ "application/xhtml+xml", new Metadata(), conf);
++ Parse parse = parser.getParse(content).get(url);
++
++ assertNotNull(parse);
++ // Title extraction may vary for XHTML
++ }
++
++ private Parse doParse(byte[] bytes) {
++ String url = "http://example.com/";
++ Content content = new Content(url, url, bytes, "text/html", new Metadata(), conf);
++ return parser.getParse(content).get(url);
++ }
++}
diff --cc src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestHtmlParser.java
index 17d244447,781e8910a..0460cce70
--- a/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestHtmlParser.java
+++ b/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestHtmlParser.java
@@@ -76,8 -75,8 +76,10 @@@ public class TestHtmlParser
+ encodingTestContent },
{ "HTML5, utf-8, BOM", "utf-8",
"\ufeff<!DOCTYPE html>\n<html>\n<head>\n" + encodingTestContent },
++ // Note: UTF-16 encoder adds BOM automatically, so don't include explicit BOM
++ // to avoid double BOM (FE FF FE FF) which confuses parsers
{ "HTML5, utf-16, BOM", "utf-16",
-- "\ufeff<!DOCTYPE html>\n<html>\n<head>\n" + encodingTestContent } };
++ "<!DOCTYPE html>\n<html>\n<head>\n" + encodingTestContent } };
private static final String resolveBaseUrlTestContent = //
"<html>\n<head>\n" + //
diff --cc src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestLinkExtractionEdgeCases.java
index 000000000,000000000..fb3a04d2e
new file mode 100644
--- /dev/null
+++ b/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestLinkExtractionEdgeCases.java
@@@ -1,0 -1,0 +1,221 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one or more
++ * contributor license agreements. See the NOTICE file distributed with
++ * this work for additional information regarding copyright ownership.
++ * The ASF licenses this file to You under the Apache License, Version 2.0
++ * (the "License"); you may not use this file except in compliance with
++ * the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++package org.apache.nutch.parse.tika;
++
++import org.apache.hadoop.conf.Configuration;
++import org.apache.nutch.metadata.Metadata;
++import org.apache.nutch.parse.Outlink;
++import org.apache.nutch.parse.Parse;
++import org.apache.nutch.protocol.Content;
++import org.apache.nutch.util.NutchConfiguration;
++import org.junit.jupiter.api.BeforeEach;
++import org.junit.jupiter.api.Test;
++
++import java.util.Arrays;
++import java.util.Set;
++import java.util.stream.Collectors;
++
++import static org.junit.jupiter.api.Assertions.*;
++
++/**
++ * Tests for link extraction edge cases.
++ * These tests help catch behavior changes in Tika's link extraction across versions.
++ */
++public class TestLinkExtractionEdgeCases {
++
++ private Configuration conf;
++ private TikaParser parser;
++
++ @BeforeEach
++ public void setup() {
++ conf = NutchConfiguration.create();
++ conf.set("plugin.includes", "parse-tika");
++ parser = new TikaParser();
++ parser.setConf(conf);
++ }
++
++ @Test
++ public void testDuplicateLinkDeduplication() {
++ // Same URL appearing multiple times should be deduplicated
++ String html = "<html><body>"
++ + "<a href='http://example.com/page'>Link 1</a>"
++ + "<a href='http://example.com/page'>Link 2</a>"
++ + "<a href='http://example.com/page'>Link 3</a>"
++ + "</body></html>";
++
++ Outlink[] outlinks = parseOutlinks(html);
++
++ // Should deduplicate to single link
++ assertEquals(1, outlinks.length, "Duplicate URLs should be deduplicated");
++ assertEquals("http://example.com/page", outlinks[0].getToUrl());
++ }
++
++ @Test
++ public void testLinkOrderPreserved() {
++ String html = "<html><body>"
++ + "<a href='http://example.com/first'>First</a>"
++ + "<a href='http://example.com/second'>Second</a>"
++ + "<a href='http://example.com/third'>Third</a>"
++ + "</body></html>";
++
++ Outlink[] outlinks = parseOutlinks(html);
++
++ assertEquals(3, outlinks.length, "Should extract all 3 links");
++ assertEquals("http://example.com/first", outlinks[0].getToUrl(),
++ "First link should be first");
++ assertEquals("http://example.com/second", outlinks[1].getToUrl(),
++ "Second link should be second");
++ assertEquals("http://example.com/third", outlinks[2].getToUrl(),
++ "Third link should be third");
++ }
++
++ @Test
++ public void testNestedAnchors() {
++ // Malformed HTML with nested anchors
++ String html = "<html><body>"
++ + "<a href='/outer'><a href='/inner'>Nested</a></a>"
++ + "</body></html>";
++
++ Outlink[] outlinks = parseOutlinks(html);
++
++ // At minimum, inner link should be extracted
++ Set<String> urls = Arrays.stream(outlinks)
++ .map(Outlink::getToUrl)
++ .collect(Collectors.toSet());
++ assertTrue(urls.contains("http://example.com/inner"),
++ "Inner link should be extracted");
++ }
++
++ @Test
++ public void testRelativeLinks() {
++ String html = "<html><body>"
++ + "<a href='page.html'>Relative</a>"
++ + "<a href='./page.html'>Dot Relative</a>"
++ + "<a href='../parent.html'>Parent</a>"
++ + "<a href='/absolute.html'>Absolute Path</a>"
++ + "</body></html>";
++
++ Outlink[] outlinks = parseOutlinks(html, "http://example.com/dir/index.html");
++
++ Set<String> urls = Arrays.stream(outlinks)
++ .map(Outlink::getToUrl)
++ .collect(Collectors.toSet());
++
++ assertTrue(urls.contains("http://example.com/dir/page.html") ||
++ urls.contains("http://example.com/page.html"),
++ "Should resolve relative link");
++ assertTrue(urls.contains("http://example.com/absolute.html"),
++ "Should resolve absolute path");
++ }
++
++ @Test
++ public void testAnchorTextExtraction() {
++ String html = "<html><body>"
++ + "<a href='http://example.com/page'> Anchor Text </a>"
++ + "</body></html>";
++
++ Outlink[] outlinks = parseOutlinks(html);
++
++ assertEquals(1, outlinks.length);
++ // Anchor text should be trimmed and whitespace normalized
++ assertEquals("Anchor Text", outlinks[0].getAnchor().trim().replaceAll("\\s+", " "));
++ }
++
++ @Test
++ public void testEmptyAnchorText() {
++ String html = "<html><body>"
++ + "<a href='http://example.com/page'></a>"
++ + "<a href='http://example.com/page2'> </a>"
++ + "</body></html>";
++
++ Outlink[] outlinks = parseOutlinks(html);
++
++ // Links should still be extracted even with empty anchor text
++ assertTrue(outlinks.length >= 1, "Should extract links with empty anchors");
++ }
++
++ @Test
++ public void testImageLinks() {
++ String html = "<html><body>"
++ + "<a href='http://example.com/page'><img src='image.jpg' alt='Image Alt'></a>"
++ + "</body></html>";
++
++ Outlink[] outlinks = parseOutlinks(html);
++
++ // Should extract the anchor link (Tika may also extract image src as separate link)
++ Set<String> urls = Arrays.stream(outlinks)
++ .map(Outlink::getToUrl)
++ .collect(Collectors.toSet());
++ assertTrue(urls.contains("http://example.com/page"),
++ "Should extract the anchor link");
++ }
++
++ @Test
++ public void testFormActionNotExtracted() {
++ conf.setBoolean("parser.html.form.use_action", false);
++ parser.setConf(conf);
++
++ String html = "<html><body>"
++ + "<form action='http://example.com/submit' method='post'>"
++ + "<input type='submit'/>"
++ + "</form>"
++ + "<a href='http://example.com/link'>Real Link</a>"
++ + "</body></html>";
++
++ Outlink[] outlinks = parseOutlinks(html);
++
++ // Form action should not be extracted as outlink
++ Set<String> urls = Arrays.stream(outlinks)
++ .map(Outlink::getToUrl)
++ .collect(Collectors.toSet());
++ assertFalse(urls.contains("http://example.com/submit"),
++ "Form action should not be extracted when disabled");
++ assertTrue(urls.contains("http://example.com/link"),
++ "Regular links should still be extracted");
++ }
++
++ @Test
++ public void testFragmentLinks() {
++ String html = "<html><body>"
++ + "<a href='#section'>Fragment Only</a>"
++ + "<a href='page.html#section'>Page with Fragment</a>"
++ + "</body></html>";
++
++ Outlink[] outlinks = parseOutlinks(html);
++
++ // Fragment-only links typically not extracted, page links should be
++ Set<String> urls = Arrays.stream(outlinks)
++ .map(Outlink::getToUrl)
++ .collect(Collectors.toSet());
++
++ // The page link should be extracted (with or without fragment)
++ boolean hasPageLink = urls.stream()
++ .anyMatch(u -> u.contains("page.html"));
++ assertTrue(hasPageLink, "Page links should be extracted");
++ }
++
++ private Outlink[] parseOutlinks(String html) {
++ return parseOutlinks(html, "http://example.com/");
++ }
++
++ private Outlink[] parseOutlinks(String html, String baseUrl) {
++ Content content = new Content(baseUrl, baseUrl, html.getBytes(),
++ "text/html", new Metadata(), conf);
++ Parse parse = parser.getParse(content).get(baseUrl);
++ return parse.getData().getOutlinks();
++ }
++}
diff --cc src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestMetadataExtraction.java
index 000000000,000000000..04c0ace6c
new file mode 100644
--- /dev/null
+++ b/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestMetadataExtraction.java
@@@ -1,0 -1,0 +1,223 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one or more
++ * contributor license agreements. See the NOTICE file distributed with
++ * this work for additional information regarding copyright ownership.
++ * The ASF licenses this file to You under the Apache License, Version 2.0
++ * (the "License"); you may not use this file except in compliance with
++ * the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++package org.apache.nutch.parse.tika;
++
++import org.apache.hadoop.conf.Configuration;
++import org.apache.nutch.metadata.Metadata;
++import org.apache.nutch.parse.Parse;
++import org.apache.nutch.protocol.Content;
++import org.apache.nutch.util.NutchConfiguration;
++import org.junit.jupiter.api.BeforeEach;
++import org.junit.jupiter.api.Test;
++
++import static org.junit.jupiter.api.Assertions.*;
++
++/**
++ * Tests for metadata extraction from HTML documents.
++ * These tests help ensure metadata extraction works correctly across Tika upgrades.
++ */
++public class TestMetadataExtraction {
++
++ private Configuration conf;
++ private TikaParser parser;
++
++ @BeforeEach
++ public void setup() {
++ conf = NutchConfiguration.create();
++ conf.set("plugin.includes", "parse-tika");
++ parser = new TikaParser();
++ parser.setConf(conf);
++ }
++
++ @Test
++ public void testBasicMetaTags() {
++ String html = "<!DOCTYPE html><html><head>"
++ + "<title>Page Title</title>"
++ + "<meta name='description' content='Page description here'>"
++ + "<meta name='keywords' content='keyword1, keyword2, keyword3'>"
++ + "<meta name='author' content='Test Author'>"
++ + "</head><body>Content</body></html>";
++
++ Parse parse = doParse(html);
++ org.apache.nutch.metadata.Metadata parseMeta = parse.getData().getParseMeta();
++
++ assertEquals("Page Title", parse.getData().getTitle());
++ assertEquals("Page description here", parseMeta.get("description"));
++ assertEquals("keyword1, keyword2, keyword3", parseMeta.get("keywords"));
++ assertEquals("Test Author", parseMeta.get("author"));
++ }
++
++ @Test
++ public void testOpenGraphMetaTags() {
++ String html = "<!DOCTYPE html><html><head>"
++ + "<title>Regular Title</title>"
++ + "<meta property='og:title' content='OG Title'>"
++ + "<meta property='og:description' content='OG Description'>"
++ + "<meta property='og:type' content='article'>"
++ + "<meta property='og:url' content='http://example.com/page'>"
++ + "</head><body>Content</body></html>";
++
++ Parse parse = doParse(html);
++ org.apache.nutch.metadata.Metadata parseMeta = parse.getData().getParseMeta();
++
++ assertEquals("Regular Title", parse.getData().getTitle());
++ // OG tags should be in metadata
++ assertNotNull(parseMeta.get("og:title"));
++ }
++
++ @Test
++ public void testTwitterCardMetaTags() {
++ String html = "<!DOCTYPE html><html><head>"
++ + "<title>Page Title</title>"
++ + "<meta name='twitter:card' content='summary'>"
++ + "<meta name='twitter:title' content='Twitter Title'>"
++ + "<meta name='twitter:description' content='Twitter Description'>"
++ + "</head><body>Content</body></html>";
++
++ Parse parse = doParse(html);
++ org.apache.nutch.metadata.Metadata parseMeta = parse.getData().getParseMeta();
++
++ assertEquals("Page Title", parse.getData().getTitle());
++ // Twitter tags should be in metadata
++ assertNotNull(parseMeta.get("twitter:card"));
++ }
++
++ @Test
++ public void testRobotsMetaNoIndex() {
++ String html = "<!DOCTYPE html><html><head>"
++ + "<meta name='robots' content='noindex'>"
++ + "<title>No Index Page</title>"
++ + "</head><body>This should not be indexed</body></html>";
++
++ Parse parse = doParse(html);
++
++ // With noindex, text should be empty
++ assertTrue(parse.getText().isEmpty() || parse.getText().isBlank(),
++ "noindex should prevent text extraction");
++ }
++
++ @Test
++ public void testRobotsMetaNoFollow() {
++ String html = "<!DOCTYPE html><html><head>"
++ + "<meta name='robots' content='nofollow'>"
++ + "</head><body>"
++ + "<a href='http://example.com/link1'>Link 1</a>"
++ + "<a href='http://example.com/link2'>Link 2</a>"
++ + "</body></html>";
++
++ Parse parse = doParse(html);
++
++ // With nofollow, outlinks should be empty
++ assertEquals(0, parse.getData().getOutlinks().length,
++ "nofollow should prevent outlink extraction");
++ }
++
++ @Test
++ public void testRobotsMetaNoIndexNoFollow() {
++ String html = "<!DOCTYPE html><html><head>"
++ + "<meta name='robots' content='noindex, nofollow'>"
++ + "<title>Blocked Page</title>"
++ + "</head><body>"
++ + "<p>Content here</p>"
++ + "<a href='http://example.com/link'>Link</a>"
++ + "</body></html>";
++
++ Parse parse = doParse(html);
++
++ // Both should be blocked
++ assertTrue(parse.getText().isEmpty() || parse.getText().isBlank(),
++ "noindex should prevent text extraction");
++ assertEquals(0, parse.getData().getOutlinks().length,
++ "nofollow should prevent outlink extraction");
++ }
++
++ @Test
++ public void testRefreshMetaTag() {
++ String html = "<!DOCTYPE html><html><head>"
++ + "<meta http-equiv='refresh' content='5;url=http://example.com/newpage'>"
++ + "<title>Redirect Page</title>"
++ + "</head><body>Redirecting...</body></html>";
++
++ Parse parse = doParse(html);
++
++ // Parse should still succeed
++ assertNotNull(parse);
++ assertEquals("Redirect Page", parse.getData().getTitle());
++ }
++
++ @Test
++ public void testBaseHref() {
++ String html = "<!DOCTYPE html><html><head>"
++ + "<base href='http://other.example.com/'>"
++ + "<title>Base HREF Test</title>"
++ + "</head><body>"
++ + "<a href='page.html'>Relative Link</a>"
++ + "</body></html>";
++
++ String url = "http://example.com/";
++ Content content = new Content(url, url, html.getBytes(),
++ "text/html", new Metadata(), conf);
++ Parse parse = parser.getParse(content).get(url);
++
++ // Link should be resolved relative to base href
++ if (parse.getData().getOutlinks().length > 0) {
++ String linkUrl = parse.getData().getOutlinks()[0].getToUrl();
++ assertTrue(linkUrl.contains("other.example.com") || linkUrl.contains("example.com"),
++ "Link should be resolved using base href or original URL");
++ }
++ }
++
++ @Test
++ public void testMultipleMetaTagsWithSameName() {
++ String html = "<!DOCTYPE html><html><head>"
++ + "<title>Multi Meta</title>"
++ + "<meta name='keywords' content='first'>"
++ + "<meta name='keywords' content='second'>"
++ + "</head><body>Content</body></html>";
++
++ Parse parse = doParse(html);
++ org.apache.nutch.metadata.Metadata parseMeta = parse.getData().getParseMeta();
++
++ // Should handle multiple meta tags with same name
++ String keywords = parseMeta.get("keywords");
++ assertNotNull(keywords, "Keywords should be extracted");
++ }
++
++ @Test
++ public void testEmptyMetaTags() {
++ String html = "<!DOCTYPE html><html><head>"
++ + "<title></title>"
++ + "<meta name='description' content=''>"
++ + "<meta name='keywords'>"
++ + "</head><body>Content</body></html>";
++
++ Parse parse = doParse(html);
++
++ // Should handle empty meta tags gracefully
++ assertNotNull(parse);
++ // Empty title should result in empty string
++ assertTrue(parse.getData().getTitle() == null ||
++ parse.getData().getTitle().isEmpty());
++ }
++
++ private Parse doParse(String html) {
++ String url = "http://example.com/";
++ Content content = new Content(url, url, html.getBytes(),
++ "text/html", new Metadata(), conf);
++ return parser.getParse(content).get(url);
++ }
++}
diff --cc src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestParserFailureHandling.java
index 000000000,000000000..2245db7a5
new file mode 100644
--- /dev/null
+++ b/src/plugin/parse-tika/src/test/org/apache/nutch/parse/tika/TestParserFailureHandling.java
@@@ -1,0 -1,0 +1,221 @@@
++/*
++ * Licensed to the Apache Software Foundation (ASF) under one or more
++ * contributor license agreements. See the NOTICE file distributed with
++ * this work for additional information regarding copyright ownership.
++ * The ASF licenses this file to You under the Apache License, Version 2.0
++ * (the "License"); you may not use this file except in compliance with
++ * the License. You may obtain a copy of the License at
++ *
++ * http://www.apache.org/licenses/LICENSE-2.0
++ *
++ * Unless required by applicable law or agreed to in writing, software
++ * distributed under the License is distributed on an "AS IS" BASIS,
++ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
++ * See the License for the specific language governing permissions and
++ * limitations under the License.
++ */
++package org.apache.nutch.parse.tika;
++
++import org.apache.hadoop.conf.Configuration;
++import org.apache.nutch.metadata.Metadata;
++import org.apache.nutch.parse.ParseResult;
++import org.apache.nutch.protocol.Content;
++import org.apache.nutch.util.NutchConfiguration;
++import org.junit.jupiter.api.BeforeEach;
++import org.junit.jupiter.api.Test;
++
++import static org.junit.jupiter.api.Assertions.*;
++
++/**
++ * Tests for parser failure handling and graceful degradation.
++ * These tests ensure the parser handles edge cases without throwing exceptions.
++ */
++public class TestParserFailureHandling {
++
++ private Configuration conf;
++ private TikaParser parser;
++
++ @BeforeEach
++ public void setup() {
++ conf = NutchConfiguration.create();
++ conf.set("plugin.includes", "parse-tika");
++ parser = new TikaParser();
++ parser.setConf(conf);
++ }
++
++ @Test
++ public void testMalformedHtml() {
++ // Severely malformed HTML
++ String html = "<<<>>>not really html at all{{{";
++
++ ParseResult result = doParse(html, "text/html");
++
++ // Should return a result even for malformed input (not throw exception)
++ assertNotNull(result, "Should return a result even for malformed input");
++ }
++
++ @Test
++ public void testEmptyContent() {
++ ParseResult result = doParse("", "text/html");
++
++ // Should handle empty content gracefully
++ assertNotNull(result, "Should handle empty content");
++ }
++
++ @Test
++ public void testNullBytes() {
++ String url = "http://example.com/";
++ Content content = new Content(url, url, new byte[0],
++ "text/html", new Metadata(), conf);
++
++ ParseResult result = parser.getParse(content);
++
++ // Should handle zero-length byte array
++ assertNotNull(result, "Should handle zero-length content");
++ }
++
++ @Test
++ public void testBinaryContentAsHtml() {
++ // Random binary content declared as HTML
++ byte[] binaryContent = new byte[]{
++ 0x00, 0x01, 0x02, (byte)0xFF, (byte)0xFE, 0x50, 0x4B, 0x03, 0x04
++ };
++
++ String url = "http://example.com/";
++ Content content = new Content(url, url, binaryContent,
++ "text/html", new Metadata(), conf);
++
++ // Should not throw exception
++ ParseResult result = parser.getParse(content);
++ assertNotNull(result, "Should handle binary content declared as HTML");
++ }
++
++ @Test
++ public void testUnknownMimeType() {
++ String url = "http://example.com/file.xyz";
++ Content content = new Content(url, url, "some content".getBytes(),
++ "application/x-unknown-type", new Metadata(), conf);
++
++ ParseResult result = parser.getParse(content);
++
++ // Should handle gracefully (may return failed status but not throw)
++ assertNotNull(result, "Should handle unknown MIME type");
++ }
++
++ @Test
++ public void testVeryLongTitle() {
++ StringBuilder sb = new StringBuilder();
++ sb.append("<!DOCTYPE html><html><head><title>");
++ for (int i = 0; i < 10000; i++) {
++ sb.append("VeryLongTitle");
++ }
++ sb.append("</title></head><body>Content</body></html>");
++
++ ParseResult result = doParse(sb.toString(), "text/html");
++
++ assertNotNull(result, "Should handle very long titles");
++ }
++
++ @Test
++ public void testDeeplyNestedHtml() {
++ StringBuilder sb = new StringBuilder();
++ sb.append("<!DOCTYPE html><html><body>");
++ // Create deeply nested divs
++ for (int i = 0; i < 500; i++) {
++ sb.append("<div>");
++ }
++ sb.append("Deep content");
++ for (int i = 0; i < 500; i++) {
++ sb.append("</div>");
++ }
++ sb.append("</body></html>");
++
++ ParseResult result = doParse(sb.toString(), "text/html");
++
++ assertNotNull(result, "Should handle deeply nested HTML");
++ }
++
++ @Test
++ public void testManyLinks() {
++ StringBuilder sb = new StringBuilder();
++ sb.append("<!DOCTYPE html><html><body>");
++ // Create many links
++ for (int i = 0; i < 1000; i++) {
++ sb.append("<a href='http://example.com/page").append(i).append("'>Link ").append(i).append("</a>");
++ }
++ sb.append("</body></html>");
++
++ ParseResult result = doParse(sb.toString(), "text/html");
++
++ assertNotNull(result, "Should handle many links");
++ }
++
++ @Test
++ public void testSpecialCharactersInUrls() {
++ String html = "<!DOCTYPE html><html><body>"
++ + "<a href='http://example.com/path?q=hello world&foo=bar'>Link 1</a>"
++ + "<a href='http://example.com/path?q=hello%20world'>Link 2</a>"
++ + "<a href='http://example.com/path#section'>Link 3</a>"
++ + "<a href='http://example.com/路径'>Link 4</a>"
++ + "</body></html>";
++
++ ParseResult result = doParse(html, "text/html");
++
++ assertNotNull(result, "Should handle special characters in URLs");
++ }
++
++ @Test
++ public void testNullCharactersInContent() {
++ // HTML with null characters embedded
++ byte[] htmlBytes = "<!DOCTYPE html><html><body>Before\0After</body></html>".getBytes();
++
++ String url = "http://example.com/";
++ Content content = new Content(url, url, htmlBytes,
++ "text/html", new Metadata(), conf);
++
++ ParseResult result = parser.getParse(content);
++
++ assertNotNull(result, "Should handle null characters in content");
++ }
++
++ @Test
++ public void testIncompleteHtml() {
++ // HTML that cuts off mid-tag
++ String html = "<!DOCTYPE html><html><head><title>Test</title></head><body><div>Content<a href='http://";
++
++ ParseResult result = doParse(html, "text/html");
++
++ assertNotNull(result, "Should handle incomplete HTML");
++ }
++
++ @Test
++ public void testHtmlWithScriptErrors() {
++ String html = "<!DOCTYPE html><html><head>"
++ + "<script>this is { not valid javascript</script>"
++ + "<title>Script Error</title>"
++ + "</head><body>Content</body></html>";
++
++ ParseResult result = doParse(html, "text/html");
++
++ assertNotNull(result, "Should handle invalid JavaScript in script tags");
++ }
++
++ @Test
++ public void testHtmlWithCssErrors() {
++ String html = "<!DOCTYPE html><html><head>"
++ + "<style>this is { not: valid; css }</style>"
++ + "<title>CSS Error</title>"
++ + "</head><body>Content</body></html>";
++
++ ParseResult result = doParse(html, "text/html");
++
++ assertNotNull(result, "Should handle invalid CSS in style tags");
++ }
++
++ private ParseResult doParse(String content, String mimeType) {
++ String url = "http://example.com/";
++ Content c = new Content(url, url, content.getBytes(),
++ mimeType, new Metadata(), conf);
++ return parser.getParse(c);
++ }
++}