Apache Nutch is an extensible and scalable web crawler
NUTCH-3145 Upgrade to JUnit 6 (#883)
Lewis John McGibbneyceb23e111caa
diff --git a/.github/workflows/master-build.yml b/.github/workflows/master-build.ymlindex d73bb3a69..1fe9da252 100644--- a/.github/workflows/master-build.yml+++ b/.github/workflows/master-build.yml@@ -20,6 +20,16 @@ on:pull_request:types: [opened, synchronize, reopened]branches: [master]++# Java Version Strategy:+# - BUILD: Requires Java 17+ (JUnit 6 dependency)+# - RUNTIME: Supports Java 11+ (javac.version=11 produces Java 11 bytecode)+#+# The 'build' job verifies bytecode compilation for both Java 11 and 17 targets.+# The 'runtime-java11' job verifies the built artifacts actually run on Java 11.+# The 'tests' job runs on JDK 17 (required by JUnit 6) with the default+# javac.version=11 bytecode target for backward compatibility.+jobs:javadoc:strategy:@@ -43,6 +53,7 @@ jobs:${{ runner.os }}-ivy-- name: Javadocrun: ant clean javadoc -buildfile build.xml+rat:strategy:matrix:@@ -73,19 +84,108 @@ jobs:- name: Fail if any unknown licensesif: ${{ env.UNKNOWN_LICENSES != '0 Unknown Licenses' }}run: exit 1++ # Build verification with Java bytecode target matrix+ # Verifies bytecode compatibility for both Java 11 and Java 17 targets+ build:+ strategy:+ fail-fast: false+ matrix:+ javac-version: ['11', '17']+ os: [ubuntu-latest]+ runs-on: ${{ matrix.os }}+ name: build (javac.version=${{ matrix.javac-version }})+ steps:+ - uses: actions/checkout@v5+ - name: Set up JDK 17+ uses: actions/setup-java@v5+ with:+ java-version: '17'+ distribution: 'temurin'+ - name: Cache Ivy dependencies+ uses: actions/cache@v4+ with:+ path: ~/.ivy2/cache+ key: ${{ runner.os }}-ivy-${{ hashFiles('ivy/ivy.xml', 'src/plugin/**/ivy.xml') }}+ restore-keys: |+ ${{ runner.os }}-ivy-+ - name: Build with javac.version=${{ matrix.javac-version }}+ run: ant clean runtime -Djavac.version=${{ matrix.javac-version }} -buildfile build.xml+ - name: Verify bytecode version+ run: |+ # Extract and verify the bytecode version of compiled classes+ # Java 11 = major version 55, Java 17 = major version 61+ EXPECTED_VERSION=${{ matrix.javac-version == '11' && '55' || '61' }}+ echo "Expected major version: $EXPECTED_VERSION (Java ${{ matrix.javac-version }})"++ # Find a real class file (exclude package-info.class which may have different version)+ cd build/classes+ CLASS_FILE=$(find . -name "*.class" ! -name "package-info.class" | head -1)+ if [ -n "$CLASS_FILE" ]; then+ echo "Checking: $CLASS_FILE"+ ACTUAL_VERSION=$(javap -verbose "$CLASS_FILE" 2>/dev/null | grep "major version" | awk '{print $NF}')+ echo "Actual major version: $ACTUAL_VERSION"+ if [ "$ACTUAL_VERSION" != "$EXPECTED_VERSION" ]; then+ echo "ERROR: Bytecode version mismatch!"+ exit 1+ fi+ echo "Bytecode version verified successfully"+ else+ echo "ERROR: No class files found"+ exit 1+ fi++ # Verify runtime compatibility on Java 11+ # This ensures the built artifacts can actually run on Java 11+ runtime-java11:+ needs: build+ runs-on: ubuntu-latest+ steps:+ - uses: actions/checkout@v5+ - name: Set up JDK 17 for building+ uses: actions/setup-java@v5+ with:+ java-version: '17'+ distribution: 'temurin'+ - name: Cache Ivy dependencies+ uses: actions/cache@v4+ with:+ path: ~/.ivy2/cache+ key: ${{ runner.os }}-ivy-${{ hashFiles('ivy/ivy.xml', 'src/plugin/**/ivy.xml') }}+ restore-keys: |+ ${{ runner.os }}-ivy-+ - name: Build with Java 11 target+ run: ant clean runtime -Djavac.version=11 -buildfile build.xml+ - name: Set up JDK 11 for runtime verification+ uses: actions/setup-java@v5+ with:+ java-version: '11'+ distribution: 'temurin'+ - name: Verify runtime on Java 11+ run: |+ echo "Verifying Nutch can run on Java 11..."+ java -version+ cd runtime/local+ # Actually load Java classes by running showproperties+ # This invokes org.apache.nutch.tools.ShowProperties and verifies the JAR loads+ bin/nutch showproperties | head -20+ echo "Java 11 runtime verification complete"++ # Tests run on JDK 17 (required by JUnit 6) with default javac.version=11+ # Java 11 runtime compatibility is verified by the runtime-java11 jobtests:strategy:+ fail-fast: falsematrix:- java: ['17']os: [ubuntu-latest, macos-latest]runs-on: ${{ matrix.os }}timeout-minutes: 45steps:- uses: actions/checkout@v5- - name: Set up JDK ${{ matrix.java }}+ - name: Set up JDK 17uses: actions/setup-java@v5with:- java-version: ${{ matrix.java }}+ java-version: '17'distribution: 'temurin'- name: Cache Ivy dependenciesuses: actions/cache@v4@@ -139,4 +239,4 @@ jobs:path: |./build/test/TEST-*.xml./build/**/test/TEST-*.xml- retention-days: 1\ No newline at end of file+ retention-days: 1diff --git a/build.xml b/build.xmlindex 38e549797..277225d24 100644--- a/build.xml+++ b/build.xml@@ -50,9 +50,19 @@<property name="ant-eclipse.jar" value="${ivy.dir}/lib/ant-eclipse-1.0-jvm1.2.jar" />- <condition property="using.jdk.11">- <matches string="${java.version}" pattern="11.+" casesensitive="false" />- </condition>+ <!--+ Java Version Strategy (see HADOOP-18887 for similar approach):++ BUILD REQUIREMENT: Java 17+ is required to build Nutch and run unit tests+ because JUnit 6 (Jupiter) requires Java 17+.++ RUNTIME COMPATIBILITY: The compiled bytecode targets Java 11 by default+ (javac.version=11 in default.properties), allowing the binary package+ to run on Java 11+ environments. This is important for Hadoop clusters+ that may not yet support Java 17 runtime.++ To build with Java 17 bytecode target: ant -Djavac.version=17 ...+ --><!-- the normal classpath --><path id="classpath">@@ -201,7 +211,6 @@otherwise the Javascript search is broken,see https://bugs.openjdk.org/browse/JDK-8215291-->- <arg value="--no-module-directories" if:set="using.jdk.11"/><packageset dir="${src.dir}"/><packageset dir="${plugins.dir}/creativecommons/src/java"/>@@ -501,6 +510,7 @@<sysproperty key="test.build.data" value="${test.build.data}"/><sysproperty key="test.src.dir" value="${test.src.dir}"/><sysproperty key="test.include.slow" value="${test.include.slow}"/>+ <sysproperty key="junit.platform.execution.failfast.enabled" value="${test.failfast}"/><sysproperty key="javax.xml.parsers.DocumentBuilderFactory" value="com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"/></fork><fileset dir="${test.build.classes}">@@ -516,6 +526,7 @@<sysproperty key="test.build.data" value="${test.build.data}"/><sysproperty key="test.src.dir" value="${test.src.dir}"/><sysproperty key="test.include.slow" value="${test.include.slow}"/>+ <sysproperty key="junit.platform.execution.failfast.enabled" value="${test.failfast}"/><sysproperty key="javax.xml.parsers.DocumentBuilderFactory" value="com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"/></fork><fileset dir="${test.build.classes}">@@ -653,7 +664,6 @@otherwise the Javascript search is broken,see https://bugs.openjdk.org/browse/JDK-8215291-->- <arg value="--no-module-directories" if:set="using.jdk.11"/><packageset dir="${src.dir}"/><packageset dir="${plugins.dir}/creativecommons/src/java"/>diff --git a/default.properties b/default.propertiesindex 09bfa5594..68a9b304d 100644--- a/default.properties+++ b/default.properties@@ -39,10 +39,14 @@ test.build.data = ${test.build.dir}/datatest.build.classes = ${test.build.dir}/classestest.build.javadoc = ${test.build.dir}/docs/api+# JUnit 6 fail-fast mode - stop on first test failure (true/false)+# Enable with: ant test -Dtest.failfast=true+test.failfast = false+# Proxy Host and Port to use for building JavaDocjavadoc.proxy.host=-J-DproxyHost=javadoc.proxy.port=-J-DproxyPort=-javadoc.link.java=https://docs.oracle.com/en/java/javase/11/docs/api/+javadoc.link.java=https://docs.oracle.com/en/java/javase/17/docs/api/javadoc.link.hadoop=https://hadoop.apache.org/docs/r3.4.2/api/javadoc.packages=org.apache.nutch.*@@ -53,6 +57,12 @@ bin.dist.version.dir=${dist.dir}/${final.name}-binjavac.debug=onjavac.optimize=onjavac.deprecation=on++# Java bytecode target version for compiled classes.+# Set to 11 for backward-compatible runtime (works on Java 11+).+# Note: Building and running tests requires Java 17+ (JUnit 6 requirement),+# but the compiled artifacts will run on Java 11+.+# Override with: ant -Djavac.version=17 to target Java 17 bytecode.javac.version=11runtime.dir=./runtimediff --git a/docker/Dockerfile b/docker/Dockerfileindex 2eb218bad..93985f228 100644--- a/docker/Dockerfile+++ b/docker/Dockerfile@@ -42,12 +42,12 @@ WORKDIR /root/# Install dependenciesRUN apk update-RUN apk --no-cache add apache-ant bash git openjdk11 supervisor+RUN apk --no-cache add apache-ant bash git openjdk17 supervisor# Establish environment variables-RUN echo 'export JAVA_HOME=/usr/lib/jvm/java-11-openjdk' >> $HOME/.bashrc-RUN echo 'export JAVA_HOME=/usr/lib/jvm/java-11-openjdk' >> $HOME/.ashrc-ENV JAVA_HOME='/usr/lib/jvm/java-11-openjdk'+RUN echo 'export JAVA_HOME=/usr/lib/jvm/java-17-openjdk' >> $HOME/.bashrc+RUN echo 'export JAVA_HOME=/usr/lib/jvm/java-17-openjdk' >> $HOME/.ashrc+ENV JAVA_HOME='/usr/lib/jvm/java-17-openjdk'ENV NUTCH_HOME='/root/nutch_source/runtime/local'# Checkout and build the Nutch master branch (1.x)diff --git a/ivy/ivy.xml b/ivy/ivy.xmlindex 9b38d2fa9..06e269bf5 100644--- a/ivy/ivy.xml+++ b/ivy/ivy.xml@@ -116,15 +116,17 @@<exclude org="org.gnu.inet" module="libidn" /><!-- LGPL 2.1 --></dependency>- <dependency org="org.hamcrest" name="hamcrest" rev="3.0" conf="test->default"/>- <!-- Required for <junitlauncher> task -->- <dependency org="org.junit.platform" name="junit-platform-launcher" rev="1.14.1" conf="test->default"/>- <!-- Required for JUnit 5 (Jupiter) test execution -->- <dependency org="org.junit.jupiter" name="junit-jupiter-engine" rev="5.14.1" conf="test->default"/>- <dependency org="org.junit.jupiter" name="junit-jupiter-api" rev="5.14.1" conf="test->default"/>- <!-- Mockito for mocking in tests -->- <dependency org="org.mockito" name="mockito-core" rev="5.18.0" conf="test->default"/>- <dependency org="org.mockito" name="mockito-junit-jupiter" rev="5.18.0" conf="test->default"/>+ <dependency org="org.hamcrest" name="hamcrest" rev="3.0" conf="test->default"/>+ <!-- JSpecify nullability annotations for improved null safety -->+ <dependency org="org.jspecify" name="jspecify" rev="1.0.0" conf="*->default"/>+ <!-- Required for <junitlauncher> task -->+ <dependency org="org.junit.platform" name="junit-platform-launcher" rev="6.0.3" conf="test->default"/>+ <!-- Required for JUnit 6 (Jupiter) test execution -->+ <dependency org="org.junit.jupiter" name="junit-jupiter-engine" rev="6.0.3" conf="test->default"/>+ <dependency org="org.junit.jupiter" name="junit-jupiter-api" rev="6.0.3" conf="test->default"/>+ <!-- Mockito for mocking in tests -->+ <dependency org="org.mockito" name="mockito-core" rev="5.18.0" conf="test->default"/>+ <dependency org="org.mockito" name="mockito-junit-jupiter" rev="5.18.0" conf="test->default"/><!-- Jetty used to serve test pages for unit tests, but is also provided as dependency of Hadoop --><dependency org="org.eclipse.jetty" name="jetty-server" rev="12.1.5" conf="test->default">diff --git a/src/plugin/build-plugin.xml b/src/plugin/build-plugin.xmlindex b0aca7103..f1787ed03 100755--- a/src/plugin/build-plugin.xml+++ b/src/plugin/build-plugin.xml@@ -218,6 +218,7 @@<jvmarg value="-Xmx1000m"/><sysproperty key="test.data" value="${build.test}/data"/><sysproperty key="test.input" value="${root}/data"/>+ <sysproperty key="junit.platform.execution.failfast.enabled" value="${test.failfast}"/><sysproperty key="javax.xml.parsers.DocumentBuilderFactory" value="com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"/></fork><fileset dir="${build.test}">@@ -232,6 +233,7 @@<jvmarg value="-Xmx1000m"/><sysproperty key="test.data" value="${build.test}/data"/><sysproperty key="test.input" value="${root}/data"/>+ <sysproperty key="junit.platform.execution.failfast.enabled" value="${test.failfast}"/><sysproperty key="javax.xml.parsers.DocumentBuilderFactory" value="com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"/></fork><fileset dir="${basedir}">diff --git a/src/test/junit-platform.properties b/src/test/junit-platform.propertiesnew file mode 100644index 000000000..b2f5ccaea--- /dev/null+++ b/src/test/junit-platform.properties@@ -0,0 +1,31 @@+# Licensed to the Apache Software Foundation (ASF) under one or more+# contributor license agreements. See the NOTICE file distributed with+# this work for additional information regarding copyright ownership.+# The ASF licenses this file to You under the Apache License, Version 2.0+# (the "License"); you may not use this file except in compliance with+# the License. You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.++# JUnit Platform Configuration+# See: https://junit.org/junit5/docs/current/user-guide/#running-tests-config-params++# Enable fail-fast mode - stops test execution on first failure+# This provides faster feedback during development+# Set to false or remove this line to run all tests even if some fail+junit.platform.execution.failfast.enabled=false++# Display names for tests - use method names by default+junit.jupiter.displayname.generator.default=org.junit.jupiter.api.DisplayNameGenerator$Standard++# Timeout configuration for individual tests (can be overridden with @Timeout)+# junit.jupiter.execution.timeout.default=5m++# Parallel execution configuration (disabled by default for deterministic results)+junit.jupiter.execution.parallel.enabled=falsediff --git a/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java b/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.javaindex 9e96071a0..581b528f3 100644--- a/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java+++ b/src/test/org/apache/nutch/crawl/CrawlDBTestUtil.java@@ -24,6 +24,8 @@ import java.util.ArrayList;import java.util.Iterator;import java.util.List;+import org.jspecify.annotations.NonNull;+import org.jspecify.annotations.Nullable;import org.slf4j.Logger;import org.slf4j.LoggerFactory;import org.apache.hadoop.conf.Configuration;@@ -53,6 +55,10 @@ import org.eclipse.jetty.server.ServerConnector;import org.eclipse.jetty.server.handler.ContextHandler;import org.eclipse.jetty.server.handler.ResourceHandler;+/**+ * Test utility for creating and manipulating CrawlDb instances.+ * Uses JSpecify annotations for null safety.+ */public class CrawlDBTestUtil {private static final Logger LOG = LoggerFactory@@ -62,6 +68,8 @@ public class CrawlDBTestUtil {/*** Creates synthetic crawldb*+ * @param conf+ * configuration to use* @param fs* filesystem where db will be created* @param crawldb@@ -70,8 +78,8 @@ public class CrawlDBTestUtil {* urls to be inserted, objects are of type URLCrawlDatum* @throws Exception*/- public static void createCrawlDb(Configuration conf, FileSystem fs,- Path crawldb, List<URLCrawlDatum> init) throws Exception {+ public static void createCrawlDb(@NonNull Configuration conf, @NonNull FileSystem fs,+ @NonNull Path crawldb, @NonNull List<URLCrawlDatum> init) throws Exception {LOG.trace("* creating crawldb: {}", crawldb);Path dir = new Path(crawldb, CrawlDb.CURRENT_NAME);Option wKeyOpt = MapFile.Writer.keyClass(Text.class);@@ -366,8 +374,9 @@ public class CrawlDBTestUtil {* override the default one and it is currently not possible to use* dynamically set values.*- * @return+ * @return a new Reducer Context with test configuration*/+ @NonNullpublic static Reducer<Text, CrawlDatum, Text, CrawlDatum>.Context createContext() {DummyContext context = new DummyContext();Configuration conf = context.getConfiguration();@@ -376,13 +385,16 @@ public class CrawlDBTestUtil {return (Reducer<Text, CrawlDatum, Text, CrawlDatum>.Context) context;}+ /** Container for URL and CrawlDatum pairs used in test data. */public static class URLCrawlDatum {+ @NonNullpublic Text url;+ @NonNullpublic CrawlDatum datum;- public URLCrawlDatum(Text url, CrawlDatum datum) {+ public URLCrawlDatum(@NonNull Text url, @NonNull CrawlDatum datum) {this.url = url;this.datum = datum;}@@ -391,20 +403,27 @@ public class CrawlDBTestUtil {/*** Generate seedlist*+ * @param fs filesystem to use+ * @param urlPath path where seed file will be created+ * @param urls list of URLs to write* @throws IOException*/- public static void generateSeedList(FileSystem fs, Path urlPath,- List<String> urls) throws IOException {+ public static void generateSeedList(@NonNull FileSystem fs, @NonNull Path urlPath,+ @NonNull List<String> urls) throws IOException {generateSeedList(fs, urlPath, urls, new ArrayList<String>());}/**- * Generate seedlist+ * Generate seedlist with optional metadata*+ * @param fs filesystem to use+ * @param urlPath path where seed file will be created+ * @param urls list of URLs to write+ * @param metadata optional metadata for each URL* @throws IOException*/- public static void generateSeedList(FileSystem fs, Path urlPath,- List<String> urls, List<String> metadata) throws IOException {+ public static void generateSeedList(@NonNull FileSystem fs, @NonNull Path urlPath,+ @NonNull List<String> urls, @NonNull List<String> metadata) throws IOException {FSDataOutputStream out;Path file = new Path(urlPath, "urls.txt");fs.mkdirs(urlPath);@@ -439,9 +458,11 @@ public class CrawlDBTestUtil {* port to listen to* @param staticContent* folder where static content lives+ * @return configured Jetty server instance* @throws UnknownHostException*/- public static Server getServer(int port, String staticContent)+ @NonNull+ public static Server getServer(int port, @NonNull String staticContent)throws UnknownHostException {Server webServer = new Server();diff --git a/src/test/org/apache/nutch/fetcher/TestFetcher.java b/src/test/org/apache/nutch/fetcher/TestFetcher.javaindex f25cab545..176a88a52 100644--- a/src/test/org/apache/nutch/fetcher/TestFetcher.java+++ b/src/test/org/apache/nutch/fetcher/TestFetcher.java@@ -28,14 +28,18 @@ import org.apache.nutch.metadata.Metadata;import org.apache.nutch.metadata.Nutch;import org.apache.nutch.parse.ParseData;import org.apache.nutch.protocol.Content;+import org.apache.nutch.util.CancellationAwareTestUtils;+import org.apache.nutch.util.CancellationAwareTestUtils.CancellationToken;import org.eclipse.jetty.server.Server;import org.junit.jupiter.api.AfterEach;import org.junit.jupiter.api.BeforeEach;import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.Timeout;import java.io.IOException;import java.util.ArrayList;import java.util.Collections;+import java.util.concurrent.TimeUnit;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertTrue;@@ -44,6 +48,8 @@ import static org.junit.jupiter.api.Assertions.assertTrue;* Basic fetcher test 1. generate seedlist 2. inject 3. generate 3. fetch 4.* Verify contents*+ * <p>This test is cancellation-aware and will exit gracefully if the test+ * suite is stopped early (e.g., due to fail-fast mode).</p>*/public class TestFetcher {@@ -81,7 +87,10 @@ public class TestFetcher {}@Test+ @Timeout(value = 5, unit = TimeUnit.MINUTES)public void testFetch() throws IOException, ClassNotFoundException, InterruptedException {+ // Create cancellation token for graceful shutdown support+ CancellationToken cancellationToken = CancellationAwareTestUtils.createToken();// generate seedlistArrayList<String> urls = new ArrayList<String>();@@ -95,15 +104,22 @@ public class TestFetcher {CrawlDBTestUtil.generateSeedList(fs, urlPath, urls);+ // Check for cancellation before long-running operations+ cancellationToken.throwIfCancelled();+// injectInjector injector = new Injector(conf);injector.inject(crawldbPath, urlPath);+ cancellationToken.throwIfCancelled();+// generateGenerator g = new Generator(conf);Path[] generatedSegment = g.generate(crawldbPath, segmentsPath, 1,Long.MAX_VALUE, Long.MAX_VALUE, false, false, false, 1, null);+ cancellationToken.throwIfCancelled();+long time = System.currentTimeMillis();// fetchFetcher fetcher = new Fetcher(conf);@@ -115,6 +131,11 @@ public class TestFetcher {time = System.currentTimeMillis() - time;+ // Skip verification if cancelled+ if (cancellationToken.isCancelled()) {+ return;+ }+// verify politeness, time taken should be more than (num_of_pages +1)*delayint minimumTime = (int) ((urls.size() + 1) * 1000 * conf.getFloat("fetcher.server.delay", 5));@@ -127,18 +148,28 @@ public class TestFetcher {ArrayList<String> handledurls = new ArrayList<String>();- READ_CONTENT: do {- Text key = new Text();- Content value = new Content();- if (!reader.next(key, value))- break READ_CONTENT;- String contentString = new String(value.getContent());- if (contentString.indexOf("Nutch fetcher test page") != -1) {- handledurls.add(key.toString());- }- } while (true);+ try {+ READ_CONTENT: do {+ // Check for cancellation periodically during I/O operations+ if (cancellationToken.isCancelled()) break READ_CONTENT;++ Text key = new Text();+ Content value = new Content();+ if (!reader.next(key, value))+ break READ_CONTENT;+ String contentString = new String(value.getContent());+ if (contentString.indexOf("Nutch fetcher test page") != -1) {+ handledurls.add(key.toString());+ }+ } while (true);+ } finally {+ reader.close();+ }- reader.close();+ // Skip remaining verification if cancelled+ if (cancellationToken.isCancelled()) {+ return;+ }Collections.sort(urls);Collections.sort(handledurls);@@ -157,22 +188,32 @@ public class TestFetcher {new Path(generatedSegment[0], ParseData.DIR_NAME), "part-r-00000/data");reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(parseData));- READ_PARSE_DATA: do {- Text key = new Text();- ParseData value = new ParseData();- if (!reader.next(key, value))- break READ_PARSE_DATA;- // make sure they all contain "nutch.segment.name" and- // "nutch.content.digest"- // keys in parse metadata- Metadata contentMeta = value.getContentMeta();- if (contentMeta.get(Nutch.SEGMENT_NAME_KEY) != null- && contentMeta.get(Nutch.SIGNATURE_KEY) != null) {- handledurls.add(key.toString());- }- } while (true);+ try {+ READ_PARSE_DATA: do {+ // Check for cancellation periodically+ if (cancellationToken.isCancelled()) break READ_PARSE_DATA;++ Text key = new Text();+ ParseData value = new ParseData();+ if (!reader.next(key, value))+ break READ_PARSE_DATA;+ // make sure they all contain "nutch.segment.name" and+ // "nutch.content.digest"+ // keys in parse metadata+ Metadata contentMeta = value.getContentMeta();+ if (contentMeta.get(Nutch.SEGMENT_NAME_KEY) != null+ && contentMeta.get(Nutch.SIGNATURE_KEY) != null) {+ handledurls.add(key.toString());+ }+ } while (true);+ } finally {+ reader.close();+ }- reader.close();+ // Skip final assertions if cancelled+ if (cancellationToken.isCancelled()) {+ return;+ }Collections.sort(handledurls);diff --git a/src/test/org/apache/nutch/segment/TestSegmentMerger.java b/src/test/org/apache/nutch/segment/TestSegmentMerger.javaindex 0df88a2de..9cc076ad6 100644--- a/src/test/org/apache/nutch/segment/TestSegmentMerger.java+++ b/src/test/org/apache/nutch/segment/TestSegmentMerger.java@@ -17,6 +17,7 @@package org.apache.nutch.segment;import java.text.DecimalFormat;+import java.util.concurrent.TimeUnit;import org.apache.hadoop.conf.Configuration;import org.apache.hadoop.fs.FileStatus;@@ -28,16 +29,23 @@ import org.apache.hadoop.io.SequenceFile;import org.apache.hadoop.io.Text;import org.apache.hadoop.mapreduce.lib.output.MapFileOutputFormat;import org.apache.nutch.parse.ParseText;+import org.apache.nutch.util.CancellationAwareTestUtils;+import org.apache.nutch.util.CancellationAwareTestUtils.CancellationToken;import org.apache.nutch.util.NutchConfiguration;import org.junit.jupiter.api.AfterEach;import org.junit.jupiter.api.BeforeAll;import org.junit.jupiter.api.Test;+import org.junit.jupiter.api.Timeout;import org.junit.jupiter.api.BeforeEach;import static org.junit.jupiter.api.Assertions.assertEquals;import static org.junit.jupiter.api.Assertions.assertTrue;import static org.junit.jupiter.api.Assumptions.assumeTrue;+/**+ * Tests for SegmentMerger functionality.+ * This test is cancellation-aware for graceful shutdown during fail-fast mode.+ */public class TestSegmentMerger {Configuration conf;FileSystem fs;@@ -106,9 +114,19 @@ public class TestSegmentMerger {}@Test+ @Timeout(value = 10, unit = TimeUnit.MINUTES)public void testLargeMerge() throws Exception {+ // Create cancellation token for graceful shutdown support+ CancellationToken cancellationToken = CancellationAwareTestUtils.createToken();+SegmentMerger merger = new SegmentMerger(conf);merger.merge(out, new Path[] { seg1, seg2 }, false, false, -1);++ // Check for cancellation before verification+ if (cancellationToken.isCancelled()) {+ return;+ }+// verify outputFileStatus[] stats = fs.listStatus(out);// there should be just one path@@ -119,20 +137,37 @@ public class TestSegmentMerger {MapFile.Reader[] readers = MapFileOutputFormat.getReaders(new Path(outSeg, ParseText.DIR_NAME), conf);int cnt1 = 0, cnt2 = 0;- for (MapFile.Reader r : readers) {- while (r.next(k, v)) {- String ks = k.toString();- String vs = v.getText();- if (ks.startsWith("seg1-")) {- cnt1++;- assertTrue(vs.startsWith("seg1 "));- } else if (ks.startsWith("seg2-")) {- cnt2++;- assertTrue(vs.startsWith("seg2 "));+ try {+ for (MapFile.Reader r : readers) {+ while (r.next(k, v)) {+ // Check for cancellation periodically during I/O+ if (cancellationToken.isCancelled()) {+ return;+ }++ String ks = k.toString();+ String vs = v.getText();+ if (ks.startsWith("seg1-")) {+ cnt1++;+ assertTrue(vs.startsWith("seg1 "));+ } else if (ks.startsWith("seg2-")) {+ cnt2++;+ assertTrue(vs.startsWith("seg2 "));+ }}}- r.close();+ } finally {+ // Ensure readers are closed even on cancellation+ for (MapFile.Reader r : readers) {+ r.close();+ }+ }++ // Skip final assertions if cancelled+ if (cancellationToken.isCancelled()) {+ return;}+assertEquals(countSeg1, cnt1);assertEquals(countSeg2, cnt2);}diff --git a/src/test/org/apache/nutch/util/CancellationAwareTestUtils.java b/src/test/org/apache/nutch/util/CancellationAwareTestUtils.javanew file mode 100644index 000000000..055a64261--- /dev/null+++ b/src/test/org/apache/nutch/util/CancellationAwareTestUtils.java@@ -0,0 +1,209 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.nutch.util;++import org.jspecify.annotations.NonNull;+import org.jspecify.annotations.Nullable;++import java.util.concurrent.atomic.AtomicBoolean;+import java.util.function.BooleanSupplier;++/**+ * Utility class for making long-running tests cancellation-aware.+ *+ * <p>This supports JUnit 6's fail-fast mode by allowing tests to check+ * for cancellation requests and exit gracefully, ensuring proper resource+ * cleanup even when the test suite is stopped early.</p>+ *+ * <p>Usage example:</p>+ * <pre>{@code+ * @Test+ * @Timeout(value = 5, unit = TimeUnit.MINUTES)+ * void testLongRunningOperation() throws Exception {+ * CancellationAwareTestUtils.CancellationToken token =+ * CancellationAwareTestUtils.createToken();+ *+ * try {+ * while (hasMoreWork() && !token.isCancelled()) {+ * doWork();+ * }+ * } finally {+ * cleanup();+ * }+ * }+ * }</pre>+ */+public class CancellationAwareTestUtils {++ /**+ * A simple cancellation token that can be checked during long-running operations.+ * The token is automatically cancelled when the current thread is interrupted.+ */+ public static class CancellationToken {+ private final AtomicBoolean cancelled = new AtomicBoolean(false);+ @Nullable+ private final BooleanSupplier additionalCheck;++ CancellationToken(@Nullable BooleanSupplier additionalCheck) {+ this.additionalCheck = additionalCheck;+ }++ /**+ * Check if cancellation has been requested.+ * This checks both explicit cancellation and thread interruption.+ *+ * @return true if the operation should be cancelled+ */+ public boolean isCancelled() {+ if (cancelled.get()) {+ return true;+ }+ if (Thread.currentThread().isInterrupted()) {+ cancelled.set(true);+ return true;+ }+ if (additionalCheck != null && additionalCheck.getAsBoolean()) {+ cancelled.set(true);+ return true;+ }+ return false;+ }++ /**+ * Explicitly request cancellation.+ */+ public void cancel() {+ cancelled.set(true);+ }++ /**+ * Throws InterruptedException if cancellation has been requested.+ * Useful for cooperative cancellation in loops.+ *+ * @throws InterruptedException if cancelled+ */+ public void throwIfCancelled() throws InterruptedException {+ if (isCancelled()) {+ throw new InterruptedException("Test cancelled");+ }+ }+ }++ /**+ * Creates a new cancellation token.+ *+ * @return a new CancellationToken instance+ */+ @NonNull+ public static CancellationToken createToken() {+ return new CancellationToken(null);+ }++ /**+ * Creates a cancellation token with an additional cancellation condition.+ *+ * @param additionalCheck additional condition that triggers cancellation+ * @return a new CancellationToken instance+ */+ @NonNull+ public static CancellationToken createToken(@NonNull BooleanSupplier additionalCheck) {+ return new CancellationToken(additionalCheck);+ }++ /**+ * Executes an operation with periodic cancellation checks.+ *+ * @param token the cancellation token to check+ * @param operation the operation to execute (should be short-lived)+ * @param iterations number of times to execute the operation+ * @param checkInterval how often to check for cancellation (every N iterations)+ * @return the number of iterations actually completed+ * @throws InterruptedException if cancelled during execution+ */+ public static int executeWithCancellation(+ @NonNull CancellationToken token,+ @NonNull Runnable operation,+ int iterations,+ int checkInterval) throws InterruptedException {++ int completed = 0;+ for (int i = 0; i < iterations; i++) {+ if (i % checkInterval == 0) {+ token.throwIfCancelled();+ }+ operation.run();+ completed++;+ }+ return completed;+ }++ /**+ * Sleeps for the specified duration while remaining cancellation-aware.+ * Checks for cancellation every 100ms.+ *+ * @param token the cancellation token+ * @param millis total milliseconds to sleep+ * @throws InterruptedException if cancelled or interrupted+ */+ public static void sleepWithCancellation(@NonNull CancellationToken token, long millis)+ throws InterruptedException {+ long remaining = millis;+ while (remaining > 0) {+ token.throwIfCancelled();+ long sleepTime = Math.min(remaining, 100);+ Thread.sleep(sleepTime);+ remaining -= sleepTime;+ }+ }++ /**+ * Interface for operations that can be interrupted and resumed.+ *+ * @param <T> the result type+ */+ @FunctionalInterface+ public interface CancellableOperation<T> {+ /**+ * Execute a portion of the operation.+ *+ * @param token cancellation token to check+ * @return the result, or null if more work is needed+ * @throws Exception if the operation fails+ */+ @Nullable+ T execute(@NonNull CancellationToken token) throws Exception;+ }++ /**+ * Runs a cancellable operation, returning null if cancelled before completion.+ *+ * @param <T> the result type+ * @param operation the operation to run+ * @return the result, or null if cancelled+ * @throws Exception if the operation fails (not due to cancellation)+ */+ @Nullable+ public static <T> T runCancellable(@NonNull CancellableOperation<T> operation) throws Exception {+ CancellationToken token = createToken();+ try {+ return operation.execute(token);+ } catch (InterruptedException e) {+ Thread.currentThread().interrupt();+ return null;+ }+ }+}diff --git a/src/test/org/apache/nutch/util/WritableTestUtils.java b/src/test/org/apache/nutch/util/WritableTestUtils.javaindex d4429dbf3..632849e71 100644--- a/src/test/org/apache/nutch/util/WritableTestUtils.java+++ b/src/test/org/apache/nutch/util/WritableTestUtils.java@@ -18,24 +18,31 @@ package org.apache.nutch.util;import org.apache.hadoop.io.*;import org.apache.hadoop.conf.*;+import org.jspecify.annotations.NonNull;+import org.jspecify.annotations.Nullable;import static org.junit.jupiter.api.Assertions.assertEquals;+/**+ * Utility methods for testing Hadoop Writable implementations.+ * Uses JSpecify annotations for null safety.+ */public class WritableTestUtils {/** Utility method for testing writables. */- public static void testWritable(Writable before) throws Exception {+ public static void testWritable(@NonNull Writable before) throws Exception {testWritable(before, null);}/** Utility method for testing writables. */- public static void testWritable(Writable before, Configuration conf)+ public static void testWritable(@NonNull Writable before, @Nullable Configuration conf)throws Exception {assertEquals(before, writeRead(before, conf));}/** Utility method for testing writables. */- public static Writable writeRead(Writable before, Configuration conf)+ @NonNull+ public static Writable writeRead(@NonNull Writable before, @Nullable Configuration conf)throws Exception {DataOutputBuffer dob = new DataOutputBuffer();