Apache Nutch is an extensible and scalable web crawler
NUTCH-3154 Implement integration testing framework for Nutch IndexWriter plugins using Testcontainers (#895)
Lewis John McGibbney64ac8b46347e
diff --git a/.github/workflows/master-build.yml b/.github/workflows/master-build.ymlindex 1fe9da252..2056c1e10 100644--- a/.github/workflows/master-build.yml+++ b/.github/workflows/master-build.yml@@ -204,6 +204,8 @@ jobs:- 'src/testresources/**'plugins:- 'src/plugin/**'+ indexer_plugins:+ - 'src/plugin/indexer-*/**'buildconf:- 'build.xml'- 'ivy/ivy.xml'@@ -220,6 +222,10 @@ jobs:- name: test pluginsif: ${{ steps.filter.outputs.plugins == 'true' && steps.filter.outputs.core == 'false' && steps.filter.outputs.buildconf == 'false' }}run: ant clean test-plugins -buildfile build.xml+ # run indexer integration tests when indexer plugin files change (Docker required, ubuntu-latest only)+ - name: test indexer integration+ if: ${{ steps.filter.outputs.indexer_plugins == 'true' && matrix.os == 'ubuntu-latest' }}+ run: ant clean test-indexer-integration -buildfile build.xml- name: Check for test resultsid: check_testsif: always() && matrix.os == 'ubuntu-latest'diff --git a/build.xml b/build.xmlindex 277225d24..57d44ee49 100644--- a/build.xml+++ b/build.xml@@ -545,6 +545,10 @@<ant dir="src/plugin" target="test-single" inheritAll="false"/></target>+ <target name="test-indexer-integration" depends="resolve-test, compile, compile-core-test, job" description="--> run indexer plugin integration tests (Testcontainers)">+ <ant dir="src/plugin" target="test-indexer-integration" inheritAll="false"/>+ </target>+<target name="nightly" depends="test, tar-src, zip-src" description="--> run the nightly target build"></target>diff --git a/ivy/ivy.xml b/ivy/ivy.xmlindex 06e269bf5..b6b8f67a9 100644--- a/ivy/ivy.xml+++ b/ivy/ivy.xml@@ -128,6 +128,10 @@<dependency org="org.mockito" name="mockito-core" rev="5.18.0" conf="test->default"/><dependency org="org.mockito" name="mockito-junit-jupiter" rev="5.18.0" conf="test->default"/>+ <!-- Testcontainers for indexer plugin integration tests -->+ <dependency org="org.testcontainers" name="testcontainers" rev="2.0.3" conf="test->default"/>+ <dependency org="org.testcontainers" name="junit-jupiter" rev="1.21.4" conf="test->default"/>+<!-- Jetty used to serve test pages for unit tests, but is also provided as dependency of Hadoop --><dependency org="org.eclipse.jetty" name="jetty-server" rev="12.1.5" conf="test->default"><exclude org="ch.qos.reload4j" module="*" />diff --git a/src/plugin/build-plugin.xml b/src/plugin/build-plugin.xmlindex f1787ed03..ef8dda56c 100755--- a/src/plugin/build-plugin.xml+++ b/src/plugin/build-plugin.xml@@ -189,7 +189,7 @@<!-- ================================================================== --><!-- Compile test code --><!-- ================================================================== -->- <target name="compile-test" depends="compile, deps-test-compile" if="test.available">+ <target name="compile-test" depends="resolve-test, compile, deps-test-compile" if="test.available"><javacencoding="${build.encoding}"srcdir="${src.test}"@@ -242,7 +242,34 @@</testclasses></junitlauncher><fail if="tests.failed">Tests failed!</fail>- </target>+ </target>++ <!-- ================================================================== -->+ <!-- Run indexer plugin integration tests (Testcontainers) -->+ <!-- ================================================================== -->+ <target name="test-indexer-integration" depends="compile-test, deploy" if="test.available">+ <echo message="Running indexer integration tests for plugin: ${name}"/>+ <junitlauncher printSummary="true" haltOnFailure="false" failureProperty="integration.tests.failed">+ <classpath refid="test.classpath"/>+ <testclasses outputDir="${build.test}">+ <listener type="legacy-plain" sendSysOut="true" sendSysErr="true"/>+ <listener type="legacy-xml" sendSysOut="true" sendSysErr="true"/>+ <fork forkMode="perTestClass">+ <jvmarg value="-Xmx2000m"/>+ <sysproperty key="test.data" value="${build.test}/data"/>+ <sysproperty key="test.input" value="${root}/data"/>+ <sysproperty key="testcontainers.reuse.enable" value="true"/>+ <sysproperty key="javax.xml.parsers.DocumentBuilderFactory" value="com.sun.org.apache.xerces.internal.jaxp.DocumentBuilderFactoryImpl"/>+ </fork>+ <fileset dir="${build.test}">+ <include name="**/IT*.class"/>+ <include name="**/*IT.class"/>+ <include name="**/*IntegrationTest.class"/>+ </fileset>+ </testclasses>+ </junitlauncher>+ <fail if="integration.tests.failed">Indexer integration tests failed!</fail>+ </target><!-- target: resolve ================================================= --><target name="resolve-default" depends="clean-lib" description="resolve and retrieve dependencies with ivy">diff --git a/src/plugin/build.xml b/src/plugin/build.xmlindex b7a5a7721..18d00da3b 100755--- a/src/plugin/build.xml+++ b/src/plugin/build.xml@@ -172,6 +172,17 @@<ant dir="${plugin}" target="test"/></target>+ <!-- ====================================================== -->+ <!-- Indexer plugin integration tests (Testcontainers) -->+ <!-- Run sequentially to avoid container resource contention -->+ <!-- ====================================================== -->+ <target name="test-indexer-integration">+ <ant dir="indexer-elastic" target="test-indexer-integration"/>+ <ant dir="indexer-kafka" target="test-indexer-integration"/>+ <ant dir="indexer-rabbit" target="test-indexer-integration"/>+ <ant dir="indexer-solr" target="test-indexer-integration"/>+ </target>+<!-- ====================================================== --><!-- Clean all of the plugins. --><!-- ====================================================== -->diff --git a/src/plugin/indexer-elastic/ivy.xml b/src/plugin/indexer-elastic/ivy.xmlindex ee812a225..04c1a071d 100644--- a/src/plugin/indexer-elastic/ivy.xml+++ b/src/plugin/indexer-elastic/ivy.xml@@ -36,7 +36,10 @@</publications><dependencies>- <dependency org="org.elasticsearch.client" name="elasticsearch-rest-high-level-client" rev="7.10.2"/>+ <dependency org="org.testcontainers" name="testcontainers-elasticsearch" rev="2.0.3" conf="test->default"/>+ <dependency org="org.elasticsearch.client" name="elasticsearch-rest-high-level-client" rev="7.10.2">+ <exclude org="org.apache.logging.log4j" name="*"/>+ </dependency><dependency org="org.apache.lucene" name="lucene-analyzers-common" rev="8.11.2"/><dependency org="org.apache.lucene" name="lucene-backward-codecs" rev="8.11.2"/><dependency org="org.apache.lucene" name="lucene-core" rev="8.11.2"/>diff --git a/src/plugin/indexer-elastic/src/test/org/apache/nutch/indexwriter/elastic/ElasticIndexWriterIT.java b/src/plugin/indexer-elastic/src/test/org/apache/nutch/indexwriter/elastic/ElasticIndexWriterIT.javanew file mode 100644index 000000000..0479213c3--- /dev/null+++ b/src/plugin/indexer-elastic/src/test/org/apache/nutch/indexwriter/elastic/ElasticIndexWriterIT.java@@ -0,0 +1,113 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.nutch.indexwriter.elastic;++import java.util.HashMap;+import java.util.Map;++import org.apache.hadoop.conf.Configuration;+import org.apache.http.HttpHost;+import org.apache.nutch.indexer.AbstractIndexWriterIT;+import org.apache.nutch.indexer.IndexWriter;+import org.apache.nutch.indexer.IndexWriterParams;+import org.apache.nutch.indexer.NutchDocument;+import org.apache.nutch.util.NutchConfiguration;+import org.elasticsearch.action.get.GetRequest;+import org.elasticsearch.action.get.GetResponse;+import org.elasticsearch.client.RequestOptions;+import org.elasticsearch.client.RestClient;+import org.elasticsearch.client.RestHighLevelClient;+import org.junit.jupiter.api.Test;+import org.testcontainers.elasticsearch.ElasticsearchContainer;+import org.testcontainers.junit.jupiter.Container;+import org.testcontainers.junit.jupiter.Testcontainers;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertNotNull;+import static org.junit.jupiter.api.Assertions.assertTrue;++/**+ * Integration tests for ElasticIndexWriter using Testcontainers.+ */+@Testcontainers(disabledWithoutDocker = true)+public class ElasticIndexWriterIT extends AbstractIndexWriterIT {++ private static final String ELASTICSEARCH_IMAGE =+ "docker.elastic.co/elasticsearch/elasticsearch:7.10.2";++ @Container+ private static final ElasticsearchContainer elasticsearchContainer =+ new ElasticsearchContainer(ELASTICSEARCH_IMAGE)+ .withEnv("discovery.type", "single-node")+ .withEnv("xpack.security.enabled", "false");++ private ElasticIndexWriter indexWriter;+ private Configuration conf;++ @Override+ public void setUpIndexWriter() throws Exception {+ conf = NutchConfiguration.create();+ indexWriter = new ElasticIndexWriter();+ indexWriter.setConf(conf);++ Map<String, String> params = new HashMap<>();+ params.put(ElasticConstants.HOSTS, elasticsearchContainer.getHost());+ params.put(ElasticConstants.PORT, String.valueOf(elasticsearchContainer.getMappedPort(9200)));+ params.put(ElasticConstants.INDEX, "test-index");+ params.put(ElasticConstants.SCHEME, "http");++ IndexWriterParams writerParams = new IndexWriterParams(params);+ indexWriter.open(writerParams);+ }++ @Override+ public void tearDownIndexWriter() throws Exception {+ if (indexWriter != null) {+ try {+ indexWriter.close();+ } catch (Exception e) {+ // Ignore if open() failed and close state is invalid+ }+ indexWriter = null;+ }+ }++ @Override+ public IndexWriter getIndexWriter() {+ return indexWriter;+ }++ @Override+ public boolean supportsDelete() {+ return true;+ }++ @Override+ public void verifyDocumentWritten(String docId, String expectedTitle) throws Exception {+ try (RestHighLevelClient client = new RestHighLevelClient(+ RestClient.builder(+ new HttpHost(elasticsearchContainer.getHost(),+ elasticsearchContainer.getMappedPort(9200),+ "http")))) {+ GetRequest getRequest = new GetRequest("test-index", docId);+ GetResponse getResponse = client.get(getRequest, RequestOptions.DEFAULT);+ assertTrue(getResponse.isExists(), "Document should exist in index");+ assertNotNull(getResponse.getSource());+ assertEquals(expectedTitle, getResponse.getSource().get("title"));+ }+ }+}diff --git a/src/plugin/indexer-kafka/ivy.xml b/src/plugin/indexer-kafka/ivy.xmlindex d6157d953..ffba6746d 100644--- a/src/plugin/indexer-kafka/ivy.xml+++ b/src/plugin/indexer-kafka/ivy.xml@@ -37,6 +37,7 @@</publications><dependencies>+ <dependency org="org.testcontainers" name="testcontainers-kafka" rev="2.0.3" conf="test->default"/><dependency org="org.apache.kafka" name="kafka_2.12" rev="3.7.0"/><dependency org="org.apache.kafka" name="connect-json" rev="3.7.0"/></dependencies>diff --git a/src/plugin/indexer-kafka/src/test/org/apache/nutch/indexwriter/kafka/KafkaIndexWriterIT.java b/src/plugin/indexer-kafka/src/test/org/apache/nutch/indexwriter/kafka/KafkaIndexWriterIT.javanew file mode 100644index 000000000..4f6a306d4--- /dev/null+++ b/src/plugin/indexer-kafka/src/test/org/apache/nutch/indexwriter/kafka/KafkaIndexWriterIT.java@@ -0,0 +1,96 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.nutch.indexwriter.kafka;++import java.util.HashMap;+import java.util.Map;++import org.apache.hadoop.conf.Configuration;+import org.apache.nutch.indexer.AbstractIndexWriterIT;+import org.apache.nutch.indexer.IndexWriter;+import org.apache.nutch.indexer.IndexWriterParams;+import org.apache.nutch.indexer.NutchDocument;+import org.apache.nutch.util.NutchConfiguration;+import org.testcontainers.junit.jupiter.Container;+import org.testcontainers.junit.jupiter.Testcontainers;+import org.testcontainers.kafka.KafkaContainer;++/**+ * Integration tests for KafkaIndexWriter using Testcontainers.+ */+@Testcontainers(disabledWithoutDocker = true)+public class KafkaIndexWriterIT extends AbstractIndexWriterIT {++ private static final String KAFKA_IMAGE = "apache/kafka-native:3.8.0";+ private static final String TEST_TOPIC = "nutch-indexer-test";++ @Container+ private static final KafkaContainer kafkaContainer =+ new KafkaContainer(KAFKA_IMAGE);++ private KafkaIndexWriter indexWriter;+ private Configuration conf;++ @Override+ public void setUpIndexWriter() throws Exception {+ conf = NutchConfiguration.create();+ indexWriter = new KafkaIndexWriter();+ indexWriter.setConf(conf);++ String bootstrapServers = kafkaContainer.getBootstrapServers();+ String hostPort = bootstrapServers.contains("://")+ ? bootstrapServers.substring(bootstrapServers.indexOf("://") + 3)+ : bootstrapServers;+ String[] parts = hostPort.split(":");+ String host = parts[0];+ int port = Integer.parseInt(parts[1]);++ Map<String, String> params = new HashMap<>();+ params.put(KafkaConstants.HOST, host);+ params.put(KafkaConstants.PORT, String.valueOf(port));+ params.put(KafkaConstants.TOPIC, TEST_TOPIC);+ params.put(KafkaConstants.VALUE_SERIALIZER,+ "org.apache.kafka.connect.json.JsonSerializer");+ params.put(KafkaConstants.KEY_SERIALIZER,+ "org.apache.kafka.common.serialization.StringSerializer");++ IndexWriterParams writerParams = new IndexWriterParams(params);+ indexWriter.open(writerParams);+ }++ @Override+ public void tearDownIndexWriter() throws Exception {+ if (indexWriter != null) {+ try {+ indexWriter.close();+ } catch (Exception e) {+ // Ignore if open() failed and close state is invalid+ }+ indexWriter = null;+ }+ }++ @Override+ public IndexWriter getIndexWriter() {+ return indexWriter;+ }++ @Override+ public boolean supportsDelete() {+ return false;+ }+}diff --git a/src/plugin/indexer-rabbit/ivy.xml b/src/plugin/indexer-rabbit/ivy.xmlindex 81822a0fb..54930331c 100644--- a/src/plugin/indexer-rabbit/ivy.xml+++ b/src/plugin/indexer-rabbit/ivy.xml@@ -35,5 +35,9 @@<!--get the artifact from our module name--><artifact conf="master"/></publications>-++ <dependencies>+ <dependency org="org.testcontainers" name="testcontainers-rabbitmq" rev="2.0.3" conf="test->default"/>+ </dependencies>+</ivy-module>diff --git a/src/plugin/indexer-rabbit/src/test/org/apache/nutch/indexwriter/rabbit/RabbitIndexWriterIT.java b/src/plugin/indexer-rabbit/src/test/org/apache/nutch/indexwriter/rabbit/RabbitIndexWriterIT.javanew file mode 100644index 000000000..ed7d05535--- /dev/null+++ b/src/plugin/indexer-rabbit/src/test/org/apache/nutch/indexwriter/rabbit/RabbitIndexWriterIT.java@@ -0,0 +1,90 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.nutch.indexwriter.rabbit;++import java.util.HashMap;+import java.util.Map;++import org.apache.hadoop.conf.Configuration;+import org.apache.nutch.indexer.AbstractIndexWriterIT;+import org.apache.nutch.indexer.IndexWriter;+import org.apache.nutch.indexer.IndexWriterParams;+import org.apache.nutch.indexer.NutchDocument;+import org.apache.nutch.util.NutchConfiguration;+import org.testcontainers.containers.RabbitMQContainer;+import org.testcontainers.junit.jupiter.Container;+import org.testcontainers.junit.jupiter.Testcontainers;++/**+ * Integration tests for RabbitIndexWriter using Testcontainers.+ */+@Testcontainers(disabledWithoutDocker = true)+public class RabbitIndexWriterIT extends AbstractIndexWriterIT {++ private static final String RABBITMQ_IMAGE = "rabbitmq:3.13-management";++ @Container+ private static final RabbitMQContainer rabbitContainer =+ new RabbitMQContainer(RABBITMQ_IMAGE);++ private RabbitIndexWriter indexWriter;+ private Configuration conf;++ @Override+ public void setUpIndexWriter() throws Exception {+ conf = NutchConfiguration.create();+ indexWriter = new RabbitIndexWriter();+ indexWriter.setConf(conf);++ Map<String, String> params = new HashMap<>();+ params.put(RabbitMQConstants.SERVER_URI, rabbitContainer.getAmqpUrl());+ params.put(RabbitMQConstants.EXCHANGE_NAME, "nutch-indexer-test");+ params.put(RabbitMQConstants.ROUTING_KEY, "indexer");+ params.put(RabbitMQConstants.COMMIT_MODE, "single");+ params.put(RabbitMQConstants.COMMIT_SIZE, "10");+ params.put(RabbitMQConstants.BINDING, "true");+ params.put(RabbitMQConstants.QUEUE_NAME, "nutch-indexer-queue");+ params.put(RabbitMQConstants.EXCHANGE_OPTIONS, "type=direct,durable=true");+ params.put(RabbitMQConstants.QUEUE_OPTIONS,+ "durable=true,exclusive=false,auto-delete=false");++ IndexWriterParams writerParams = new IndexWriterParams(params);+ indexWriter.open(writerParams);+ }++ @Override+ public void tearDownIndexWriter() throws Exception {+ if (indexWriter != null) {+ try {+ indexWriter.close();+ } catch (Exception e) {+ // Ignore if open() failed and close state is invalid+ }+ indexWriter = null;+ }+ }++ @Override+ public IndexWriter getIndexWriter() {+ return indexWriter;+ }++ @Override+ public boolean supportsDelete() {+ return true;+ }+}diff --git a/src/plugin/indexer-solr/ivy.xml b/src/plugin/indexer-solr/ivy.xmlindex 99a713c18..4d2120955 100644--- a/src/plugin/indexer-solr/ivy.xml+++ b/src/plugin/indexer-solr/ivy.xml@@ -38,6 +38,7 @@</publications><dependencies>+ <dependency org="org.testcontainers" name="testcontainers-solr" rev="2.0.3" conf="test->default"/><dependency org="org.apache.solr" name="solr-solrj"rev="8.11.4" conf="*->default"><!-- exclusions of dependencies provided by Nutch core -->diff --git a/src/plugin/indexer-solr/src/test/org/apache/nutch/indexwriter/solr/SolrIndexWriterIT.java b/src/plugin/indexer-solr/src/test/org/apache/nutch/indexwriter/solr/SolrIndexWriterIT.javanew file mode 100644index 000000000..dcd88bdac--- /dev/null+++ b/src/plugin/indexer-solr/src/test/org/apache/nutch/indexwriter/solr/SolrIndexWriterIT.java@@ -0,0 +1,137 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.nutch.indexwriter.solr;++import java.util.Collection;+import java.util.HashMap;+import java.util.Map;++import org.apache.hadoop.conf.Configuration;+import org.apache.nutch.indexer.IndexerMapReduce;+import org.apache.nutch.indexer.AbstractIndexWriterIT;+import org.apache.nutch.indexer.IndexWriter;+import org.apache.nutch.indexer.IndexWriterParams;+import org.apache.nutch.indexer.NutchDocument;+import org.apache.nutch.util.NutchConfiguration;+import org.apache.solr.client.solrj.SolrClient;+import org.apache.solr.client.solrj.impl.Http2SolrClient;+import org.apache.solr.client.solrj.response.QueryResponse;+import org.apache.solr.common.params.ModifiableSolrParams;+import org.testcontainers.junit.jupiter.Container;+import org.testcontainers.junit.jupiter.Testcontainers;+import org.testcontainers.solr.SolrContainer;++import static org.junit.jupiter.api.Assertions.assertEquals;+import static org.junit.jupiter.api.Assertions.assertTrue;++/**+ * Integration tests for SolrIndexWriter using Testcontainers.+ */+@Testcontainers(disabledWithoutDocker = true)+public class SolrIndexWriterIT extends AbstractIndexWriterIT {++ private static final String SOLR_IMAGE = "solr:8.11.2";+ private static final String COLLECTION = "nutch-test";++ @Container+ private static final SolrContainer solrContainer =+ new SolrContainer(SOLR_IMAGE).withCollection(COLLECTION);++ private SolrIndexWriter indexWriter;+ private Configuration conf;++ @Override+ public void setUpIndexWriter() throws Exception {+ conf = NutchConfiguration.create();+ conf.setBoolean(IndexerMapReduce.INDEXER_DELETE, false);++ indexWriter = new SolrIndexWriter();+ indexWriter.setConf(conf);++ String solrUrl = "http://" + solrContainer.getHost() + ":"+ + solrContainer.getSolrPort() + "/solr/" + COLLECTION;++ Map<String, String> params = new HashMap<>();+ params.put(SolrConstants.SERVER_TYPE, "http");+ params.put(SolrConstants.SERVER_URLS, solrUrl);+ params.put(SolrConstants.COLLECTION, COLLECTION);+ params.put(SolrConstants.COMMIT_SIZE, "100");++ IndexWriterParams writerParams = new IndexWriterParams(params);+ indexWriter.open(writerParams);+ }++ @Override+ public void tearDownIndexWriter() throws Exception {+ if (indexWriter != null) {+ try {+ indexWriter.close();+ } catch (Exception e) {+ // Ignore if open() failed and close state is invalid+ }+ indexWriter = null;+ }+ }++ @Override+ public IndexWriter getIndexWriter() {+ return indexWriter;+ }++ @Override+ public boolean supportsDelete() {+ return true;+ }++ @Override+ public void verifyDocumentWritten(String docId, String expectedTitle) throws Exception {+ try (SolrClient client = new Http2SolrClient.Builder(+ "http://" + solrContainer.getHost() + ":"+ + solrContainer.getSolrPort() + "/solr/" + COLLECTION).build()) {+ ModifiableSolrParams queryParams = new ModifiableSolrParams();+ queryParams.set("q", "id:" + docId);+ QueryResponse response = client.query(queryParams);+ assertTrue(response.getResults().getNumFound() >= 1,+ "Document should exist in Solr");+ Object titleValue = response.getResults().get(0).getFieldValue("title");+ String title = titleValue instanceof Collection+ ? ((Collection<?>) titleValue).iterator().next().toString()+ : titleValue.toString();+ assertEquals(expectedTitle, title);+ }+ }++ @Override+ public IndexWriter prepareWriterForDeleteTest() throws Exception {+ tearDownIndexWriter();++ Configuration deleteConf = NutchConfiguration.create();+ deleteConf.setBoolean(IndexerMapReduce.INDEXER_DELETE, true);+ SolrIndexWriter deleteWriter = new SolrIndexWriter();+ deleteWriter.setConf(deleteConf);++ String solrUrl = "http://" + solrContainer.getHost() + ":"+ + solrContainer.getSolrPort() + "/solr/" + COLLECTION;+ Map<String, String> params = new HashMap<>();+ params.put(SolrConstants.SERVER_TYPE, "http");+ params.put(SolrConstants.SERVER_URLS, solrUrl);+ params.put(SolrConstants.COLLECTION, COLLECTION);+ deleteWriter.open(new IndexWriterParams(params));++ return deleteWriter;+ }+}diff --git a/src/test/org/apache/nutch/indexer/AbstractIndexWriterIT.java b/src/test/org/apache/nutch/indexer/AbstractIndexWriterIT.javanew file mode 100644index 000000000..b0bf6e023--- /dev/null+++ b/src/test/org/apache/nutch/indexer/AbstractIndexWriterIT.java@@ -0,0 +1,90 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.nutch.indexer;++import static org.junit.jupiter.api.Assertions.assertDoesNotThrow;++import org.apache.nutch.indexer.NutchDocument;+import org.junit.jupiter.api.AfterEach;+import org.junit.jupiter.api.BeforeEach;+import org.junit.jupiter.api.Test;+import org.testcontainers.junit.jupiter.Testcontainers;++/**+ * Abstract base for IndexWriter integration tests. Provides common test logic+ * for write/commit and delete operations.+ */+@Testcontainers(disabledWithoutDocker = true)+public abstract class AbstractIndexWriterIT implements IndexWriterIntegrationTest {++ @BeforeEach+ void setUp() throws Exception {+ setUpIndexWriter();+ }++ @AfterEach+ void tearDown() throws Exception {+ tearDownIndexWriter();+ }++ @Test+ void testWriteAndCommitDocument() throws Exception {+ NutchDocument doc = createTestDocument("test-doc-1", "Test Document",+ "This is a test document for integration testing.");+ assertDoesNotThrow(() -> getIndexWriter().write(doc));+ assertDoesNotThrow(() -> getIndexWriter().commit());+ tearDownIndexWriter();+ verifyDocumentWritten("test-doc-1", "Test Document");+ }++ @Test+ void testDeleteDocument() throws Exception {+ if (!supportsDelete()) {+ return;+ }+ String docId = "test-doc-to-delete";+ NutchDocument doc = createTestDocument(docId, "Document to Delete", "");++ IndexWriter writer = getIndexWriter();+ writer.write(doc);+ writer.commit();++ IndexWriter deleteWriter = prepareWriterForDeleteTest();+ if (deleteWriter == null) {+ deleteWriter = writer;+ }+ final IndexWriter writerForDelete = deleteWriter;+ assertDoesNotThrow(() -> writerForDelete.delete(docId));+ assertDoesNotThrow(() -> writerForDelete.commit());+ if (deleteWriter != writer) {+ try {+ deleteWriter.close();+ } catch (Exception e) {+ // Ignore+ }+ }+ }++ /** Create a NutchDocument with id, title, and content. */+ protected NutchDocument createTestDocument(String id, String title, String content) {+ NutchDocument doc = new NutchDocument();+ doc.add("id", id);+ doc.add("title", title);+ doc.add("content", content);+ return doc;+ }+}diff --git a/src/test/org/apache/nutch/indexer/IndexWriterIntegrationTest.java b/src/test/org/apache/nutch/indexer/IndexWriterIntegrationTest.javanew file mode 100644index 000000000..c6f1027da--- /dev/null+++ b/src/test/org/apache/nutch/indexer/IndexWriterIntegrationTest.java@@ -0,0 +1,53 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.nutch.indexer;++/**+ * Contract for IndexWriter integration tests. Implementations run against+ * real backends via Testcontainers.+ */+public interface IndexWriterIntegrationTest {++ /** Open the index writer before tests. */+ void setUpIndexWriter() throws Exception;++ /** Close the index writer after tests. */+ void tearDownIndexWriter() throws Exception;++ /** The IndexWriter under test. */+ IndexWriter getIndexWriter();++ /** Whether this writer supports document deletion (e.g. Kafka does not). */+ boolean supportsDelete();++ /**+ * Optional verification that a document was indexed.+ * Default no-op; override for Elastic, Solr.+ */+ default void verifyDocumentWritten(String docId, String expectedTitle) throws Exception {+ // no-op+ }++ /**+ * Optional writer configured for delete operations. Used when the main+ * writer has delete disabled (e.g. Solr requires INDEXER_DELETE=true).+ * Default returns null to use {@link #getIndexWriter()}.+ */+ default IndexWriter prepareWriterForDeleteTest() throws Exception {+ return null;+ }+}