Apache Solr open-source search software
SOLR-16765: Export what the limit param specifies, and don't be off by one in the counter. (#1636)
Eric Pughbe68ec9404c1
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 3b68d3f2135..0cff2c95355 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -173,6 +173,8 @@ Bug Fixes* SOLR-16760: bin/solr package tool supports -h and -help command like other tools. (Eric Pugh)+* SOLR-16765: bin/solr export tool limit property was off by 1. Now limit results exported to the exact number. (Eric Pugh)+Dependency Upgrades---------------------* PR#1494: Upgrade forbiddenapis to 3.5 (Uwe Schindler)diff --git a/solr/core/src/java/org/apache/solr/cli/ExportTool.java b/solr/core/src/java/org/apache/solr/cli/ExportTool.javaindex a262018206a..a50cfb9bbaa 100644--- a/solr/core/src/java/org/apache/solr/cli/ExportTool.java+++ b/solr/core/src/java/org/apache/solr/cli/ExportTool.java@@ -518,7 +518,7 @@ public class ExportTool extends ToolBase {addConsumer(consumerlatch);addProducers(m);if (output != null) {- output.println("NO: of shards : " + corehandlers.size());+ output.println("Number of shards : " + corehandlers.size());}CountDownLatch producerLatch = new CountDownLatch(corehandlers.size());corehandlers.forEach(@@ -550,10 +550,10 @@ public class ExportTool extends ToolBase {}System.out.println("\nTotal Docs exported: "- + (docsWritten.get() - 1)- + ". Time taken: "+ + docsWritten.get()+ + ". Time elapsed: "+ ((System.currentTimeMillis() - startTime) / 1000)- + "secs");+ + "seconds");}}@@ -582,7 +582,9 @@ public class ExportTool extends ToolBase {}if (doc == EOFDOC) break;try {- if (docsWritten.get() > limit) continue;+ if (docsWritten.get() >= limit) {+ continue;+ }sink.accept(doc);} catch (Exception e) {if (output != null) output.println("Failed to write to file " + e.getMessage());@@ -605,7 +607,7 @@ public class ExportTool extends ToolBase {boolean exportDocsFromCore() throws IOException, SolrServerException {try (SolrClient client = new Http2SolrClient.Builder(baseurl).build()) {- expectedDocs = getDocCount(replica.getCoreName(), client);+ expectedDocs = getDocCount(replica.getCoreName(), client, query);GenericSolrRequest request;ModifiableSolrParams params = new ModifiableSolrParams();params.add(Q, query);@@ -638,17 +640,18 @@ public class ExportTool extends ToolBase {NamedList<Object> rsp = client.request(request);String nextCursorMark = (String) rsp.get(CursorMarkParams.CURSOR_MARK_NEXT);if (nextCursorMark == null || Objects.equals(cursorMark, nextCursorMark)) {- if (output != null)+ if (output != null) {output.println(StrUtils.formatString(- "\nExport complete for : {0}, docs : {1}",- replica.getCoreName(), receivedDocs.get()));+ "\nExport complete from shard {0}, core {1}, docs received: {2}",+ replica.getShard(), replica.getCoreName(), receivedDocs.get()));+ }if (expectedDocs != receivedDocs.get()) {if (output != null) {output.println(StrUtils.formatString(- "Could not download all docs for core {0} , expected: {1} , actual",- replica.getCoreName(), expectedDocs, receivedDocs));+ "Could not download all docs from core {0}, docs expected: {1}, received: {2}",+ replica.getCoreName(), expectedDocs, receivedDocs.get()));return false;}}@@ -672,9 +675,9 @@ public class ExportTool extends ToolBase {}}- static long getDocCount(String coreName, SolrClient client)+ static long getDocCount(String coreName, SolrClient client, String query)throws SolrServerException, IOException {- SolrQuery q = new SolrQuery("*:*");+ SolrQuery q = new SolrQuery(query);q.setRows(0);q.add("distrib", "false");GenericSolrRequest request =diff --git a/solr/core/src/test/org/apache/solr/cli/TestExportTool.java b/solr/core/src/test/org/apache/solr/cli/TestExportTool.javaindex 56c6e9b8656..a59dbafe7fb 100644--- a/solr/core/src/test/org/apache/solr/cli/TestExportTool.java+++ b/solr/core/src/test/org/apache/solr/cli/TestExportTool.java@@ -180,7 +180,7 @@ public class TestExportTool extends SolrCloudTestCase {for (Slice slice : coll.getSlices()) {Replica replica = slice.getLeader();try (SolrClient client = new Http2SolrClient.Builder(replica.getBaseUrl()).build()) {- long count = ExportTool.getDocCount(replica.getCoreName(), client);+ long count = ExportTool.getDocCount(replica.getCoreName(), client, "*:*");docCounts.put(replica.getCoreName(), count);totalDocsFromCores += count;}diff --git a/solr/packaging/test/test_export.bats b/solr/packaging/test/test_export.batsindex 0e9aba9aef5..699eba33911 100644--- a/solr/packaging/test/test_export.bats+++ b/solr/packaging/test/test_export.bats@@ -55,8 +55,9 @@ teardown() {run solr export -url "http://localhost:8983/solr/techproducts" -query "*:* -id:test" -format jsonl -out "${BATS_TEST_TMPDIR}/output"assert [ -e ${BATS_TEST_TMPDIR}/output.jsonl ]- run solr export -url "http://localhost:8983/solr/techproducts" -query "*:* -id:test" -compress -format jsonl -out "${BATS_TEST_TMPDIR}/output"+ run solr export -url "http://localhost:8983/solr/techproducts" -query "*:* -id:test" -limit 10 -compress -format jsonl -out "${BATS_TEST_TMPDIR}/output"assert [ -e ${BATS_TEST_TMPDIR}/output.jsonl.gz ]+ assert_output --partial 'Total Docs exported: 10'}