Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-17867: Export Tool should export regular docs cleanly in .json, .jsonl, and javabin (second take) (#2636)

Eric Pugh585b1d00a005
diff --git a/ExportTool.java.original b/ExportTool.java.original
new file mode 100644
index 00000000000..050092c2017
--- /dev/null
+++ b/ExportTool.java.original
@@ -0,0 +1,738 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.cli;
+
+import static org.apache.solr.common.params.CommonParams.FL;
+import static org.apache.solr.common.params.CommonParams.JAVABIN;
+import static org.apache.solr.common.params.CommonParams.JSON;
+import static org.apache.solr.common.params.CommonParams.Q;
+import static org.apache.solr.common.params.CommonParams.SORT;
+import static org.apache.solr.common.util.JavaBinCodec.SOLRINPUTDOC;
+
+import java.io.BufferedOutputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.io.OutputStreamWriter;
+import java.io.Writer;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.time.Instant;
+import java.time.format.DateTimeFormatter;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Set;
+import java.util.concurrent.ArrayBlockingQueue;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicLong;
+import java.util.function.BiConsumer;
+import java.util.function.Consumer;
+import java.util.zip.GZIPOutputStream;
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.Option;
+import org.apache.commons.cli.Options;
+import org.apache.lucene.util.SuppressForbidden;
+import org.apache.solr.client.solrj.SolrClient;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrRequest;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.StreamingResponseCallback;
+import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.impl.ClusterStateProvider;
+import org.apache.solr.client.solrj.impl.Http2SolrClient;
+import org.apache.solr.client.solrj.impl.StreamingJavaBinResponseParser;
+import org.apache.solr.client.solrj.request.GenericSolrRequest;
+import org.apache.solr.client.solrj.request.QueryRequest;
+import org.apache.solr.common.SolrDocument;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.Slice;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.CursorMarkParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.CollectionUtil;
+import org.apache.solr.common.util.ExecutorUtil;
+import org.apache.solr.common.util.JavaBinCodec;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SolrNamedThreadFactory;
+import org.apache.solr.common.util.StrUtils;
+import org.noggit.CharArr;
+import org.noggit.JSONWriter;
+
+/** Supports export command in the bin/solr script. */
+public class ExportTool extends ToolBase {
+
+ private static final Option COLLECTION_NAME_OPTION =
+ Option.builder("c")
+ .longOpt("name")
+ .hasArg()
+ .argName("NAME")
+ .desc("Name of the collection.")
+ .build();
+
+ private static final Option OUTPUT_OPTION =
+ Option.builder()
+ .longOpt("output")
+ .hasArg()
+ .argName("PATH")
+ .desc(
+ "Path to output the exported data, and optionally the file name, defaults to 'collection-name'.")
+ .build();
+
+ private static final Option FORMAT_OPTION =
+ Option.builder()
+ .longOpt("format")
+ .hasArg()
+ .argName("FORMAT")
+ .desc("Output format for exported docs (json, jsonl or javabin), defaulting to json.")
+ .build();
+
+ private static final Option COMPRESS_OPTION =
+ Option.builder().longOpt("compress").desc("Compress the output. Defaults to false.").build();
+
+ private static final Option LIMIT_OPTION =
+ Option.builder()
+ .longOpt("limit")
+ .hasArg()
+ .argName("#")
+ .desc("Maximum number of docs to download. Default is 100, use -1 for all docs.")
+ .build();
+
+ private static final Option QUERY_OPTION =
+ Option.builder()
+ .longOpt("query")
+ .hasArg()
+ .argName("QUERY")
+ .desc("A custom query, default is '*:*'.")
+ .build();
+
+ private static final Option FIELDS_OPTION =
+ Option.builder()
+ .longOpt("fields")
+ .hasArg()
+ .argName("FIELDA,FIELDB")
+ .desc("Comma separated list of fields to export. By default all fields are fetched.")
+ .build();
+
+ public ExportTool(ToolRuntime runtime) {
+ super(runtime);
+ }
+
+ @Override
+ public String getName() {
+ return "export";
+ }
+
+ @Override
+ public Options getOptions() {
+ return super.getOptions()
+ .addOption(COLLECTION_NAME_OPTION)
+ .addOption(OUTPUT_OPTION)
+ .addOption(FORMAT_OPTION)
+ .addOption(COMPRESS_OPTION)
+ .addOption(LIMIT_OPTION)
+ .addOption(QUERY_OPTION)
+ .addOption(FIELDS_OPTION)
+ .addOption(CommonCLIOptions.SOLR_URL_OPTION)
+ .addOption(CommonCLIOptions.CREDENTIALS_OPTION);
+ }
+
+ public abstract static class Info {
+ final ToolRuntime runtime;
+
+ String baseurl;
+ String credentials;
+ String format;
+ boolean compress;
+ String query;
+ String coll;
+ String out;
+ String fields;
+ long limit = 100;
+ AtomicLong docsWritten = new AtomicLong(0);
+ int bufferSize = 1024 * 1024;
+ String uniqueKey;
+ CloudSolrClient solrClient;
+ DocsSink sink;
+
+ public Info(ToolRuntime runtime, String url, String credentials) {
+ this.runtime = runtime;
+ setUrl(url);
+ setCredentials(credentials);
+ setOutFormat(null, "jsonl", false);
+ }
+
+ public void setUrl(String url) {
+ int idx = url.lastIndexOf('/');
+ baseurl = url.substring(0, idx);
+ coll = url.substring(idx + 1);
+ query = "*:*";
+ }
+
+ public void setCredentials(String credentials) {
+ this.credentials = credentials;
+ }
+
+ public void setLimit(String maxDocsStr) {
+ limit = Long.parseLong(maxDocsStr);
+ if (limit == -1) limit = Long.MAX_VALUE;
+ }
+
+ public void setOutFormat(String out, String format, boolean compress) {
+ this.compress = compress;
+ this.format = format;
+ this.out = out;
+ if (this.format == null) {
+ this.format = JSON;
+ }
+ if (!formats.contains(this.format)) {
+ throw new IllegalArgumentException("format must be one of: " + formats);
+ }
+ if (this.out == null) {
+ this.out = coll;
+ } else if (Files.isDirectory(Path.of(this.out))) {
+ this.out = this.out + "/" + coll;
+ }
+ this.out = this.out + '.' + this.format;
+ if (compress) {
+ this.out = this.out + ".gz";
+ }
+ }
+
+ DocsSink getSink() {
+ DocsSink docSink = null;
+ switch (format) {
+ case JAVABIN:
+ docSink = new JavabinSink(this);
+ break;
+ case JSON:
+ docSink = new JsonSink(this);
+ break;
+ case "jsonl":
+ docSink = new JsonWithLinesSink(this);
+ break;
+ }
+ return docSink;
+ }
+
+ abstract void exportDocs() throws Exception;
+
+ void fetchUniqueKey() throws SolrServerException, IOException {
+ Http2SolrClient.Builder builder =
+ new Http2SolrClient.Builder().withOptionalBasicAuthCredentials(credentials);
+
+ solrClient =
+ new CloudHttp2SolrClient.Builder(Collections.singletonList(baseurl))
+ .withInternalClientBuilder(builder)
+ .build();
+ NamedList<Object> response =
+ solrClient.request(
+ new GenericSolrRequest(
+ SolrRequest.METHOD.GET,
+ "/schema/uniquekey",
+ SolrParams.of("collection", coll))
+ .setRequiresCollection(true));
+ uniqueKey = (String) response.get("uniqueKey");
+ }
+
+ public static StreamingResponseCallback getStreamer(Consumer<SolrDocument> sink) {
+ return new StreamingResponseCallback() {
+ @Override
+ public void streamSolrDocument(SolrDocument doc) {
+ try {
+ sink.accept(doc);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void streamDocListInfo(long numFound, long start, Float maxScore) {}
+ };
+ }
+ }
+
+ static Set<String> formats = Set.of(JAVABIN, JSON, "jsonl");
+
+ @Override
+ public void runImpl(CommandLine cli) throws Exception {
+ String url = null;
+ if (cli.hasOption(CommonCLIOptions.SOLR_URL_OPTION)) {
+ if (!cli.hasOption(COLLECTION_NAME_OPTION)) {
+ throw new IllegalArgumentException(
+ "Must specify -c / --name parameter with --solr-url to post documents.");
+ }
+ url = CLIUtils.normalizeSolrUrl(cli) + "/solr/" + cli.getOptionValue(COLLECTION_NAME_OPTION);
+
+ } else {
+ // think about support --zk-host someday.
+ throw new IllegalArgumentException("Must specify --solr-url.");
+ }
+ String credentials = cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION);
+ Info info = new MultiThreadedRunner(runtime, url, credentials);
+ info.query = cli.getOptionValue(QUERY_OPTION, "*:*");
+
+ info.setOutFormat(
+ cli.getOptionValue(OUTPUT_OPTION),
+ cli.getOptionValue(FORMAT_OPTION),
+ cli.hasOption(COMPRESS_OPTION));
+ info.fields = cli.getOptionValue(FIELDS_OPTION);
+ info.setLimit(cli.getOptionValue(LIMIT_OPTION, "100"));
+ info.exportDocs();
+ }
+
+ abstract static class DocsSink {
+ Info info;
+ OutputStream fos;
+
+ abstract void start() throws IOException;
+
+ @SuppressForbidden(reason = "Command line tool prints out to console")
+ void accept(SolrDocument document) throws IOException {
+ long count = info.docsWritten.incrementAndGet();
+
+ if (count % 100000 == 0) {
+ System.out.println("\nDOCS: " + count);
+ }
+ }
+
+ void end() throws IOException {}
+ }
+
+ static class JsonWithLinesSink extends DocsSink {
+ private final CharArr charArr = new CharArr(1024 * 2);
+ JSONWriter jsonWriter = new JSONWriter(charArr, -1);
+ private Writer writer;
+
+ public JsonWithLinesSink(Info info) {
+ this.info = info;
+ }
+
+ @Override
+ public void start() throws IOException {
+ fos = new FileOutputStream(info.out);
+ if (info.compress) {
+ fos = new GZIPOutputStream(fos);
+ }
+ if (info.bufferSize > 0) {
+ fos = new BufferedOutputStream(fos, info.bufferSize);
+ }
+ writer = new OutputStreamWriter(fos, StandardCharsets.UTF_8);
+ }
+
+ @Override
+ public void end() throws IOException {
+ writer.flush();
+ fos.flush();
+ fos.close();
+ }
+
+ @Override
+ public synchronized void accept(SolrDocument doc) throws IOException {
+ charArr.reset();
+ Map<String, Object> m = CollectionUtil.newLinkedHashMap(doc.size());
+ doc.forEach(
+ (s, field) -> {
+ if (s.equals("_version_") || s.equals("_roor_")) return;
+ if (field instanceof List) {
+ if (((List<?>) field).size() == 1) {
+ field = ((List<?>) field).get(0);
+ }
+ }
+ field = constructDateStr(field);
+ if (field instanceof List<?> list) {
+ if (hasdate(list)) {
+ ArrayList<Object> listCopy = new ArrayList<>(list.size());
+ for (Object o : list) listCopy.add(constructDateStr(o));
+ field = listCopy;
+ }
+ }
+ m.put(s, field);
+ });
+ jsonWriter.write(m);
+ writer.write(charArr.getArray(), charArr.getStart(), charArr.getEnd());
+ writer.append('\n');
+ super.accept(doc);
+ }
+
+ private boolean hasdate(List<?> list) {
+ boolean hasDate = false;
+ for (Object o : list) {
+ if (o instanceof Date) {
+ hasDate = true;
+ break;
+ }
+ }
+ return hasDate;
+ }
+
+ private Object constructDateStr(Object field) {
+ if (field instanceof Date) {
+ field =
+ DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochMilli(((Date) field).getTime()));
+ }
+ return field;
+ }
+ }
+
+ static class JsonSink extends DocsSink {
+ private final CharArr charArr = new CharArr(1024 * 2);
+ JSONWriter jsonWriter = new JSONWriter(charArr, -1);
+ private Writer writer;
+ private boolean firstDoc = true;
+
+ public JsonSink(Info info) {
+ this.info = info;
+ }
+
+ @Override
+ public void start() throws IOException {
+ fos = new FileOutputStream(info.out);
+ if (info.compress) {
+ fos = new GZIPOutputStream(fos);
+ }
+ if (info.bufferSize > 0) {
+ fos = new BufferedOutputStream(fos, info.bufferSize);
+ }
+ writer = new OutputStreamWriter(fos, StandardCharsets.UTF_8);
+ writer.append('[');
+ }
+
+ @Override
+ public void end() throws IOException {
+ writer.append(']');
+ writer.flush();
+ fos.flush();
+ fos.close();
+ }
+
+ @Override
+ public synchronized void accept(SolrDocument doc) throws IOException {
+ charArr.reset();
+ Map<String, Object> m = CollectionUtil.newLinkedHashMap(doc.size());
+ doc.forEach(
+ (s, field) -> {
+ if (s.equals("_version_") || s.equals("_roor_")) return;
+ if (field instanceof List) {
+ if (((List<?>) field).size() == 1) {
+ field = ((List<?>) field).get(0);
+ }
+ }
+ field = constructDateStr(field);
+ if (field instanceof List<?> list) {
+ if (hasdate(list)) {
+ ArrayList<Object> listCopy = new ArrayList<>(list.size());
+ for (Object o : list) listCopy.add(constructDateStr(o));
+ field = listCopy;
+ }
+ }
+ m.put(s, field);
+ });
+ if (firstDoc) {
+ firstDoc = false;
+ } else {
+ writer.append(',');
+ }
+ jsonWriter.write(m);
+ writer.write(charArr.getArray(), charArr.getStart(), charArr.getEnd());
+ writer.append('\n');
+ super.accept(doc);
+ }
+
+ private boolean hasdate(List<?> list) {
+ boolean hasDate = false;
+ for (Object o : list) {
+ if (o instanceof Date) {
+ hasDate = true;
+ break;
+ }
+ }
+ return hasDate;
+ }
+
+ private Object constructDateStr(Object field) {
+ if (field instanceof Date) {
+ field =
+ DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochMilli(((Date) field).getTime()));
+ }
+ return field;
+ }
+ }
+
+ static class JavabinSink extends DocsSink {
+ JavaBinCodec codec;
+
+ public JavabinSink(Info info) {
+ this.info = info;
+ }
+
+ @Override
+ public void start() throws IOException {
+ fos = new FileOutputStream(info.out);
+ if (info.compress) {
+ fos = new GZIPOutputStream(fos);
+ }
+ if (info.bufferSize > 0) {
+ fos = new BufferedOutputStream(fos, info.bufferSize);
+ }
+ codec = new JavaBinCodec(fos, null);
+ codec.writeTag(JavaBinCodec.NAMED_LST, 2);
+ codec.writeStr("params");
+ codec.writeNamedList(new NamedList<>());
+ codec.writeStr("docs");
+ codec.writeTag(JavaBinCodec.ITERATOR);
+ }
+
+ @Override
+ public void end() throws IOException {
+ codec.writeTag(JavaBinCodec.END);
+ codec.close();
+ fos.flush();
+ fos.close();
+ }
+
+ private final BiConsumer<String, Object> bic =
+ new BiConsumer<>() {
+ @Override
+ public void accept(String s, Object o) {
+ try {
+ if (s.equals("_version_") || s.equals("_root_")) return;
+ codec.writeExternString(s);
+ codec.writeVal(o);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+ };
+
+ @Override
+ public synchronized void accept(SolrDocument doc) throws IOException {
+ int sz = doc.size();
+ if (doc.containsKey("_version_")) sz--;
+ if (doc.containsKey("_root_")) sz--;
+ codec.writeTag(SOLRINPUTDOC, sz);
+ codec.writeFloat(1f); // document boost
+ doc.forEach(bic);
+ super.accept(doc);
+ }
+ }
+
+ static class MultiThreadedRunner extends Info {
+ ExecutorService producerThreadpool, consumerThreadpool;
+ ArrayBlockingQueue<SolrDocument> queue = new ArrayBlockingQueue<>(1000);
+ SolrDocument EOFDOC = new SolrDocument();
+ volatile boolean failed = false;
+ Map<String, CoreHandler> corehandlers = new HashMap<>();
+ private final long startTime;
+
+ @SuppressForbidden(reason = "Need to print out time")
+ public MultiThreadedRunner(ToolRuntime runtime, String url, String credentials) {
+ super(runtime, url, credentials);
+ startTime = System.currentTimeMillis();
+ }
+
+ @Override
+ @SuppressForbidden(reason = "Need to print out time")
+ void exportDocs() throws Exception {
+ sink = getSink();
+ fetchUniqueKey();
+ ClusterStateProvider stateProvider = solrClient.getClusterStateProvider();
+ DocCollection coll = stateProvider.getCollection(this.coll);
+ Map<String, Slice> m = coll.getSlicesMap();
+ producerThreadpool =
+ ExecutorUtil.newMDCAwareFixedThreadPool(
+ m.size(), new SolrNamedThreadFactory("solrcli-exporter-producers"));
+ consumerThreadpool =
+ ExecutorUtil.newMDCAwareFixedThreadPool(
+ 1, new SolrNamedThreadFactory("solrcli-exporter-consumer"));
+ sink.start();
+ CountDownLatch consumerlatch = new CountDownLatch(1);
+ try {
+ addConsumer(consumerlatch);
+ addProducers(m);
+ runtime.println("Number of shards : " + corehandlers.size());
+ CountDownLatch producerLatch = new CountDownLatch(corehandlers.size());
+ corehandlers.forEach(
+ (s, coreHandler) ->
+ producerThreadpool.execute(
+ () -> {
+ try {
+ coreHandler.exportDocsFromCore();
+ } catch (Exception e) {
+ runtime.println("Error exporting docs from : " + s);
+ }
+ producerLatch.countDown();
+ }));
+
+ producerLatch.await();
+ queue.offer(EOFDOC, 10, TimeUnit.SECONDS);
+ consumerlatch.await();
+ } finally {
+ sink.end();
+ solrClient.close();
+ producerThreadpool.shutdownNow();
+ consumerThreadpool.shutdownNow();
+ if (failed) {
+ try {
+ Files.delete(Path.of(out));
+ } catch (IOException e) {
+ // ignore
+ }
+ }
+ System.out.println(
+ "\nTotal Docs exported: "
+ + docsWritten.get()
+ + ". Time elapsed: "
+ + ((System.currentTimeMillis() - startTime) / 1000)
+ + "seconds");
+ }
+ }
+
+ private void addProducers(Map<String, Slice> m) {
+ for (Map.Entry<String, Slice> entry : m.entrySet()) {
+ Slice slice = entry.getValue();
+ Replica replica = slice.getLeader();
+ if (replica == null)
+ replica = slice.getReplicas().iterator().next(); // get a random replica
+ CoreHandler coreHandler = new CoreHandler(replica);
+ corehandlers.put(replica.getCoreName(), coreHandler);
+ }
+ }
+
+ private void addConsumer(CountDownLatch consumerlatch) {
+ consumerThreadpool.execute(
+ () -> {
+ while (true) {
+ SolrDocument doc;
+ try {
+ doc = queue.poll(30, TimeUnit.SECONDS);
+ } catch (InterruptedException e) {
+ runtime.println("Consumer interrupted");
+ failed = true;
+ break;
+ }
+ if (doc == EOFDOC) break;
+ try {
+ if (docsWritten.get() >= limit) {
+ continue;
+ }
+ sink.accept(doc);
+ } catch (Exception e) {
+ runtime.println("Failed to write to file " + e.getMessage());
+ failed = true;
+ }
+ }
+ consumerlatch.countDown();
+ });
+ }
+
+ class CoreHandler {
+ final Replica replica;
+ long expectedDocs;
+ AtomicLong receivedDocs = new AtomicLong();
+
+ CoreHandler(Replica replica) {
+ this.replica = replica;
+ }
+
+ boolean exportDocsFromCore() throws IOException, SolrServerException {
+ // reference the replica's node URL, not the baseUrl in scope, which could be anywhere
+ try (SolrClient client = CLIUtils.getSolrClient(replica.getBaseUrl(), credentials)) {
+ expectedDocs = getDocCount(replica.getCoreName(), client, query);
+ QueryRequest request;
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.add(Q, query);
+ if (fields != null) params.add(FL, fields);
+ params.add(SORT, uniqueKey + " asc");
+ params.add(CommonParams.DISTRIB, "false");
+ params.add(CommonParams.ROWS, "1000");
+ String cursorMark = CursorMarkParams.CURSOR_MARK_START;
+ Consumer<SolrDocument> wrapper =
+ doc -> {
+ try {
+ queue.offer(doc, 10, TimeUnit.SECONDS);
+ receivedDocs.incrementAndGet();
+ } catch (InterruptedException e) {
+ failed = true;
+ runtime.println("Failed to write docs from" + e.getMessage());
+ }
+ };
+ StreamingJavaBinResponseParser responseParser =
+ new StreamingJavaBinResponseParser(getStreamer(wrapper));
+ while (true) {
+ if (failed) return false;
+ if (docsWritten.get() > limit) return true;
+ params.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark);
+ request = new QueryRequest(params);
+ request.setResponseParser(responseParser);
+ try {
+ NamedList<Object> rsp = client.request(request, replica.getCoreName());
+ String nextCursorMark = (String) rsp.get(CursorMarkParams.CURSOR_MARK_NEXT);
+ if (nextCursorMark == null || Objects.equals(cursorMark, nextCursorMark)) {
+ runtime.println(
+ StrUtils.formatString(
+ "\nExport complete from shard {0}, core {1}, docs received: {2}",
+ replica.getShard(), replica.getCoreName(), receivedDocs.get()));
+ if (expectedDocs != receivedDocs.get()) {
+ runtime.println(
+ StrUtils.formatString(
+ "Could not download all docs from core {0}, docs expected: {1}, received: {2}",
+ replica.getCoreName(), expectedDocs, receivedDocs.get()));
+ return false;
+ }
+ return true;
+ }
+ cursorMark = nextCursorMark;
+ runtime.print(".");
+ } catch (SolrServerException e) {
+ runtime.println(
+ "Error reading from server "
+ + replica.getBaseUrl()
+ + "/"
+ + replica.getCoreName());
+ failed = true;
+ return false;
+ }
+ }
+ }
+ }
+ }
+ }
+
+ static long getDocCount(String coreName, SolrClient client, String query)
+ throws SolrServerException, IOException {
+ SolrQuery q = new SolrQuery(query);
+ q.setRows(0);
+ q.add("distrib", "false");
+ final var request = new QueryRequest(q);
+ NamedList<Object> res = client.request(request, coreName);
+ SolrDocumentList sdl = (SolrDocumentList) res.get("response");
+ return sdl.getNumFound();
+ }
+}
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 3baf170ec67..73dd813599c 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -51,6 +51,8 @@ Improvements
* SOLR-17852: Migrate Schema Designer to use FileStore API instead of BlobHandler for persisting working data. (Eric Pugh)
+* SOLR-17867: Export tool should properly output exported documents in json, json w/ lines, and javabin formats. (Eric Pugh)
+
Optimizations
---------------------
* SOLR-17568: The CLI bin/solr export tool now contacts the appropriate nodes directly for data instead of proxying through one.
diff --git a/solr/core/src/java/org/apache/solr/cli/ExportTool.java b/solr/core/src/java/org/apache/solr/cli/ExportTool.java
index 050092c2017..c742c00999c 100644
--- a/solr/core/src/java/org/apache/solr/cli/ExportTool.java
+++ b/solr/core/src/java/org/apache/solr/cli/ExportTool.java
@@ -219,12 +219,20 @@ public class ExportTool extends ToolBase {
} else if (Files.isDirectory(Path.of(this.out))) {
this.out = this.out + "/" + coll;
}
- this.out = this.out + '.' + this.format;
- if (compress) {
+ if (!hasExtension(this.out)) {
+ this.out = this.out + '.' + this.format;
+ }
+ if (compress & !this.out.endsWith(".gz")) {
this.out = this.out + ".gz";
}
}
+ public static boolean hasExtension(String filename) {
+ return filename.contains(".json")
+ || filename.contains(".jsonl")
+ || filename.contains(".javabin");
+ }
+
DocsSink getSink() {
DocsSink docSink = null;
switch (format) {
@@ -311,6 +319,51 @@ public class ExportTool extends ToolBase {
Info info;
OutputStream fos;
+ /** Process a SolrDocument into a Map, handling special fields and date conversion. */
+ protected Map<String, Object> processDocument(SolrDocument doc) {
+ Map<String, Object> m = CollectionUtil.newLinkedHashMap(doc.size());
+ doc.forEach(
+ (s, field) -> {
+ if (s.equals("_version_") || s.equals("_roor_")) return;
+ if (field instanceof List) {
+ if (((List<?>) field).size() == 1) {
+ field = ((List<?>) field).get(0);
+ }
+ }
+ field = constructDateStr(field);
+ if (field instanceof List<?> list) {
+ if (hasDate(list)) {
+ ArrayList<Object> listCopy = new ArrayList<>(list.size());
+ for (Object o : list) listCopy.add(constructDateStr(o));
+ field = listCopy;
+ }
+ }
+ m.put(s, field);
+ });
+ return m;
+ }
+
+ /** Check if a list contains any Date objects */
+ protected boolean hasDate(List<?> list) {
+ boolean hasDate = false;
+ for (Object o : list) {
+ if (o instanceof Date) {
+ hasDate = true;
+ break;
+ }
+ }
+ return hasDate;
+ }
+
+ /** Convert Date objects to ISO formatted strings */
+ protected Object constructDateStr(Object field) {
+ if (field instanceof Date) {
+ field =
+ DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochMilli(((Date) field).getTime()));
+ }
+ return field;
+ }
+
abstract void start() throws IOException;
@SuppressForbidden(reason = "Command line tool prints out to console")
@@ -356,49 +409,12 @@ public class ExportTool extends ToolBase {
@Override
public synchronized void accept(SolrDocument doc) throws IOException {
charArr.reset();
- Map<String, Object> m = CollectionUtil.newLinkedHashMap(doc.size());
- doc.forEach(
- (s, field) -> {
- if (s.equals("_version_") || s.equals("_roor_")) return;
- if (field instanceof List) {
- if (((List<?>) field).size() == 1) {
- field = ((List<?>) field).get(0);
- }
- }
- field = constructDateStr(field);
- if (field instanceof List<?> list) {
- if (hasdate(list)) {
- ArrayList<Object> listCopy = new ArrayList<>(list.size());
- for (Object o : list) listCopy.add(constructDateStr(o));
- field = listCopy;
- }
- }
- m.put(s, field);
- });
+ Map<String, Object> m = processDocument(doc);
jsonWriter.write(m);
writer.write(charArr.getArray(), charArr.getStart(), charArr.getEnd());
writer.append('\n');
super.accept(doc);
}
-
- private boolean hasdate(List<?> list) {
- boolean hasDate = false;
- for (Object o : list) {
- if (o instanceof Date) {
- hasDate = true;
- break;
- }
- }
- return hasDate;
- }
-
- private Object constructDateStr(Object field) {
- if (field instanceof Date) {
- field =
- DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochMilli(((Date) field).getTime()));
- }
- return field;
- }
}
static class JsonSink extends DocsSink {
@@ -435,25 +451,7 @@ public class ExportTool extends ToolBase {
@Override
public synchronized void accept(SolrDocument doc) throws IOException {
charArr.reset();
- Map<String, Object> m = CollectionUtil.newLinkedHashMap(doc.size());
- doc.forEach(
- (s, field) -> {
- if (s.equals("_version_") || s.equals("_roor_")) return;
- if (field instanceof List) {
- if (((List<?>) field).size() == 1) {
- field = ((List<?>) field).get(0);
- }
- }
- field = constructDateStr(field);
- if (field instanceof List<?> list) {
- if (hasdate(list)) {
- ArrayList<Object> listCopy = new ArrayList<>(list.size());
- for (Object o : list) listCopy.add(constructDateStr(o));
- field = listCopy;
- }
- }
- m.put(s, field);
- });
+ Map<String, Object> m = processDocument(doc);
if (firstDoc) {
firstDoc = false;
} else {
@@ -464,25 +462,6 @@ public class ExportTool extends ToolBase {
writer.append('\n');
super.accept(doc);
}
-
- private boolean hasdate(List<?> list) {
- boolean hasDate = false;
- for (Object o : list) {
- if (o instanceof Date) {
- hasDate = true;
- break;
- }
- }
- return hasDate;
- }
-
- private Object constructDateStr(Object field) {
- if (field instanceof Date) {
- field =
- DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochMilli(((Date) field).getTime()));
- }
- return field;
- }
}
static class JavabinSink extends DocsSink {
diff --git a/solr/core/src/test/org/apache/solr/cli/TestExportTool.java b/solr/core/src/test/org/apache/solr/cli/TestExportTool.java
index f74eedbb0d8..d356c439849 100644
--- a/solr/core/src/test/org/apache/solr/cli/TestExportTool.java
+++ b/solr/core/src/test/org/apache/solr/cli/TestExportTool.java
@@ -19,6 +19,7 @@ package org.apache.solr.cli;
import java.io.FileInputStream;
import java.io.IOException;
+import java.io.InputStream;
import java.io.InputStreamReader;
import java.io.Reader;
import java.nio.charset.StandardCharsets;
@@ -28,6 +29,7 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.function.Predicate;
+import java.util.zip.GZIPInputStream;
import org.apache.lucene.tests.util.TestUtil;
import org.apache.solr.SolrTestCaseJ4;
import org.apache.solr.client.solrj.SolrClient;
@@ -50,6 +52,31 @@ import org.junit.Test;
@SolrTestCaseJ4.SuppressSSL
public class TestExportTool extends SolrCloudTestCase {
+ public void testOutputFormatToFileNameMapping() {
+
+ ToolRuntime runtime = new CLITestHelper.TestingRuntime(false);
+ String url = "http://example:8983/solr/mycollection";
+ ExportTool.Info info = new ExportTool.MultiThreadedRunner(runtime, url, null);
+
+ info.setOutFormat(null, "json", false);
+ assertEquals("mycollection.json", info.out);
+
+ info.setOutFormat(null, "jsonl", false);
+ assertEquals("mycollection.jsonl", info.out);
+
+ info.setOutFormat(null, "javabin", false);
+ assertEquals("mycollection.javabin", info.out);
+
+ String tempFile = createTempDir() + "/myoutput.json";
+ info.setOutFormat(tempFile, "json", false);
+ assertEquals(tempFile, info.out);
+
+ // test with compression
+ tempFile = createTempDir() + "/myoutput.myoutput.json.gz";
+ info.setOutFormat(tempFile, "json", true);
+ assertEquals(tempFile, info.out);
+ }
+
@Test
public void testBasic() throws Exception {
String COLLECTION_NAME = "globalLoaderColl";
@@ -92,7 +119,8 @@ public class TestExportTool extends SolrCloudTestCase {
info.fields = "id,desc_s,a_dt";
info.exportDocs();
- assertJsonDocsCount(info, 200, record -> "2019-09-30T05:58:03Z".equals(record.get("a_dt")));
+ assertJsonLinesDocsCount(
+ info, 200, record -> "2019-09-30T05:58:03Z".equals(record.get("a_dt")));
info = new ExportTool.MultiThreadedRunner(runtime, url, null);
absolutePath =
@@ -102,7 +130,7 @@ public class TestExportTool extends SolrCloudTestCase {
info.fields = "id,desc_s";
info.exportDocs();
- assertJsonDocsCount(info, 1000, null);
+ assertJsonLinesDocsCount(info, 1000, null);
info = new ExportTool.MultiThreadedRunner(runtime, url, null);
absolutePath =
@@ -131,7 +159,7 @@ public class TestExportTool extends SolrCloudTestCase {
info.fields = "id,desc_s";
info.exportDocs();
- assertJsonDocsCount2(info, 200);
+ assertJsonDocsCount(info, 200);
info = new ExportTool.MultiThreadedRunner(runtime, url, null);
absolutePath =
@@ -141,7 +169,7 @@ public class TestExportTool extends SolrCloudTestCase {
info.fields = "id,desc_s";
info.exportDocs();
- assertJsonDocsCount2(info, 1000);
+ assertJsonDocsCount(info, 1000);
} finally {
cluster.shutdown();
@@ -197,11 +225,9 @@ public class TestExportTool extends SolrCloudTestCase {
assertEquals(docCount, totalDocsFromCores);
ToolRuntime runtime = new CLITestHelper.TestingRuntime(false);
- ExportTool.MultiThreadedRunner info;
- String absolutePath;
- info = new ExportTool.MultiThreadedRunner(runtime, url, null);
- absolutePath =
+ ExportTool.MultiThreadedRunner info = new ExportTool.MultiThreadedRunner(runtime, url, null);
+ String absolutePath =
baseDir.resolve(COLLECTION_NAME + random().nextInt(100000) + ".javabin").toString();
info.setOutFormat(absolutePath, "javabin", false);
info.setLimit("-1");
@@ -211,6 +237,7 @@ public class TestExportTool extends SolrCloudTestCase {
assertEquals(
e.getValue().longValue(), info.corehandlers.get(e.getKey()).receivedDocs.get());
}
+
info = new ExportTool.MultiThreadedRunner(runtime, url, null);
absolutePath =
baseDir.resolve(COLLECTION_NAME + random().nextInt(100000) + ".jsonl").toString();
@@ -280,7 +307,7 @@ public class TestExportTool extends SolrCloudTestCase {
}
}
- private void assertJsonDocsCount2(ExportTool.Info info, int expected) {
+ private void assertJsonDocsCount(ExportTool.Info info, int expected) {
assertTrue(
"" + info.docsWritten.get() + " expected " + expected, info.docsWritten.get() >= expected);
}
@@ -310,4 +337,34 @@ public class TestExportTool extends SolrCloudTestCase {
rdr.close();
}
}
+
+ private void assertJsonLinesDocsCount(
+ ExportTool.Info info, int expected, Predicate<Map<String, Object>> predicate)
+ throws IOException {
+ assertTrue(
+ "" + info.docsWritten.get() + " expected " + expected, info.docsWritten.get() >= expected);
+
+ JsonRecordReader jsonReader;
+ Reader rdr;
+ jsonReader = JsonRecordReader.getInst("/", List.of("$FQN:/**"));
+ InputStream is = new FileInputStream(info.out);
+ if (info.compress) {
+ is = new GZIPInputStream(is);
+ }
+ rdr = new InputStreamReader(is, StandardCharsets.UTF_8);
+ try {
+ int[] count = new int[] {0};
+ jsonReader.streamRecords(
+ rdr,
+ (record, path) -> {
+ if (predicate != null) {
+ assertTrue(predicate.test(record));
+ }
+ count[0]++;
+ });
+ assertTrue(count[0] >= expected);
+ } finally {
+ rdr.close();
+ }
+ }
}
diff --git a/solr/packaging/test/test_export.bats b/solr/packaging/test/test_export.bats
index cb089ecd5fe..f497fde4839 100644
--- a/solr/packaging/test/test_export.bats
+++ b/solr/packaging/test/test_export.bats
@@ -45,9 +45,8 @@ teardown() {
assert [ -e techproducts.javabin ]
rm techproducts.javabin
- # old pattern of putting a suffix on the output that controlled the format no longer supported ;-).
- run solr export --solr-url http://localhost:${SOLR_PORT} -c techproducts --query "*:* -id:test" --output "${BATS_TEST_TMPDIR}/output.javabin"
- assert [ -e ${BATS_TEST_TMPDIR}/output.javabin.json ]
+ run solr export --solr-url http://localhost:${SOLR_PORT} -c techproducts --query "*:* -id:test" --output "${BATS_TEST_TMPDIR}/output.json"
+ assert [ -e ${BATS_TEST_TMPDIR}/output.json ]
run solr export --solr-url http://localhost:${SOLR_PORT} -c techproducts --query "*:* -id:test" --output "${BATS_TEST_TMPDIR}"
assert [ -e ${BATS_TEST_TMPDIR}/techproducts.json ]
diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/solr-control-script-reference.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/solr-control-script-reference.adoc
index 48fe8e18347..842d294b06e 100644
--- a/solr/solr-ref-guide/modules/deployment-guide/pages/solr-control-script-reference.adoc
+++ b/solr/solr-ref-guide/modules/deployment-guide/pages/solr-control-script-reference.adoc
@@ -1619,12 +1619,12 @@ Examples of this command:
=== Exporting Documents to a File
-The `export` command will allow you to export documents from a collection in JSON, JSON with Lines, or Javabin format.
-All documents can be exported, or only those that match a query.
+The `export` command will allow you to export documents from a collection in JSON, https://jsonlines.org/[JSON Lines], or Javabin format.
+All documents can be exported, or only those that match a query. You may need to wrap some parameters with quotes.
NOTE: This hasn't been tested with nested child documents and your results will vary.
-NOTE: The `export` command only works with in a Solr running in cloud mode.
+NOTE: The `export` command only works with Solr running in cloud mode.
`bin/solr export [options]`
@@ -1661,9 +1661,11 @@ Name of the collection to run an export against.
|Optional |Default: `json`
|===
+
-The file format of the export, `json`, `jsonl`, or `javabin`.
-Choosing `javabin` exports in the native Solr format, and is compact and fast to import.
-`jsonl` is the Json with Lines format, learn more at https://jsonlines.org/.
+The file format of the export, `json` (default) or `jsonl` or `javabin`, this also specifies the file extension to be used.
+`json` and `jsonl` both export documents in the same format as using `wt=json`. The `json` output file is suitable for
+immediately posting back to solr via the `/update/json` endpoint. `jsonl` outputs each Solr document on it's own line,
+and it useful for parallel processing tasks. Learn more at https://jsonlines.org/.
+Choosing `javabin` exports in the native binary Solr format and is compact and faster to import.
`--output <path>`::
+
@@ -1673,6 +1675,7 @@ Choosing `javabin` exports in the native Solr format, and is compact and fast to
|===
+
Either the path to the directory for the exported data to be written to, or a specific file to be written out.
+If the file name ends with `.gz` the output will be compressed into a .gz file.
+
If only a directory is specified then the file will be created with the name of the collection, as in `<collection>.json`.
@@ -1692,7 +1695,7 @@ If you specify `--compress` then the resulting outputting file with will be gzip
|Optional |Default: `\*:*`
|===
+
-A custom query.
+A custom query to select documents for exporting.
The default is `\*:*` which will export all documents.
`--fields <fields>`::
@@ -1727,7 +1730,7 @@ This parameter is unnecessary if `SOLR_AUTH_TYPE` is defined in `solr.in.sh` or
*Examples*
-Export all documents from a collection `gettingstarted`:
+Export all documents from a collection `gettingstarted` into a file called `gettingstarted.json`:
[source,bash]
bin/solr export --solr-url http://localhost:8983 -c gettingstarted --limit -1
@@ -1741,7 +1744,12 @@ bin/solr export --solr-url http://localhost:8983 -c gettingstarted --limit -1 --
=== Importing Documents into a Collection
-Once you have exported documents in a file, you can use the xref:indexing-guide:indexing-with-update-handlers.adoc[/update request handler] to import them to a new Solr collection.
+Once you have exported documents in a file, you can use the xref:indexing-guide:indexing-with-update-handlers.adoc#json-formatted-index-updates[/update request handler] to import them to a new Solr collection.
+Notice the different endpoints used depending on the format.
+
+*Example: import `json` files*
+
+`curl -X POST --header "Content-Type: application/json" -d @gettingstarted.json http://localhost:8983/solr/gettingstarted/update/json?commit=true`
*Example: import `json` files*
@@ -1763,7 +1771,7 @@ Now import the data with either of these methods:
[,console]
----
-$ curl -X POST -d @gettingstarted.json 'http://localhost:8983/solr/test_collection/update/json/docs?commit=true'
+$ curl -X POST --header "Content-Type: application/json" -d @gettingstarted.json 'http://localhost:8983/solr/test_collection/update/json/docs?commit=true'
----
or
[,console]