Apache Solr open-source search software
SOLR-17867: Export Tool should export regular docs cleanly in .json, .jsonl, and javabin (second take) (#2636)
Eric Pugh585b1d00a005
diff --git a/ExportTool.java.original b/ExportTool.java.originalnew file mode 100644index 00000000000..050092c2017--- /dev/null+++ b/ExportTool.java.original@@ -0,0 +1,738 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++package org.apache.solr.cli;++import static org.apache.solr.common.params.CommonParams.FL;+import static org.apache.solr.common.params.CommonParams.JAVABIN;+import static org.apache.solr.common.params.CommonParams.JSON;+import static org.apache.solr.common.params.CommonParams.Q;+import static org.apache.solr.common.params.CommonParams.SORT;+import static org.apache.solr.common.util.JavaBinCodec.SOLRINPUTDOC;++import java.io.BufferedOutputStream;+import java.io.FileOutputStream;+import java.io.IOException;+import java.io.OutputStream;+import java.io.OutputStreamWriter;+import java.io.Writer;+import java.nio.charset.StandardCharsets;+import java.nio.file.Files;+import java.nio.file.Path;+import java.time.Instant;+import java.time.format.DateTimeFormatter;+import java.util.ArrayList;+import java.util.Collections;+import java.util.Date;+import java.util.HashMap;+import java.util.List;+import java.util.Map;+import java.util.Objects;+import java.util.Set;+import java.util.concurrent.ArrayBlockingQueue;+import java.util.concurrent.CountDownLatch;+import java.util.concurrent.ExecutorService;+import java.util.concurrent.TimeUnit;+import java.util.concurrent.atomic.AtomicLong;+import java.util.function.BiConsumer;+import java.util.function.Consumer;+import java.util.zip.GZIPOutputStream;+import org.apache.commons.cli.CommandLine;+import org.apache.commons.cli.Option;+import org.apache.commons.cli.Options;+import org.apache.lucene.util.SuppressForbidden;+import org.apache.solr.client.solrj.SolrClient;+import org.apache.solr.client.solrj.SolrQuery;+import org.apache.solr.client.solrj.SolrRequest;+import org.apache.solr.client.solrj.SolrServerException;+import org.apache.solr.client.solrj.StreamingResponseCallback;+import org.apache.solr.client.solrj.impl.CloudHttp2SolrClient;+import org.apache.solr.client.solrj.impl.CloudSolrClient;+import org.apache.solr.client.solrj.impl.ClusterStateProvider;+import org.apache.solr.client.solrj.impl.Http2SolrClient;+import org.apache.solr.client.solrj.impl.StreamingJavaBinResponseParser;+import org.apache.solr.client.solrj.request.GenericSolrRequest;+import org.apache.solr.client.solrj.request.QueryRequest;+import org.apache.solr.common.SolrDocument;+import org.apache.solr.common.SolrDocumentList;+import org.apache.solr.common.cloud.DocCollection;+import org.apache.solr.common.cloud.Replica;+import org.apache.solr.common.cloud.Slice;+import org.apache.solr.common.params.CommonParams;+import org.apache.solr.common.params.CursorMarkParams;+import org.apache.solr.common.params.ModifiableSolrParams;+import org.apache.solr.common.params.SolrParams;+import org.apache.solr.common.util.CollectionUtil;+import org.apache.solr.common.util.ExecutorUtil;+import org.apache.solr.common.util.JavaBinCodec;+import org.apache.solr.common.util.NamedList;+import org.apache.solr.common.util.SolrNamedThreadFactory;+import org.apache.solr.common.util.StrUtils;+import org.noggit.CharArr;+import org.noggit.JSONWriter;++/** Supports export command in the bin/solr script. */+public class ExportTool extends ToolBase {++ private static final Option COLLECTION_NAME_OPTION =+ Option.builder("c")+ .longOpt("name")+ .hasArg()+ .argName("NAME")+ .desc("Name of the collection.")+ .build();++ private static final Option OUTPUT_OPTION =+ Option.builder()+ .longOpt("output")+ .hasArg()+ .argName("PATH")+ .desc(+ "Path to output the exported data, and optionally the file name, defaults to 'collection-name'.")+ .build();++ private static final Option FORMAT_OPTION =+ Option.builder()+ .longOpt("format")+ .hasArg()+ .argName("FORMAT")+ .desc("Output format for exported docs (json, jsonl or javabin), defaulting to json.")+ .build();++ private static final Option COMPRESS_OPTION =+ Option.builder().longOpt("compress").desc("Compress the output. Defaults to false.").build();++ private static final Option LIMIT_OPTION =+ Option.builder()+ .longOpt("limit")+ .hasArg()+ .argName("#")+ .desc("Maximum number of docs to download. Default is 100, use -1 for all docs.")+ .build();++ private static final Option QUERY_OPTION =+ Option.builder()+ .longOpt("query")+ .hasArg()+ .argName("QUERY")+ .desc("A custom query, default is '*:*'.")+ .build();++ private static final Option FIELDS_OPTION =+ Option.builder()+ .longOpt("fields")+ .hasArg()+ .argName("FIELDA,FIELDB")+ .desc("Comma separated list of fields to export. By default all fields are fetched.")+ .build();++ public ExportTool(ToolRuntime runtime) {+ super(runtime);+ }++ @Override+ public String getName() {+ return "export";+ }++ @Override+ public Options getOptions() {+ return super.getOptions()+ .addOption(COLLECTION_NAME_OPTION)+ .addOption(OUTPUT_OPTION)+ .addOption(FORMAT_OPTION)+ .addOption(COMPRESS_OPTION)+ .addOption(LIMIT_OPTION)+ .addOption(QUERY_OPTION)+ .addOption(FIELDS_OPTION)+ .addOption(CommonCLIOptions.SOLR_URL_OPTION)+ .addOption(CommonCLIOptions.CREDENTIALS_OPTION);+ }++ public abstract static class Info {+ final ToolRuntime runtime;++ String baseurl;+ String credentials;+ String format;+ boolean compress;+ String query;+ String coll;+ String out;+ String fields;+ long limit = 100;+ AtomicLong docsWritten = new AtomicLong(0);+ int bufferSize = 1024 * 1024;+ String uniqueKey;+ CloudSolrClient solrClient;+ DocsSink sink;++ public Info(ToolRuntime runtime, String url, String credentials) {+ this.runtime = runtime;+ setUrl(url);+ setCredentials(credentials);+ setOutFormat(null, "jsonl", false);+ }++ public void setUrl(String url) {+ int idx = url.lastIndexOf('/');+ baseurl = url.substring(0, idx);+ coll = url.substring(idx + 1);+ query = "*:*";+ }++ public void setCredentials(String credentials) {+ this.credentials = credentials;+ }++ public void setLimit(String maxDocsStr) {+ limit = Long.parseLong(maxDocsStr);+ if (limit == -1) limit = Long.MAX_VALUE;+ }++ public void setOutFormat(String out, String format, boolean compress) {+ this.compress = compress;+ this.format = format;+ this.out = out;+ if (this.format == null) {+ this.format = JSON;+ }+ if (!formats.contains(this.format)) {+ throw new IllegalArgumentException("format must be one of: " + formats);+ }+ if (this.out == null) {+ this.out = coll;+ } else if (Files.isDirectory(Path.of(this.out))) {+ this.out = this.out + "/" + coll;+ }+ this.out = this.out + '.' + this.format;+ if (compress) {+ this.out = this.out + ".gz";+ }+ }++ DocsSink getSink() {+ DocsSink docSink = null;+ switch (format) {+ case JAVABIN:+ docSink = new JavabinSink(this);+ break;+ case JSON:+ docSink = new JsonSink(this);+ break;+ case "jsonl":+ docSink = new JsonWithLinesSink(this);+ break;+ }+ return docSink;+ }++ abstract void exportDocs() throws Exception;++ void fetchUniqueKey() throws SolrServerException, IOException {+ Http2SolrClient.Builder builder =+ new Http2SolrClient.Builder().withOptionalBasicAuthCredentials(credentials);++ solrClient =+ new CloudHttp2SolrClient.Builder(Collections.singletonList(baseurl))+ .withInternalClientBuilder(builder)+ .build();+ NamedList<Object> response =+ solrClient.request(+ new GenericSolrRequest(+ SolrRequest.METHOD.GET,+ "/schema/uniquekey",+ SolrParams.of("collection", coll))+ .setRequiresCollection(true));+ uniqueKey = (String) response.get("uniqueKey");+ }++ public static StreamingResponseCallback getStreamer(Consumer<SolrDocument> sink) {+ return new StreamingResponseCallback() {+ @Override+ public void streamSolrDocument(SolrDocument doc) {+ try {+ sink.accept(doc);+ } catch (Exception e) {+ throw new RuntimeException(e);+ }+ }++ @Override+ public void streamDocListInfo(long numFound, long start, Float maxScore) {}+ };+ }+ }++ static Set<String> formats = Set.of(JAVABIN, JSON, "jsonl");++ @Override+ public void runImpl(CommandLine cli) throws Exception {+ String url = null;+ if (cli.hasOption(CommonCLIOptions.SOLR_URL_OPTION)) {+ if (!cli.hasOption(COLLECTION_NAME_OPTION)) {+ throw new IllegalArgumentException(+ "Must specify -c / --name parameter with --solr-url to post documents.");+ }+ url = CLIUtils.normalizeSolrUrl(cli) + "/solr/" + cli.getOptionValue(COLLECTION_NAME_OPTION);++ } else {+ // think about support --zk-host someday.+ throw new IllegalArgumentException("Must specify --solr-url.");+ }+ String credentials = cli.getOptionValue(CommonCLIOptions.CREDENTIALS_OPTION);+ Info info = new MultiThreadedRunner(runtime, url, credentials);+ info.query = cli.getOptionValue(QUERY_OPTION, "*:*");++ info.setOutFormat(+ cli.getOptionValue(OUTPUT_OPTION),+ cli.getOptionValue(FORMAT_OPTION),+ cli.hasOption(COMPRESS_OPTION));+ info.fields = cli.getOptionValue(FIELDS_OPTION);+ info.setLimit(cli.getOptionValue(LIMIT_OPTION, "100"));+ info.exportDocs();+ }++ abstract static class DocsSink {+ Info info;+ OutputStream fos;++ abstract void start() throws IOException;++ @SuppressForbidden(reason = "Command line tool prints out to console")+ void accept(SolrDocument document) throws IOException {+ long count = info.docsWritten.incrementAndGet();++ if (count % 100000 == 0) {+ System.out.println("\nDOCS: " + count);+ }+ }++ void end() throws IOException {}+ }++ static class JsonWithLinesSink extends DocsSink {+ private final CharArr charArr = new CharArr(1024 * 2);+ JSONWriter jsonWriter = new JSONWriter(charArr, -1);+ private Writer writer;++ public JsonWithLinesSink(Info info) {+ this.info = info;+ }++ @Override+ public void start() throws IOException {+ fos = new FileOutputStream(info.out);+ if (info.compress) {+ fos = new GZIPOutputStream(fos);+ }+ if (info.bufferSize > 0) {+ fos = new BufferedOutputStream(fos, info.bufferSize);+ }+ writer = new OutputStreamWriter(fos, StandardCharsets.UTF_8);+ }++ @Override+ public void end() throws IOException {+ writer.flush();+ fos.flush();+ fos.close();+ }++ @Override+ public synchronized void accept(SolrDocument doc) throws IOException {+ charArr.reset();+ Map<String, Object> m = CollectionUtil.newLinkedHashMap(doc.size());+ doc.forEach(+ (s, field) -> {+ if (s.equals("_version_") || s.equals("_roor_")) return;+ if (field instanceof List) {+ if (((List<?>) field).size() == 1) {+ field = ((List<?>) field).get(0);+ }+ }+ field = constructDateStr(field);+ if (field instanceof List<?> list) {+ if (hasdate(list)) {+ ArrayList<Object> listCopy = new ArrayList<>(list.size());+ for (Object o : list) listCopy.add(constructDateStr(o));+ field = listCopy;+ }+ }+ m.put(s, field);+ });+ jsonWriter.write(m);+ writer.write(charArr.getArray(), charArr.getStart(), charArr.getEnd());+ writer.append('\n');+ super.accept(doc);+ }++ private boolean hasdate(List<?> list) {+ boolean hasDate = false;+ for (Object o : list) {+ if (o instanceof Date) {+ hasDate = true;+ break;+ }+ }+ return hasDate;+ }++ private Object constructDateStr(Object field) {+ if (field instanceof Date) {+ field =+ DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochMilli(((Date) field).getTime()));+ }+ return field;+ }+ }++ static class JsonSink extends DocsSink {+ private final CharArr charArr = new CharArr(1024 * 2);+ JSONWriter jsonWriter = new JSONWriter(charArr, -1);+ private Writer writer;+ private boolean firstDoc = true;++ public JsonSink(Info info) {+ this.info = info;+ }++ @Override+ public void start() throws IOException {+ fos = new FileOutputStream(info.out);+ if (info.compress) {+ fos = new GZIPOutputStream(fos);+ }+ if (info.bufferSize > 0) {+ fos = new BufferedOutputStream(fos, info.bufferSize);+ }+ writer = new OutputStreamWriter(fos, StandardCharsets.UTF_8);+ writer.append('[');+ }++ @Override+ public void end() throws IOException {+ writer.append(']');+ writer.flush();+ fos.flush();+ fos.close();+ }++ @Override+ public synchronized void accept(SolrDocument doc) throws IOException {+ charArr.reset();+ Map<String, Object> m = CollectionUtil.newLinkedHashMap(doc.size());+ doc.forEach(+ (s, field) -> {+ if (s.equals("_version_") || s.equals("_roor_")) return;+ if (field instanceof List) {+ if (((List<?>) field).size() == 1) {+ field = ((List<?>) field).get(0);+ }+ }+ field = constructDateStr(field);+ if (field instanceof List<?> list) {+ if (hasdate(list)) {+ ArrayList<Object> listCopy = new ArrayList<>(list.size());+ for (Object o : list) listCopy.add(constructDateStr(o));+ field = listCopy;+ }+ }+ m.put(s, field);+ });+ if (firstDoc) {+ firstDoc = false;+ } else {+ writer.append(',');+ }+ jsonWriter.write(m);+ writer.write(charArr.getArray(), charArr.getStart(), charArr.getEnd());+ writer.append('\n');+ super.accept(doc);+ }++ private boolean hasdate(List<?> list) {+ boolean hasDate = false;+ for (Object o : list) {+ if (o instanceof Date) {+ hasDate = true;+ break;+ }+ }+ return hasDate;+ }++ private Object constructDateStr(Object field) {+ if (field instanceof Date) {+ field =+ DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochMilli(((Date) field).getTime()));+ }+ return field;+ }+ }++ static class JavabinSink extends DocsSink {+ JavaBinCodec codec;++ public JavabinSink(Info info) {+ this.info = info;+ }++ @Override+ public void start() throws IOException {+ fos = new FileOutputStream(info.out);+ if (info.compress) {+ fos = new GZIPOutputStream(fos);+ }+ if (info.bufferSize > 0) {+ fos = new BufferedOutputStream(fos, info.bufferSize);+ }+ codec = new JavaBinCodec(fos, null);+ codec.writeTag(JavaBinCodec.NAMED_LST, 2);+ codec.writeStr("params");+ codec.writeNamedList(new NamedList<>());+ codec.writeStr("docs");+ codec.writeTag(JavaBinCodec.ITERATOR);+ }++ @Override+ public void end() throws IOException {+ codec.writeTag(JavaBinCodec.END);+ codec.close();+ fos.flush();+ fos.close();+ }++ private final BiConsumer<String, Object> bic =+ new BiConsumer<>() {+ @Override+ public void accept(String s, Object o) {+ try {+ if (s.equals("_version_") || s.equals("_root_")) return;+ codec.writeExternString(s);+ codec.writeVal(o);+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ }+ };++ @Override+ public synchronized void accept(SolrDocument doc) throws IOException {+ int sz = doc.size();+ if (doc.containsKey("_version_")) sz--;+ if (doc.containsKey("_root_")) sz--;+ codec.writeTag(SOLRINPUTDOC, sz);+ codec.writeFloat(1f); // document boost+ doc.forEach(bic);+ super.accept(doc);+ }+ }++ static class MultiThreadedRunner extends Info {+ ExecutorService producerThreadpool, consumerThreadpool;+ ArrayBlockingQueue<SolrDocument> queue = new ArrayBlockingQueue<>(1000);+ SolrDocument EOFDOC = new SolrDocument();+ volatile boolean failed = false;+ Map<String, CoreHandler> corehandlers = new HashMap<>();+ private final long startTime;++ @SuppressForbidden(reason = "Need to print out time")+ public MultiThreadedRunner(ToolRuntime runtime, String url, String credentials) {+ super(runtime, url, credentials);+ startTime = System.currentTimeMillis();+ }++ @Override+ @SuppressForbidden(reason = "Need to print out time")+ void exportDocs() throws Exception {+ sink = getSink();+ fetchUniqueKey();+ ClusterStateProvider stateProvider = solrClient.getClusterStateProvider();+ DocCollection coll = stateProvider.getCollection(this.coll);+ Map<String, Slice> m = coll.getSlicesMap();+ producerThreadpool =+ ExecutorUtil.newMDCAwareFixedThreadPool(+ m.size(), new SolrNamedThreadFactory("solrcli-exporter-producers"));+ consumerThreadpool =+ ExecutorUtil.newMDCAwareFixedThreadPool(+ 1, new SolrNamedThreadFactory("solrcli-exporter-consumer"));+ sink.start();+ CountDownLatch consumerlatch = new CountDownLatch(1);+ try {+ addConsumer(consumerlatch);+ addProducers(m);+ runtime.println("Number of shards : " + corehandlers.size());+ CountDownLatch producerLatch = new CountDownLatch(corehandlers.size());+ corehandlers.forEach(+ (s, coreHandler) ->+ producerThreadpool.execute(+ () -> {+ try {+ coreHandler.exportDocsFromCore();+ } catch (Exception e) {+ runtime.println("Error exporting docs from : " + s);+ }+ producerLatch.countDown();+ }));++ producerLatch.await();+ queue.offer(EOFDOC, 10, TimeUnit.SECONDS);+ consumerlatch.await();+ } finally {+ sink.end();+ solrClient.close();+ producerThreadpool.shutdownNow();+ consumerThreadpool.shutdownNow();+ if (failed) {+ try {+ Files.delete(Path.of(out));+ } catch (IOException e) {+ // ignore+ }+ }+ System.out.println(+ "\nTotal Docs exported: "+ + docsWritten.get()+ + ". Time elapsed: "+ + ((System.currentTimeMillis() - startTime) / 1000)+ + "seconds");+ }+ }++ private void addProducers(Map<String, Slice> m) {+ for (Map.Entry<String, Slice> entry : m.entrySet()) {+ Slice slice = entry.getValue();+ Replica replica = slice.getLeader();+ if (replica == null)+ replica = slice.getReplicas().iterator().next(); // get a random replica+ CoreHandler coreHandler = new CoreHandler(replica);+ corehandlers.put(replica.getCoreName(), coreHandler);+ }+ }++ private void addConsumer(CountDownLatch consumerlatch) {+ consumerThreadpool.execute(+ () -> {+ while (true) {+ SolrDocument doc;+ try {+ doc = queue.poll(30, TimeUnit.SECONDS);+ } catch (InterruptedException e) {+ runtime.println("Consumer interrupted");+ failed = true;+ break;+ }+ if (doc == EOFDOC) break;+ try {+ if (docsWritten.get() >= limit) {+ continue;+ }+ sink.accept(doc);+ } catch (Exception e) {+ runtime.println("Failed to write to file " + e.getMessage());+ failed = true;+ }+ }+ consumerlatch.countDown();+ });+ }++ class CoreHandler {+ final Replica replica;+ long expectedDocs;+ AtomicLong receivedDocs = new AtomicLong();++ CoreHandler(Replica replica) {+ this.replica = replica;+ }++ boolean exportDocsFromCore() throws IOException, SolrServerException {+ // reference the replica's node URL, not the baseUrl in scope, which could be anywhere+ try (SolrClient client = CLIUtils.getSolrClient(replica.getBaseUrl(), credentials)) {+ expectedDocs = getDocCount(replica.getCoreName(), client, query);+ QueryRequest request;+ ModifiableSolrParams params = new ModifiableSolrParams();+ params.add(Q, query);+ if (fields != null) params.add(FL, fields);+ params.add(SORT, uniqueKey + " asc");+ params.add(CommonParams.DISTRIB, "false");+ params.add(CommonParams.ROWS, "1000");+ String cursorMark = CursorMarkParams.CURSOR_MARK_START;+ Consumer<SolrDocument> wrapper =+ doc -> {+ try {+ queue.offer(doc, 10, TimeUnit.SECONDS);+ receivedDocs.incrementAndGet();+ } catch (InterruptedException e) {+ failed = true;+ runtime.println("Failed to write docs from" + e.getMessage());+ }+ };+ StreamingJavaBinResponseParser responseParser =+ new StreamingJavaBinResponseParser(getStreamer(wrapper));+ while (true) {+ if (failed) return false;+ if (docsWritten.get() > limit) return true;+ params.set(CursorMarkParams.CURSOR_MARK_PARAM, cursorMark);+ request = new QueryRequest(params);+ request.setResponseParser(responseParser);+ try {+ NamedList<Object> rsp = client.request(request, replica.getCoreName());+ String nextCursorMark = (String) rsp.get(CursorMarkParams.CURSOR_MARK_NEXT);+ if (nextCursorMark == null || Objects.equals(cursorMark, nextCursorMark)) {+ runtime.println(+ StrUtils.formatString(+ "\nExport complete from shard {0}, core {1}, docs received: {2}",+ replica.getShard(), replica.getCoreName(), receivedDocs.get()));+ if (expectedDocs != receivedDocs.get()) {+ runtime.println(+ StrUtils.formatString(+ "Could not download all docs from core {0}, docs expected: {1}, received: {2}",+ replica.getCoreName(), expectedDocs, receivedDocs.get()));+ return false;+ }+ return true;+ }+ cursorMark = nextCursorMark;+ runtime.print(".");+ } catch (SolrServerException e) {+ runtime.println(+ "Error reading from server "+ + replica.getBaseUrl()+ + "/"+ + replica.getCoreName());+ failed = true;+ return false;+ }+ }+ }+ }+ }+ }++ static long getDocCount(String coreName, SolrClient client, String query)+ throws SolrServerException, IOException {+ SolrQuery q = new SolrQuery(query);+ q.setRows(0);+ q.add("distrib", "false");+ final var request = new QueryRequest(q);+ NamedList<Object> res = client.request(request, coreName);+ SolrDocumentList sdl = (SolrDocumentList) res.get("response");+ return sdl.getNumFound();+ }+}diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 3baf170ec67..73dd813599c 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -51,6 +51,8 @@ Improvements* SOLR-17852: Migrate Schema Designer to use FileStore API instead of BlobHandler for persisting working data. (Eric Pugh)+* SOLR-17867: Export tool should properly output exported documents in json, json w/ lines, and javabin formats. (Eric Pugh)+Optimizations---------------------* SOLR-17568: The CLI bin/solr export tool now contacts the appropriate nodes directly for data instead of proxying through one.diff --git a/solr/core/src/java/org/apache/solr/cli/ExportTool.java b/solr/core/src/java/org/apache/solr/cli/ExportTool.javaindex 050092c2017..c742c00999c 100644--- a/solr/core/src/java/org/apache/solr/cli/ExportTool.java+++ b/solr/core/src/java/org/apache/solr/cli/ExportTool.java@@ -219,12 +219,20 @@ public class ExportTool extends ToolBase {} else if (Files.isDirectory(Path.of(this.out))) {this.out = this.out + "/" + coll;}- this.out = this.out + '.' + this.format;- if (compress) {+ if (!hasExtension(this.out)) {+ this.out = this.out + '.' + this.format;+ }+ if (compress & !this.out.endsWith(".gz")) {this.out = this.out + ".gz";}}+ public static boolean hasExtension(String filename) {+ return filename.contains(".json")+ || filename.contains(".jsonl")+ || filename.contains(".javabin");+ }+DocsSink getSink() {DocsSink docSink = null;switch (format) {@@ -311,6 +319,51 @@ public class ExportTool extends ToolBase {Info info;OutputStream fos;+ /** Process a SolrDocument into a Map, handling special fields and date conversion. */+ protected Map<String, Object> processDocument(SolrDocument doc) {+ Map<String, Object> m = CollectionUtil.newLinkedHashMap(doc.size());+ doc.forEach(+ (s, field) -> {+ if (s.equals("_version_") || s.equals("_roor_")) return;+ if (field instanceof List) {+ if (((List<?>) field).size() == 1) {+ field = ((List<?>) field).get(0);+ }+ }+ field = constructDateStr(field);+ if (field instanceof List<?> list) {+ if (hasDate(list)) {+ ArrayList<Object> listCopy = new ArrayList<>(list.size());+ for (Object o : list) listCopy.add(constructDateStr(o));+ field = listCopy;+ }+ }+ m.put(s, field);+ });+ return m;+ }++ /** Check if a list contains any Date objects */+ protected boolean hasDate(List<?> list) {+ boolean hasDate = false;+ for (Object o : list) {+ if (o instanceof Date) {+ hasDate = true;+ break;+ }+ }+ return hasDate;+ }++ /** Convert Date objects to ISO formatted strings */+ protected Object constructDateStr(Object field) {+ if (field instanceof Date) {+ field =+ DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochMilli(((Date) field).getTime()));+ }+ return field;+ }+abstract void start() throws IOException;@SuppressForbidden(reason = "Command line tool prints out to console")@@ -356,49 +409,12 @@ public class ExportTool extends ToolBase {@Overridepublic synchronized void accept(SolrDocument doc) throws IOException {charArr.reset();- Map<String, Object> m = CollectionUtil.newLinkedHashMap(doc.size());- doc.forEach(- (s, field) -> {- if (s.equals("_version_") || s.equals("_roor_")) return;- if (field instanceof List) {- if (((List<?>) field).size() == 1) {- field = ((List<?>) field).get(0);- }- }- field = constructDateStr(field);- if (field instanceof List<?> list) {- if (hasdate(list)) {- ArrayList<Object> listCopy = new ArrayList<>(list.size());- for (Object o : list) listCopy.add(constructDateStr(o));- field = listCopy;- }- }- m.put(s, field);- });+ Map<String, Object> m = processDocument(doc);jsonWriter.write(m);writer.write(charArr.getArray(), charArr.getStart(), charArr.getEnd());writer.append('\n');super.accept(doc);}-- private boolean hasdate(List<?> list) {- boolean hasDate = false;- for (Object o : list) {- if (o instanceof Date) {- hasDate = true;- break;- }- }- return hasDate;- }-- private Object constructDateStr(Object field) {- if (field instanceof Date) {- field =- DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochMilli(((Date) field).getTime()));- }- return field;- }}static class JsonSink extends DocsSink {@@ -435,25 +451,7 @@ public class ExportTool extends ToolBase {@Overridepublic synchronized void accept(SolrDocument doc) throws IOException {charArr.reset();- Map<String, Object> m = CollectionUtil.newLinkedHashMap(doc.size());- doc.forEach(- (s, field) -> {- if (s.equals("_version_") || s.equals("_roor_")) return;- if (field instanceof List) {- if (((List<?>) field).size() == 1) {- field = ((List<?>) field).get(0);- }- }- field = constructDateStr(field);- if (field instanceof List<?> list) {- if (hasdate(list)) {- ArrayList<Object> listCopy = new ArrayList<>(list.size());- for (Object o : list) listCopy.add(constructDateStr(o));- field = listCopy;- }- }- m.put(s, field);- });+ Map<String, Object> m = processDocument(doc);if (firstDoc) {firstDoc = false;} else {@@ -464,25 +462,6 @@ public class ExportTool extends ToolBase {writer.append('\n');super.accept(doc);}-- private boolean hasdate(List<?> list) {- boolean hasDate = false;- for (Object o : list) {- if (o instanceof Date) {- hasDate = true;- break;- }- }- return hasDate;- }-- private Object constructDateStr(Object field) {- if (field instanceof Date) {- field =- DateTimeFormatter.ISO_INSTANT.format(Instant.ofEpochMilli(((Date) field).getTime()));- }- return field;- }}static class JavabinSink extends DocsSink {diff --git a/solr/core/src/test/org/apache/solr/cli/TestExportTool.java b/solr/core/src/test/org/apache/solr/cli/TestExportTool.javaindex f74eedbb0d8..d356c439849 100644--- a/solr/core/src/test/org/apache/solr/cli/TestExportTool.java+++ b/solr/core/src/test/org/apache/solr/cli/TestExportTool.java@@ -19,6 +19,7 @@ package org.apache.solr.cli;import java.io.FileInputStream;import java.io.IOException;+import java.io.InputStream;import java.io.InputStreamReader;import java.io.Reader;import java.nio.charset.StandardCharsets;@@ -28,6 +29,7 @@ import java.util.HashMap;import java.util.List;import java.util.Map;import java.util.function.Predicate;+import java.util.zip.GZIPInputStream;import org.apache.lucene.tests.util.TestUtil;import org.apache.solr.SolrTestCaseJ4;import org.apache.solr.client.solrj.SolrClient;@@ -50,6 +52,31 @@ import org.junit.Test;@SolrTestCaseJ4.SuppressSSLpublic class TestExportTool extends SolrCloudTestCase {+ public void testOutputFormatToFileNameMapping() {++ ToolRuntime runtime = new CLITestHelper.TestingRuntime(false);+ String url = "http://example:8983/solr/mycollection";+ ExportTool.Info info = new ExportTool.MultiThreadedRunner(runtime, url, null);++ info.setOutFormat(null, "json", false);+ assertEquals("mycollection.json", info.out);++ info.setOutFormat(null, "jsonl", false);+ assertEquals("mycollection.jsonl", info.out);++ info.setOutFormat(null, "javabin", false);+ assertEquals("mycollection.javabin", info.out);++ String tempFile = createTempDir() + "/myoutput.json";+ info.setOutFormat(tempFile, "json", false);+ assertEquals(tempFile, info.out);++ // test with compression+ tempFile = createTempDir() + "/myoutput.myoutput.json.gz";+ info.setOutFormat(tempFile, "json", true);+ assertEquals(tempFile, info.out);+ }+@Testpublic void testBasic() throws Exception {String COLLECTION_NAME = "globalLoaderColl";@@ -92,7 +119,8 @@ public class TestExportTool extends SolrCloudTestCase {info.fields = "id,desc_s,a_dt";info.exportDocs();- assertJsonDocsCount(info, 200, record -> "2019-09-30T05:58:03Z".equals(record.get("a_dt")));+ assertJsonLinesDocsCount(+ info, 200, record -> "2019-09-30T05:58:03Z".equals(record.get("a_dt")));info = new ExportTool.MultiThreadedRunner(runtime, url, null);absolutePath =@@ -102,7 +130,7 @@ public class TestExportTool extends SolrCloudTestCase {info.fields = "id,desc_s";info.exportDocs();- assertJsonDocsCount(info, 1000, null);+ assertJsonLinesDocsCount(info, 1000, null);info = new ExportTool.MultiThreadedRunner(runtime, url, null);absolutePath =@@ -131,7 +159,7 @@ public class TestExportTool extends SolrCloudTestCase {info.fields = "id,desc_s";info.exportDocs();- assertJsonDocsCount2(info, 200);+ assertJsonDocsCount(info, 200);info = new ExportTool.MultiThreadedRunner(runtime, url, null);absolutePath =@@ -141,7 +169,7 @@ public class TestExportTool extends SolrCloudTestCase {info.fields = "id,desc_s";info.exportDocs();- assertJsonDocsCount2(info, 1000);+ assertJsonDocsCount(info, 1000);} finally {cluster.shutdown();@@ -197,11 +225,9 @@ public class TestExportTool extends SolrCloudTestCase {assertEquals(docCount, totalDocsFromCores);ToolRuntime runtime = new CLITestHelper.TestingRuntime(false);- ExportTool.MultiThreadedRunner info;- String absolutePath;- info = new ExportTool.MultiThreadedRunner(runtime, url, null);- absolutePath =+ ExportTool.MultiThreadedRunner info = new ExportTool.MultiThreadedRunner(runtime, url, null);+ String absolutePath =baseDir.resolve(COLLECTION_NAME + random().nextInt(100000) + ".javabin").toString();info.setOutFormat(absolutePath, "javabin", false);info.setLimit("-1");@@ -211,6 +237,7 @@ public class TestExportTool extends SolrCloudTestCase {assertEquals(e.getValue().longValue(), info.corehandlers.get(e.getKey()).receivedDocs.get());}+info = new ExportTool.MultiThreadedRunner(runtime, url, null);absolutePath =baseDir.resolve(COLLECTION_NAME + random().nextInt(100000) + ".jsonl").toString();@@ -280,7 +307,7 @@ public class TestExportTool extends SolrCloudTestCase {}}- private void assertJsonDocsCount2(ExportTool.Info info, int expected) {+ private void assertJsonDocsCount(ExportTool.Info info, int expected) {assertTrue("" + info.docsWritten.get() + " expected " + expected, info.docsWritten.get() >= expected);}@@ -310,4 +337,34 @@ public class TestExportTool extends SolrCloudTestCase {rdr.close();}}++ private void assertJsonLinesDocsCount(+ ExportTool.Info info, int expected, Predicate<Map<String, Object>> predicate)+ throws IOException {+ assertTrue(+ "" + info.docsWritten.get() + " expected " + expected, info.docsWritten.get() >= expected);++ JsonRecordReader jsonReader;+ Reader rdr;+ jsonReader = JsonRecordReader.getInst("/", List.of("$FQN:/**"));+ InputStream is = new FileInputStream(info.out);+ if (info.compress) {+ is = new GZIPInputStream(is);+ }+ rdr = new InputStreamReader(is, StandardCharsets.UTF_8);+ try {+ int[] count = new int[] {0};+ jsonReader.streamRecords(+ rdr,+ (record, path) -> {+ if (predicate != null) {+ assertTrue(predicate.test(record));+ }+ count[0]++;+ });+ assertTrue(count[0] >= expected);+ } finally {+ rdr.close();+ }+ }}diff --git a/solr/packaging/test/test_export.bats b/solr/packaging/test/test_export.batsindex cb089ecd5fe..f497fde4839 100644--- a/solr/packaging/test/test_export.bats+++ b/solr/packaging/test/test_export.bats@@ -45,9 +45,8 @@ teardown() {assert [ -e techproducts.javabin ]rm techproducts.javabin- # old pattern of putting a suffix on the output that controlled the format no longer supported ;-).- run solr export --solr-url http://localhost:${SOLR_PORT} -c techproducts --query "*:* -id:test" --output "${BATS_TEST_TMPDIR}/output.javabin"- assert [ -e ${BATS_TEST_TMPDIR}/output.javabin.json ]+ run solr export --solr-url http://localhost:${SOLR_PORT} -c techproducts --query "*:* -id:test" --output "${BATS_TEST_TMPDIR}/output.json"+ assert [ -e ${BATS_TEST_TMPDIR}/output.json ]run solr export --solr-url http://localhost:${SOLR_PORT} -c techproducts --query "*:* -id:test" --output "${BATS_TEST_TMPDIR}"assert [ -e ${BATS_TEST_TMPDIR}/techproducts.json ]diff --git a/solr/solr-ref-guide/modules/deployment-guide/pages/solr-control-script-reference.adoc b/solr/solr-ref-guide/modules/deployment-guide/pages/solr-control-script-reference.adocindex 48fe8e18347..842d294b06e 100644--- a/solr/solr-ref-guide/modules/deployment-guide/pages/solr-control-script-reference.adoc+++ b/solr/solr-ref-guide/modules/deployment-guide/pages/solr-control-script-reference.adoc@@ -1619,12 +1619,12 @@ Examples of this command:=== Exporting Documents to a File-The `export` command will allow you to export documents from a collection in JSON, JSON with Lines, or Javabin format.-All documents can be exported, or only those that match a query.+The `export` command will allow you to export documents from a collection in JSON, https://jsonlines.org/[JSON Lines], or Javabin format.+All documents can be exported, or only those that match a query. You may need to wrap some parameters with quotes.NOTE: This hasn't been tested with nested child documents and your results will vary.-NOTE: The `export` command only works with in a Solr running in cloud mode.+NOTE: The `export` command only works with Solr running in cloud mode.`bin/solr export [options]`@@ -1661,9 +1661,11 @@ Name of the collection to run an export against.|Optional |Default: `json`|===+-The file format of the export, `json`, `jsonl`, or `javabin`.-Choosing `javabin` exports in the native Solr format, and is compact and fast to import.-`jsonl` is the Json with Lines format, learn more at https://jsonlines.org/.+The file format of the export, `json` (default) or `jsonl` or `javabin`, this also specifies the file extension to be used.+`json` and `jsonl` both export documents in the same format as using `wt=json`. The `json` output file is suitable for+immediately posting back to solr via the `/update/json` endpoint. `jsonl` outputs each Solr document on it's own line,+and it useful for parallel processing tasks. Learn more at https://jsonlines.org/.+Choosing `javabin` exports in the native binary Solr format and is compact and faster to import.`--output <path>`::+@@ -1673,6 +1675,7 @@ Choosing `javabin` exports in the native Solr format, and is compact and fast to|===+Either the path to the directory for the exported data to be written to, or a specific file to be written out.+If the file name ends with `.gz` the output will be compressed into a .gz file.+If only a directory is specified then the file will be created with the name of the collection, as in `<collection>.json`.@@ -1692,7 +1695,7 @@ If you specify `--compress` then the resulting outputting file with will be gzip|Optional |Default: `\*:*`|===+-A custom query.+A custom query to select documents for exporting.The default is `\*:*` which will export all documents.`--fields <fields>`::@@ -1727,7 +1730,7 @@ This parameter is unnecessary if `SOLR_AUTH_TYPE` is defined in `solr.in.sh` or*Examples*-Export all documents from a collection `gettingstarted`:+Export all documents from a collection `gettingstarted` into a file called `gettingstarted.json`:[source,bash]bin/solr export --solr-url http://localhost:8983 -c gettingstarted --limit -1@@ -1741,7 +1744,12 @@ bin/solr export --solr-url http://localhost:8983 -c gettingstarted --limit -1 --=== Importing Documents into a Collection-Once you have exported documents in a file, you can use the xref:indexing-guide:indexing-with-update-handlers.adoc[/update request handler] to import them to a new Solr collection.+Once you have exported documents in a file, you can use the xref:indexing-guide:indexing-with-update-handlers.adoc#json-formatted-index-updates[/update request handler] to import them to a new Solr collection.+Notice the different endpoints used depending on the format.++*Example: import `json` files*++`curl -X POST --header "Content-Type: application/json" -d @gettingstarted.json http://localhost:8983/solr/gettingstarted/update/json?commit=true`*Example: import `json` files*@@ -1763,7 +1771,7 @@ Now import the data with either of these methods:[,console]-----$ curl -X POST -d @gettingstarted.json 'http://localhost:8983/solr/test_collection/update/json/docs?commit=true'+$ curl -X POST --header "Content-Type: application/json" -d @gettingstarted.json 'http://localhost:8983/solr/test_collection/update/json/docs?commit=true'----or[,console]