Apache Solr open-source search software
SOLR-16812: Support CBOR format for update/query (#1655)
Noble Paulfaf5d3cb48e9
diff --git a/solr/core/build.gradle b/solr/core/build.gradleindex 9d8d164a00e..d06f79132ef 100644--- a/solr/core/build.gradle+++ b/solr/core/build.gradle@@ -120,6 +120,7 @@ dependencies {implementation 'com.fasterxml.jackson.core:jackson-core'implementation 'com.fasterxml.jackson.core:jackson-databind'implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-smile'+ implementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-cbor'implementation 'org.apache.httpcomponents:httpclient'implementation 'org.apache.httpcomponents:httpcore'diff --git a/solr/core/src/java/org/apache/solr/core/SolrCore.java b/solr/core/src/java/org/apache/solr/core/SolrCore.javaindex 277e8894fda..661e9f1e29f 100644--- a/solr/core/src/java/org/apache/solr/core/SolrCore.java+++ b/solr/core/src/java/org/apache/solr/core/SolrCore.java@@ -124,6 +124,7 @@ import org.apache.solr.request.SolrRequestHandler;import org.apache.solr.request.SolrRequestInfo;import org.apache.solr.response.BinaryResponseWriter;import org.apache.solr.response.CSVResponseWriter;+import org.apache.solr.response.CborResponseWriter;import org.apache.solr.response.GeoJSONResponseWriter;import org.apache.solr.response.GraphMLResponseWriter;import org.apache.solr.response.JacksonJsonWriter;@@ -3034,6 +3035,7 @@ public class SolrCore implements SolrInfoBean, Closeable {m.put("ruby", new RubyResponseWriter());m.put("raw", new RawResponseWriter());m.put(CommonParams.JAVABIN, new BinaryResponseWriter());+ m.put("cbor", new CborResponseWriter());m.put("csv", new CSVResponseWriter());m.put("schema.xml", new SchemaXmlResponseWriter());m.put("smile", new SmileResponseWriter());diff --git a/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java b/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.javaindex 368b40cdb55..805a712e0a5 100644--- a/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java+++ b/solr/core/src/java/org/apache/solr/handler/UpdateRequestHandler.java@@ -32,6 +32,7 @@ import org.apache.solr.common.params.UpdateParams;import org.apache.solr.common.util.ContentStream;import org.apache.solr.common.util.NamedList;import org.apache.solr.handler.loader.CSVLoader;+import org.apache.solr.handler.loader.CborLoader;import org.apache.solr.handler.loader.ContentStreamLoader;import org.apache.solr.handler.loader.JavabinLoader;import org.apache.solr.handler.loader.JsonLoader;@@ -146,6 +147,7 @@ public class UpdateRequestHandler extends ContentStreamHandlerBaseregistry.put("application/json", new JsonLoader().init(p));registry.put("application/csv", new CSVLoader().init(p));registry.put("application/javabin", new JavabinLoader(instance).init(p));+ registry.put("application/cbor", CborLoader.createLoader(p));registry.put("text/csv", registry.get("application/csv"));registry.put("text/xml", registry.get("application/xml"));registry.put("text/json", registry.get("application/json"));@@ -154,6 +156,7 @@ public class UpdateRequestHandler extends ContentStreamHandlerBasepathVsLoaders.put(DOC_PATH, registry.get("application/json"));pathVsLoaders.put(CSV_PATH, registry.get("application/csv"));pathVsLoaders.put(BIN_PATH, registry.get("application/javabin"));+ pathVsLoaders.put(CBOR_PATH, registry.get("application/cbor"));return registry;}@@ -184,4 +187,5 @@ public class UpdateRequestHandler extends ContentStreamHandlerBasepublic static final String JSON_PATH = "/update/json";public static final String CSV_PATH = "/update/csv";public static final String BIN_PATH = "/update/bin";+ public static final String CBOR_PATH = "/update/cbor";}diff --git a/solr/core/src/java/org/apache/solr/handler/loader/CborLoader.java b/solr/core/src/java/org/apache/solr/handler/loader/CborLoader.javanew file mode 100644index 00000000000..fd8c269d7cc--- /dev/null+++ b/solr/core/src/java/org/apache/solr/handler/loader/CborLoader.java@@ -0,0 +1,154 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.handler.loader;++import com.fasterxml.jackson.core.JsonToken;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.dataformat.cbor.CBORFactory;+import com.fasterxml.jackson.dataformat.cbor.CBORParser;+import java.io.IOException;+import java.io.InputStream;+import java.util.ArrayList;+import java.util.List;+import java.util.function.Consumer;+import org.apache.solr.common.SolrInputDocument;+import org.apache.solr.common.params.SolrParams;+import org.apache.solr.common.params.UpdateParams;+import org.apache.solr.common.util.ContentStream;+import org.apache.solr.request.SolrQueryRequest;+import org.apache.solr.response.SolrQueryResponse;+import org.apache.solr.update.AddUpdateCommand;+import org.apache.solr.update.processor.UpdateRequestProcessor;++/**+ * This class can load a single document or a stream of documents in CBOR format this is equivalent+ * of loading a single json documet or an array of json documents+ */+public class CborLoader {+ final CBORFactory cborFactory;+ private final Consumer<SolrInputDocument> sink;++ public CborLoader(CBORFactory cborFactory, Consumer<SolrInputDocument> sink) {+ this.cborFactory = cborFactory == null ? new CBORFactory() : cborFactory;+ this.sink = sink;+ }++ public void stream(InputStream is) throws IOException {+ ObjectMapper mapper = new ObjectMapper(new CBORFactory());+ try (CBORParser parser = (CBORParser) mapper.getFactory().createParser(is)) {+ JsonToken t;+ while ((t = parser.nextToken()) != null) {++ if (t == JsonToken.START_ARRAY) {+ // this is an array of docs+ t = parser.nextToken();+ if (t == JsonToken.START_OBJECT) {+ handleDoc(parser);+ }++ } else if (t == JsonToken.START_OBJECT) {+ // this is just a single doc+ handleDoc(parser);+ }+ }+ }+ }++ private void handleDoc(CBORParser p) throws IOException {+ SolrInputDocument doc = new SolrInputDocument();+ for (; ; ) {+ JsonToken t = p.nextToken();+ if (t == JsonToken.END_OBJECT) {+ if (!doc.isEmpty()) {+ sink.accept(doc);+ }+ return;+ }+ String name;+ if (t == JsonToken.FIELD_NAME) {+ name = p.getCurrentName();+ t = p.nextToken();+ if (t == JsonToken.START_ARRAY) {++ List<Object> l = new ArrayList<>();+ for (; ; ) {+ t = p.nextToken();+ if (t == JsonToken.END_ARRAY) break;+ else {+ l.add(readVal(t, p));+ }+ }+ if (!l.isEmpty()) {+ doc.addField(name, l);+ }++ } else {+ doc.addField(name, readVal(t, p));+ }+ }+ }+ }++ private Object readVal(JsonToken t, CBORParser p) throws IOException {+ if (t == JsonToken.VALUE_NULL) {+ return null;+ }+ if (t == JsonToken.VALUE_STRING) {+ return p.getValueAsString();+ }+ if (t == JsonToken.VALUE_TRUE) {+ return Boolean.TRUE;+ }+ if (t == JsonToken.VALUE_FALSE) {+ return Boolean.FALSE;+ }+ if (t == JsonToken.VALUE_NUMBER_INT || t == JsonToken.VALUE_NUMBER_FLOAT) {+ return p.getNumberValue();+ }+ throw new RuntimeException("Unknown type :" + t);+ }++ public static ContentStreamLoader createLoader(SolrParams p) {+ CBORFactory factory = new CBORFactory();+ return new ContentStreamLoader() {+ @Override+ public void load(+ SolrQueryRequest req,+ SolrQueryResponse rsp,+ ContentStream stream,+ UpdateRequestProcessor processor)+ throws IOException {+ int commitWithin = req.getParams().getInt(UpdateParams.COMMIT_WITHIN, -1);+ boolean overwrite = req.getParams().getBool(UpdateParams.OVERWRITE, true);+ new CborLoader(+ factory,+ doc -> {+ AddUpdateCommand add = new AddUpdateCommand(req);+ add.commitWithin = commitWithin;+ add.solrDoc = doc;+ add.overwrite = overwrite;+ try {+ processor.processAdd(add);+ } catch (IOException e) {+ throw new RuntimeException(e);+ }+ })+ .stream(stream.getStream());+ }+ }.init(p);+ }+}diff --git a/solr/core/src/java/org/apache/solr/response/CborResponseWriter.java b/solr/core/src/java/org/apache/solr/response/CborResponseWriter.javanew file mode 100644index 00000000000..8f52093cac0--- /dev/null+++ b/solr/core/src/java/org/apache/solr/response/CborResponseWriter.java@@ -0,0 +1,193 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.response;++import com.fasterxml.jackson.dataformat.cbor.CBORFactory;+import com.fasterxml.jackson.dataformat.cbor.CBORGenerator;+import java.io.IOException;+import java.io.OutputStream;+import java.math.BigDecimal;+import java.math.BigInteger;+import org.apache.solr.client.solrj.impl.BinaryResponseParser;+import org.apache.solr.common.util.NamedList;+import org.apache.solr.request.SolrQueryRequest;++/**+ * A response writer impl that can write results in CBOR (cbor.io) format when wt=cbor. It uses the+ * jackson library to write the stream out+ */+public class CborResponseWriter extends BinaryResponseWriter {+ final CBORFactory cborFactory;++ public CborResponseWriter() {+ cborFactory = new CBORFactory();+ }++ @Override+ public void write(OutputStream out, SolrQueryRequest req, SolrQueryResponse response)+ throws IOException {+ WriterImpl writer = new WriterImpl(cborFactory, out, req, response);+ writer.writeResponse();+ writer.gen.flush();+ }++ @Override+ public String getContentType(SolrQueryRequest request, SolrQueryResponse response) {+ return BinaryResponseParser.BINARY_CONTENT_TYPE;+ }++ static class WriterImpl extends JSONWriter {++ final CBORGenerator gen;++ public WriterImpl(+ CBORFactory factory, OutputStream out, SolrQueryRequest req, SolrQueryResponse rsp)+ throws IOException {+ super(null, req, rsp);+ gen = factory.createGenerator(out);+ }++ @Override+ public void writeResponse() throws IOException {+ super.writeNamedList(null, rsp.getValues());+ }++ @SuppressWarnings("rawtypes")+ public void write(NamedList nl) throws IOException {+ super.writeNamedList(null, nl);+ }++ @Override+ public void writeNumber(String name, Number val) throws IOException {+ if (val instanceof Integer) {+ gen.writeNumber(val.intValue());+ } else if (val instanceof Long) {+ gen.writeNumber(val.longValue());+ } else if (val instanceof Float) {+ gen.writeNumber(val.floatValue());+ } else if (val instanceof Double) {+ gen.writeNumber(val.doubleValue());+ } else if (val instanceof Short) {+ gen.writeNumber(val.shortValue());+ } else if (val instanceof Byte) {+ gen.writeNumber(val.byteValue());+ } else if (val instanceof BigInteger) {+ gen.writeNumber((BigInteger) val);+ } else if (val instanceof BigDecimal) {+ gen.writeNumber((BigDecimal) val);+ } else {+ gen.writeString(val.getClass().getName() + ':' + val.toString());+ // default... for debugging only+ }+ }++ @Override+ public void writeBool(String name, Boolean val) throws IOException {+ gen.writeBoolean(val);+ }++ @Override+ public void writeNull(String name) throws IOException {+ gen.writeNull();+ }++ @Override+ public void writeStr(String name, String val, boolean needsEscaping) throws IOException {+ gen.writeString(val);+ }++ @Override+ public void writeLong(String name, long val) throws IOException {+ gen.writeNumber(val);+ }++ @Override+ public void writeInt(String name, int val) throws IOException {+ gen.writeNumber(val);+ }++ @Override+ public void writeBool(String name, boolean val) throws IOException {+ gen.writeBoolean(val);+ }++ @Override+ public void writeFloat(String name, float val) throws IOException {+ gen.writeNumber(val);+ }++ @Override+ public void writeArrayCloser() throws IOException {+ gen.writeEndArray();+ }++ @Override+ public void writeArraySeparator() {+ // do nothing+ }++ @Override+ public void writeArrayOpener(int size) throws IOException, IllegalArgumentException {+ gen.writeStartArray();+ }++ @Override+ public void writeMapCloser() throws IOException {+ gen.writeEndObject();+ }++ @Override+ public void writeKey(String fname, boolean needsEscaping) throws IOException {+ gen.writeFieldName(fname);+ }++ @Override+ public void writeMapOpener(int size) throws IOException, IllegalArgumentException {+ gen.writeStartObject();+ }++ @Override+ public void writeByteArr(String name, byte[] buf, int offset, int len) throws IOException {+ gen.writeBinary(buf, offset, len);+ }++ @Override+ public void writeMapSeparator() throws IOException {+ // do nothing+ }++ @Override+ public void indent() throws IOException {+ // do nothing+ }++ @Override+ public void indent(int lev) throws IOException {+ // do nothing+ }++ @Override+ public int incLevel() {+ return 0;+ }++ @Override+ public int decLevel() {+ return 0;+ }+ }+}diff --git a/solr/core/src/resources/ImplicitPlugins.json b/solr/core/src/resources/ImplicitPlugins.jsonindex b0ee1fec3ca..c4da631c17a 100644--- a/solr/core/src/resources/ImplicitPlugins.json+++ b/solr/core/src/resources/ImplicitPlugins.json@@ -11,6 +11,13 @@"update.contentType": "application/json"}},+ "/update/cbor": {+ "useParams":"_UPDATE_CBOR",+ "class": "solr.UpdateRequestHandler",+ "invariants": {+ "update.contentType": "application/cbor"+ }+ },"/update/csv": {"useParams":"_UPDATE_CSV","class": "solr.UpdateRequestHandler",diff --git a/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java b/solr/core/src/test/org/apache/solr/core/SolrCoreTest.javaindex b7cd61e99e7..e401bcc695a 100644--- a/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java+++ b/solr/core/src/test/org/apache/solr/core/SolrCoreTest.java@@ -129,6 +129,8 @@ public class SolrCoreTest extends SolrTestCaseJ4 {++ihCount;assertEquals(pathToClassMap.get("/update/json/docs"), "solr.UpdateRequestHandler");++ihCount;+ assertEquals(pathToClassMap.get("/update/cbor"), "solr.UpdateRequestHandler");+ ++ihCount;assertEquals(pathToClassMap.get("/analysis/document"), "solr.DocumentAnalysisRequestHandler");++ihCount;assertEquals(pathToClassMap.get("/analysis/field"), "solr.FieldAnalysisRequestHandler");diff --git a/solr/core/src/test/org/apache/solr/util/TestCborDataFormat.java b/solr/core/src/test/org/apache/solr/util/TestCborDataFormat.javanew file mode 100644index 00000000000..536a21757aa--- /dev/null+++ b/solr/core/src/test/org/apache/solr/util/TestCborDataFormat.java@@ -0,0 +1,221 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.util;++import com.fasterxml.jackson.core.JsonFactory;+import com.fasterxml.jackson.core.JsonGenerator;+import com.fasterxml.jackson.databind.JsonNode;+import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.dataformat.cbor.CBORFactory;+import java.io.ByteArrayInputStream;+import java.io.ByteArrayOutputStream;+import java.io.File;+import java.io.IOException;+import java.io.InputStream;+import java.nio.file.Files;+import java.nio.file.Path;+import java.util.List;+import java.util.Map;+import java.util.concurrent.atomic.LongAdder;+import org.apache.solr.client.solrj.SolrQuery;+import org.apache.solr.client.solrj.SolrRequest;+import org.apache.solr.client.solrj.SolrServerException;+import org.apache.solr.client.solrj.impl.CloudSolrClient;+import org.apache.solr.client.solrj.impl.InputStreamResponseParser;+import org.apache.solr.client.solrj.request.CollectionAdminRequest;+import org.apache.solr.client.solrj.request.GenericSolrRequest;+import org.apache.solr.client.solrj.request.QueryRequest;+import org.apache.solr.client.solrj.request.RequestWriter;+import org.apache.solr.client.solrj.request.UpdateRequest;+import org.apache.solr.cloud.MiniSolrCloudCluster;+import org.apache.solr.cloud.SolrCloudTestCase;+import org.apache.solr.common.params.MapSolrParams;+import org.apache.solr.common.util.JavaBinCodec;+import org.apache.solr.common.util.NamedList;+import org.apache.solr.common.util.Utils;+import org.apache.solr.handler.loader.CborLoader;+import org.apache.solr.response.XMLResponseWriter;++public class TestCborDataFormat extends SolrCloudTestCase {++ @SuppressWarnings("unchecked")+ public void testRoundTrip() throws Exception {+ String testCollection = "testRoundTrip";++ MiniSolrCloudCluster cluster =+ configureCluster(1)+ .addConfig(+ "conf", TEST_PATH().resolve("configsets").resolve("cloud-managed").resolve("conf"))+ .configure();+ try {+ System.setProperty("managed.schema.mutable", "true");+ CloudSolrClient client = cluster.getSolrClient();+ CollectionAdminRequest.createCollection(testCollection, "conf", 1, 1).process(client);+ modifySchema(testCollection, client);++ byte[] b =+ Files.readAllBytes(+ new File(ExternalPaths.SOURCE_HOME, "example/films/films.json").toPath());+ // every operation is performed twice. We should only take the second number+ // so that we give JVM a chance to optimize that code+ index(testCollection, client, createJsonReq(b), true);+ index(testCollection, client, createJsonReq(b), true);++ index(testCollection, client, createJavabinReq(b), true);+ index(testCollection, client, createJavabinReq(b), true);++ index(testCollection, client, createCborReq(b), true);+ index(testCollection, client, createCborReq(b), false);++ runQuery(testCollection, client, "javabin");+ runQuery(testCollection, client, "javabin");+ runQuery(testCollection, client, "json");+ runQuery(testCollection, client, "json");+ b = runQuery(testCollection, client, "cbor");+ b = runQuery(testCollection, client, "cbor");+ ObjectMapper objectMapper = new ObjectMapper(new CBORFactory());+ Object o = objectMapper.readValue(b, Object.class);+ List<Object> l = (List<Object>) Utils.getObjectByPath(o, false, "response/docs");+ assertEquals(1100, l.size());+ } finally {+ System.clearProperty("managed.schema.mutable");+ cluster.shutdown();+ }+ }++ private void index(+ String testCollection, CloudSolrClient client, GenericSolrRequest r, boolean del)+ throws Exception {+ RTimer timer = new RTimer();+ client.request(r, testCollection);+ System.out.println("INDEX_TIME: " + r.contentWriter.getContentType() + " : " + timer.getTime());+ if (del) {+ UpdateRequest req = new UpdateRequest().deleteByQuery("*:*");+ req.setParam("commit", "true");+ client.request(req, testCollection);+ }+ }++ private byte[] runQuery(String testCollection, CloudSolrClient client, String wt)+ throws SolrServerException, IOException {+ NamedList<Object> result;+ QueryRequest request;+ RTimer timer = new RTimer();+ request = new QueryRequest(new SolrQuery("*:*").setRows(1111));+ request.setResponseParser(new InputStreamResponseParser(wt));+ result = client.request(request, testCollection);+ byte[] b = copyStream((InputStream) result.get("stream"));+ System.out.println(wt + "_time : " + timer.getTime());+ return b;+ }++ private static byte[] copyStream(InputStream inputStream) throws IOException {+ ByteArrayOutputStream outputStream = new ByteArrayOutputStream();+ byte[] buffer = new byte[4096];+ int bytesRead;+ while ((bytesRead = inputStream.read(buffer)) != -1) {+ outputStream.write(buffer, 0, bytesRead);+ }+ return outputStream.toByteArray();+ }++ private void modifySchema(String testCollection, CloudSolrClient client)+ throws SolrServerException, IOException {+ GenericSolrRequest req =+ new GenericSolrRequest(SolrRequest.METHOD.POST, "/schema")+ .setContentWriter(+ new RequestWriter.StringPayloadContentWriter(+ "{\n"+ + "\"add-field-type\" : {"+ + "\"name\":\"knn_vector_10\",\"class\":\"solr.DenseVectorField\",\"vectorDimension\":10,\"similarityFunction\":\"cosine\",\"knnAlgorithm\":\"hnsw\"},\n"+ + "\"add-field\" : ["+ + "{\"name\":\"name\",\"type\":\"sortabletext\",\"multiValued\":false,\"stored\":true},\n"+ + "{\"name\":\"initial_release_date\",\"type\":\"string\",\"stored\":true},\n"+ + "{\"name\":\"directed_by\",\"type\":\"string\",\"multiValued\":true,\"stored\":true},\n"+ + "{\"name\":\"genre\",\"type\":\"string\",\"multiValued\":true,\"stored\":true},\n"+ + "{\"name\":\"film_vector\",\"type\":\"knn_vector_10\",\"indexed\":true,\"stored\":true}]}",+ XMLResponseWriter.CONTENT_TYPE_XML_UTF8));++ client.request(req, testCollection);+ }++ private GenericSolrRequest createJsonReq(byte[] b) {+ return new GenericSolrRequest(+ SolrRequest.METHOD.POST,+ "/update/json/docs",+ new MapSolrParams(Map.of("commit", "true")))+ .withContent(b, "application/json");+ }++ @SuppressWarnings("rawtypes")+ private GenericSolrRequest createJavabinReq(byte[] b) throws IOException {+ List l = (List) Utils.fromJSON(b);+ ByteArrayOutputStream baos = new ByteArrayOutputStream();+ new JavaBinCodec().marshal(l.iterator(), baos);++ return new GenericSolrRequest(+ SolrRequest.METHOD.POST, "/update", new MapSolrParams(Map.of("commit", "true")))+ .withContent(baos.toByteArray(), "application/javabin");+ }++ private GenericSolrRequest createCborReq(byte[] b) throws IOException {+ return new GenericSolrRequest(+ SolrRequest.METHOD.POST, "/update/cbor", new MapSolrParams(Map.of("commit", "true")))+ .withContent(serializeToCbor(b), "application/cbor");+ }++ @SuppressWarnings("unchecked")+ public void test() throws Exception {+ Path filmsJson = new File(ExternalPaths.SOURCE_HOME, "example/films/films.json").toPath();++ long sz = Files.size(filmsJson);+ assertEquals(633600, sz);++ List<Object> films = null;+ try (InputStream is = Files.newInputStream(filmsJson)) {+ films = (List<Object>) Utils.fromJSON(is);+ }+ ByteArrayOutputStream baos = new ByteArrayOutputStream();++ new JavaBinCodec().marshal(Map.of("films", films), baos);+ assertEquals(234520, baos.toByteArray().length);++ byte[] b = Files.readAllBytes(filmsJson);+ byte[] bytes = serializeToCbor(b);+ assertEquals(290672, bytes.length);+ LongAdder docsSz = new LongAdder();+ new CborLoader(null, (document) -> docsSz.increment()).stream(new ByteArrayInputStream(bytes));+ assertEquals(films.size(), docsSz.intValue());+ }++ private byte[] serializeToCbor(byte[] is) throws IOException {+ ByteArrayOutputStream baos;+ ObjectMapper jsonMapper = new ObjectMapper(new JsonFactory());++ // Read JSON file as a JsonNode+ JsonNode jsonNode = jsonMapper.readTree(is);+ // Create a CBOR ObjectMapper+ CBORFactory jf = new CBORFactory();+ ObjectMapper cborMapper = new ObjectMapper(jf);+ baos = new ByteArrayOutputStream();+ JsonGenerator jsonGenerator = cborMapper.createGenerator(baos);++ jsonGenerator.writeTree(jsonNode);+ jsonGenerator.close();+ return baos.toByteArray();+ }+}diff --git a/solr/solrj/build.gradle b/solr/solrj/build.gradleindex 524bebbb90a..10bbc78c5c1 100644--- a/solr/solrj/build.gradle+++ b/solr/solrj/build.gradle@@ -62,6 +62,8 @@ dependencies {testImplementation 'org.hamcrest:hamcrest'testImplementation 'commons-io:commons-io'+ testImplementation 'com.fasterxml.jackson.core:jackson-databind'+ testImplementation 'com.fasterxml.jackson.dataformat:jackson-dataformat-cbor'testImplementation 'org.eclipse.jetty.toolchain:jetty-servlet-api'diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/request/GenericSolrRequest.java b/solr/solrj/src/java/org/apache/solr/client/solrj/request/GenericSolrRequest.javaindex 7f08f454a8f..e9babd55371 100644--- a/solr/solrj/src/java/org/apache/solr/client/solrj/request/GenericSolrRequest.java+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/request/GenericSolrRequest.java@@ -16,6 +16,8 @@*/package org.apache.solr.client.solrj.request;+import java.io.IOException;+import java.io.OutputStream;import org.apache.solr.client.solrj.SolrClient;import org.apache.solr.client.solrj.SolrRequest;import org.apache.solr.client.solrj.request.RequestWriter.ContentWriter;@@ -42,6 +44,22 @@ public class GenericSolrRequest extends SolrRequest<SimpleSolrResponse> {return this;}+ public GenericSolrRequest withContent(byte[] buf, String type) {+ contentWriter =+ new ContentWriter() {+ @Override+ public void write(OutputStream os) throws IOException {+ os.write(buf);+ }++ @Override+ public String getContentType() {+ return type;+ }+ };+ return this;+ }+@Overridepublic ContentWriter getContentWriter(String expectedType) {return contentWriter;diff --git a/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleCborTest.java b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleCborTest.javanew file mode 100644index 00000000000..f021e028386--- /dev/null+++ b/solr/solrj/src/test/org/apache/solr/client/solrj/SolrExampleCborTest.java@@ -0,0 +1,353 @@+/*+ * Licensed to the Apache Software Foundation (ASF) under one or more+ * contributor license agreements. See the NOTICE file distributed with+ * this work for additional information regarding copyright ownership.+ * The ASF licenses this file to You under the Apache License, Version 2.0+ * (the "License"); you may not use this file except in compliance with+ * the License. You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */+package org.apache.solr.client.solrj;++import com.fasterxml.jackson.databind.ObjectMapper;+import com.fasterxml.jackson.dataformat.cbor.CBORFactory;+import java.io.IOException;+import java.io.InputStream;+import java.io.OutputStream;+import java.io.Reader;+import java.util.ArrayList;+import java.util.LinkedHashMap;+import java.util.List;+import java.util.Map;+import org.apache.solr.client.solrj.impl.BinaryRequestWriter;+import org.apache.solr.client.solrj.impl.HttpSolrClient;+import org.apache.solr.client.solrj.request.RequestWriter;+import org.apache.solr.client.solrj.request.UpdateRequest;+import org.apache.solr.common.SolrDocument;+import org.apache.solr.common.SolrDocumentList;+import org.apache.solr.common.SolrException;+import org.apache.solr.common.SolrInputDocument;+import org.apache.solr.common.util.NamedList;+import org.junit.BeforeClass;+import org.junit.Ignore;++/**+ * This test only tests a subset of tests specified in the parent class. It's because we do not+ * provide a full impl of all types of responses in SolrJ. It just handles very common types and+ * converts them from Map to NamedList+ */+public class SolrExampleCborTest extends SolrExampleTests {+ @BeforeClass+ public static void beforeTest() throws Exception {+ createAndStartJetty(legacyExampleCollection1SolrHome());+ }++ @Override+ public SolrClient createNewSolrClient() {+ return new HttpSolrClient.Builder(getServerUrl())+ .allowMultiPartPost(random().nextBoolean())+ .withRequestWriter(cborRequestWriter())+ .withResponseParser(cborResponseparser())+ .build();+ }++ @Override+ @Ignore+ public void testQueryPerf() {+ /*Ignore*/+ }++ @Override+ @Ignore+ public void testExampleConfig() {+ /*Ignore*/+ }++ @Override+ @Ignore+ public void testAugmentFields() {+ /*Ignore*/+ }++ @Override+ @Ignore+ public void testRawFields() {+ /*Ignore*/+ }++ @Override+ @Ignore+ public void testUpdateRequestWithParameters() {+ /*Ignore*/+ }++ @Override+ @Ignore+ public void testContentStreamRequest() {+ /*Ignore*/+ }++ @Override+ @Ignore+ public void testStreamingRequest() {+ /*Ignore*/+ }++ @Override+ @Ignore+ public void testMultiContentWriterRequest() {+ /*Ignore*/+ }++ @Override+ @Ignore+ public void testMultiContentStreamRequest() {+ /*Ignore*/+ }++ @Override+ @Ignore+ public void testLukeHandler() {+ /*Ignore*/+ }++ @Override+ @Ignore+ public void testStatistics() {+ /*Ignore*/+ }++ @Override+ @Ignore+ public void testFaceting() {+ /*Ignore*/+ }++ @Override+ @Ignore+ public void testPivotFacets() {+ /*Ignore*/+ }++ @Override+ public void testPivotFacetsStats() {+ /*Ignore*/+ }++ @Override+ @Ignore+ public void testPivotFacetsStatsNotSupported() {+ /*Ignore*/+ }++ @Override+ @Ignore+ public void testPivotFacetsQueries() {+ /*Ignore*/+ }++ @Override+ @Ignore+ public void testPivotFacetsRanges() {+ /*Ignore*/+ }++ @Override+ @Ignore+ public void testPivotFacetsMissing() {+ /*Ignore*/+ }++ @Override+ @Ignore+ public void testUpdateField() {+ /*Ignore*/+ }++ @Override+ @Ignore+ public void testUpdateMultiValuedField() {+ /*Ignore*/+ }++ @Override+ @Ignore+ public void testSetNullUpdates() {+ /*Ignore*/+ }++ @Override+ @Ignore+ public void testSetNullUpdateOrder() {+ /*Ignore*/+ }++ @Override+ @Ignore+ public void testQueryWithParams() {+ /*Ignore*/+ }++ @Override+ @Ignore+ public void testChildDoctransformer() {+ /*Ignore*/+ }++ @Override+ @Ignore+ public void testExpandComponent() {+ /*Ignore*/+ }++ @Override+ @Ignore+ public void testMoreLikeThis() {+ /*Ignore*/+ }++ @Override+ @Ignore+ public void testAddChildToChildFreeDoc() {+ /*Ignore*/+ }++ @Override+ @Ignore+ public void testDeleteParentDoc() {+ /*Ignore*/+ }++ @Override+ @Ignore+ public void testCommitWithinOnAdd() {+ /*Ignore*/+ }++ @Override+ @Ignore+ public void testAddDelete() {+ /*Ignore*/+ }++ @SuppressWarnings("rawtypes")+ private static RequestWriter cborRequestWriter() {+ return new BinaryRequestWriter() {++ @Override+ public ContentWriter getContentWriter(SolrRequest<?> request) {+ if (request instanceof UpdateRequest) {+ UpdateRequest updateRequest = (UpdateRequest) request;+ List<SolrInputDocument> docs = updateRequest.getDocuments();+ if (docs == null || docs.isEmpty()) return super.getContentWriter(request);+ return new ContentWriter() {+ @Override+ public void write(OutputStream os) throws IOException {++ List<Map> mapDocs = new ArrayList<>();+ for (SolrInputDocument doc : docs) {+ mapDocs.add(doc.toMap(new LinkedHashMap<>()));+ }++ CBORFactory jf = new CBORFactory();+ ObjectMapper cborMapper = new ObjectMapper(jf);+ cborMapper.writeValue(os, mapDocs);+ }++ @Override+ public String getContentType() {+ return "application/cbor";+ }+ };+ } else {+ return super.getContentWriter(request);+ }+ }++ @Override+ public String getUpdateContentType() {+ return "application/cbor";+ }++ @Override+ public String getPath(SolrRequest<?> req) {+ if (req instanceof UpdateRequest) {+ UpdateRequest updateRequest = (UpdateRequest) req;+ List<SolrInputDocument> docs = updateRequest.getDocuments();+ if (docs == null || docs.isEmpty()) return super.getPath(req);+ return "/update/cbor";+ } else {+ return super.getPath(req);+ }+ }+ };+ }++ private static ResponseParser cborResponseparser() {+ return new ResponseParser() {++ @Override+ public String getWriterType() {+ return "cbor";+ }++ @Override+ @SuppressWarnings({"rawtypes", "unchecked"})+ public NamedList<Object> processResponse(InputStream b, String encoding) {+ ObjectMapper objectMapper = new ObjectMapper(new CBORFactory());+ try {+ Map m = (Map) objectMapper.readValue(b, Object.class);+ NamedList nl = new NamedList();+ m.forEach(+ (k, v) -> {+ if (v instanceof Map) {+ Map map = (Map) v;+ if ("response".equals(k)) {+ v = convertResponse((Map) v);+ } else {+ v = new NamedList(map);+ }+ }+ nl.add(k.toString(), v);+ });+ return nl;+ } catch (IOException e) {+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, "parsing error", e);+ }+ }++ @SuppressWarnings({"rawtypes", "unchecked"})+ private SolrDocumentList convertResponse(Map m) {+ List<Map> docs = (List<Map>) m.get("docs");+ SolrDocumentList sdl = new SolrDocumentList();+ Object o = m.get("numFound");+ if (o != null) sdl.setNumFound(((Number) o).longValue());+ o = m.get("start");+ if (o != null) sdl.setStart(((Number) o).longValue());+ o = m.get("numFoundExact");+ if (o != null) sdl.setNumFoundExact((Boolean) o);++ if (docs != null) {+ for (Map doc : docs) {+ SolrDocument sd = new SolrDocument();+ doc.forEach((k, v) -> sd.addField(k.toString(), v));+ sdl.add(sd);+ }+ }+ return sdl;+ }++ @Override+ public NamedList<Object> processResponse(Reader reader) {+ return null;+ }+ };+ }+}diff --git a/versions.lock b/versions.lockindex 9b2c89e52e8..a995f7107d8 100644--- a/versions.lock+++ b/versions.lock@@ -10,6 +10,7 @@ com.fasterxml.jackson:jackson-bom:2.14.2 (12 constraints: 42f8fc44)com.fasterxml.jackson.core:jackson-annotations:2.14.2 (10 constraints: 95be250f)com.fasterxml.jackson.core:jackson-core:2.14.2 (13 constraints: 3902ce18)com.fasterxml.jackson.core:jackson-databind:2.14.2 (18 constraints: 78638fbe)+com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.14.2 (2 constraints: 5f1c13f1)com.fasterxml.jackson.dataformat:jackson-dataformat-smile:2.14.2 (1 constraints: b90ea866)com.fasterxml.jackson.module:jackson-module-jaxb-annotations:2.14.2 (2 constraints: a52484e0)com.fasterxml.woodstox:woodstox-core:6.5.0 (3 constraints: ab2868b8)@@ -386,7 +387,6 @@ com.amazonaws:aws-java-sdk-core:1.12.15 (2 constraints: 501a5183)com.amazonaws:aws-java-sdk-kms:1.12.15 (1 constraints: d60cb42a)com.amazonaws:aws-java-sdk-s3:1.12.15 (1 constraints: e0125c30)com.amazonaws:jmespath-java:1.12.15 (2 constraints: 501a5183)-com.fasterxml.jackson.dataformat:jackson-dataformat-cbor:2.14.2 (2 constraints: 5f1c13f1)com.fasterxml.jackson.dataformat:jackson-dataformat-xml:2.14.2 (2 constraints: a619c912)com.fasterxml.jackson.datatype:jackson-datatype-jdk8:2.14.2 (2 constraints: 0b24ff81)com.fasterxml.jackson.datatype:jackson-datatype-jsr310:2.14.2 (3 constraints: 7c3ddb0f)