Apache Solr open-source search software
SOLR-17050: Use compact JSON for Learning to Rank (LTR) feature and model storage. (#2030)
Florin Babesce9930338d2d
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex da916a6be6c..9e63d1ce42e 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -113,6 +113,8 @@ Improvements* SOLR-16959: Make the internal CoreSorter implementation configurable in solr.xml (Vincent Primault)+* SOLR-17050: Use compact JSON for Learning to Rank (LTR) feature and model storage. (Florin Babes, Christine Poerschke, Alessandro Benedetti)+Optimizations---------------------* SOLR-17084: LBSolrClient (used by CloudSolrClient) now returns the count of core tracked as not live AKA zombiesdiff --git a/solr/core/src/java/org/apache/solr/rest/ManagedResourceStorage.java b/solr/core/src/java/org/apache/solr/rest/ManagedResourceStorage.javaindex b28f4f57c36..af04c36eaab 100644--- a/solr/core/src/java/org/apache/solr/rest/ManagedResourceStorage.java+++ b/solr/core/src/java/org/apache/solr/rest/ManagedResourceStorage.java@@ -47,6 +47,7 @@ import org.apache.solr.common.cloud.ZkStateReader;import org.apache.solr.common.util.NamedList;import org.apache.solr.common.util.Utils;import org.apache.solr.core.SolrResourceLoader;+import org.noggit.JSONWriter;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -420,9 +421,20 @@ public abstract class ManagedResourceStorage {/** Default storage implementation that uses JSON as the storage format for managed data. */public static class JsonStorage extends ManagedResourceStorage {+ private final int indentSize;+ /** Uses {@link JSONWriter#DEFAULT_INDENT} space characters as an indent. */public JsonStorage(StorageIO storageIO, SolrResourceLoader loader) {+ this(storageIO, loader, JSONWriter.DEFAULT_INDENT);+ }++ /**+ * @param indentSize The number of space characters to use as an indent. 0=newlines but no+ * spaces, -1=no indent at all.+ */+ public JsonStorage(StorageIO storageIO, SolrResourceLoader loader, int indentSize) {super(storageIO, loader);+ this.indentSize = indentSize;}/**@@ -441,7 +453,7 @@ public abstract class ManagedResourceStorage {@Overridepublic void store(String resourceId, Object toStore) throws IOException {- String json = toJSONString(toStore);+ String json = toJSONString(toStore, indentSize);String storedResourceId = getStoredResourceId(resourceId);OutputStreamWriter writer = null;try {diff --git a/solr/core/src/test/org/apache/solr/util/TestUtils.java b/solr/core/src/test/org/apache/solr/util/TestUtils.javaindex e05f3130360..3e18f8e766c 100644--- a/solr/core/src/test/org/apache/solr/util/TestUtils.java+++ b/solr/core/src/test/org/apache/solr/util/TestUtils.java@@ -294,4 +294,44 @@ public class TestUtils extends SolrTestCaseJ4 {assertEquals(2L, Utils.getObjectByPath(sink, true, List.of(DEFAULTS, COLLECTION_PROP, NRT_REPLICAS)));}++ @SuppressWarnings({"unchecked"})+ public void testToJson() {+ Map<String, Object> object =+ (Map<String, Object>) Utils.fromJSONString("{k2:v2, k1: {a:b, p:r, k21:{xx:yy}}}");++ assertEquals(+ "{\n"+ + " \"k2\":\"v2\",\n"+ + " \"k1\":{\n"+ + " \"a\":\"b\",\n"+ + " \"p\":\"r\",\n"+ + " \"k21\":{\"xx\":\"yy\"}}}",+ new String(Utils.toJSON(object), UTF_8));+ }++ @SuppressWarnings({"unchecked"})+ public void testToJsonCompacted() {+ Map<String, Object> object =+ (Map<String, Object>) Utils.fromJSONString("{k2:v2, k1: {a:b, p:r, k21:{xx:yy}}}");++ assertEquals(+ "{\"k2\":\"v2\",\"k1\":{\"a\":\"b\",\"p\":\"r\",\"k21\":{\"xx\":\"yy\"}}}",+ new String(Utils.toJSON(object, -1), UTF_8));+ }++ @SuppressWarnings({"unchecked"})+ public void testToJsonNoIndent() {+ Map<String, Object> object =+ (Map<String, Object>) Utils.fromJSONString("{k2:v2, k1: {a:b, p:r, k21:{xx:yy}}}");++ assertEquals(+ "{\n"+ + "\"k2\":\"v2\",\n"+ + "\"k1\":{\n"+ + "\"a\":\"b\",\n"+ + "\"p\":\"r\",\n"+ + "\"k21\":{\"xx\":\"yy\"}}}",+ new String(Utils.toJSON(object, 0), UTF_8));+ }}diff --git a/solr/modules/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedFeatureStore.java b/solr/modules/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedFeatureStore.javaindex e867e7bbbad..96122f1604c 100644--- a/solr/modules/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedFeatureStore.java+++ b/solr/modules/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedFeatureStore.java@@ -85,6 +85,12 @@ public class ManagedFeatureStore extends ManagedResourcesuper(resourceId, loader, storageIO);}+ @Override+ protected ManagedResourceStorage createStorage(+ ManagedResourceStorage.StorageIO storageIO, SolrResourceLoader loader) throws SolrException {+ return new ManagedResourceStorage.JsonStorage(storageIO, loader, -1);+ }+public Map<String, FeatureStore> getStores() {return stores;}diff --git a/solr/modules/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedModelStore.java b/solr/modules/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedModelStore.javaindex 918b06bd724..f6778e54425 100644--- a/solr/modules/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedModelStore.java+++ b/solr/modules/ltr/src/java/org/apache/solr/ltr/store/rest/ManagedModelStore.java@@ -93,6 +93,12 @@ public class ManagedModelStore extends ManagedResourcestore = new ModelStore();}+ @Override+ protected ManagedResourceStorage createStorage(+ ManagedResourceStorage.StorageIO storageIO, SolrResourceLoader loader) throws SolrException {+ return new ManagedResourceStorage.JsonStorage(storageIO, loader, -1);+ }+public void setManagedFeatureStore(ManagedFeatureStore managedFeatureStore) {log.info("INIT model store");this.managedFeatureStore = managedFeatureStore;diff --git a/solr/modules/ltr/src/test/org/apache/solr/ltr/store/rest/TestModelManagerPersistence.java b/solr/modules/ltr/src/test/org/apache/solr/ltr/store/rest/TestModelManagerPersistence.javaindex 2adcc1f19f8..7b159e0e650 100644--- a/solr/modules/ltr/src/test/org/apache/solr/ltr/store/rest/TestModelManagerPersistence.java+++ b/solr/modules/ltr/src/test/org/apache/solr/ltr/store/rest/TestModelManagerPersistence.java@@ -16,6 +16,8 @@*/package org.apache.solr.ltr.store.rest;+import static java.nio.charset.StandardCharsets.UTF_8;+import java.io.BufferedWriter;import java.io.OutputStreamWriter;import java.nio.charset.StandardCharsets;@@ -137,6 +139,24 @@ public class TestModelManagerPersistence extends TestRerankBase {assertJQ(ManagedModelStore.REST_END_POINT, "/models/==[]");}+ @Test+ public void testFeaturesAndModelAreStoredCompact() throws Exception {+ loadFeature("feature", ValueFeature.class.getName(), "test", "{\"value\":2}");+ loadModel(+ "test-model",+ LinearModel.class.getName(),+ new String[] {"feature"},+ "test",+ "{\"weights\":{\"feature\":1.0}}");++ final String fstorecontent = Files.readString(fstorefile, StandardCharsets.UTF_8);+ final String mstorecontent = Files.readString(mstorefile, StandardCharsets.UTF_8);+ Object fStoreObject = Utils.fromJSONString(fstorecontent);+ Object mStoreObject = Utils.fromJSONString(mstorecontent);+ assertEquals(new String(Utils.toJSON(fStoreObject, -1), UTF_8), fstorecontent);+ assertEquals(new String(Utils.toJSON(mStoreObject, -1), UTF_8), mstorecontent);+ }+@Testpublic void testFilePersistence() throws Exception {// check whether models and features are emptydiff --git a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java b/solr/solrj/src/java/org/apache/solr/common/util/Utils.javaindex 0ae4886d670..2a3ea47b103 100644--- a/solr/solrj/src/java/org/apache/solr/common/util/Utils.java+++ b/solr/solrj/src/java/org/apache/solr/common/util/Utils.java@@ -217,10 +217,29 @@ public class Utils {public static byte[] toJSON(Object o) {if (o == null) return new byte[0];CharArr out = new CharArr();- new JSONWriter(out, 2).write(o); // indentation by default+ new JSONWriter(out, JSONWriter.DEFAULT_INDENT).write(o); // indentation by defaultreturn toUTF8(out);}+ /**+ * @param indentSize The number of space characters to use as an indent. 0=newlines but no spaces,+ * -1=no indent at all.+ */+ public static byte[] toJSON(Object o, int indentSize) {+ if (o == null) return new byte[0];+ CharArr out = new CharArr();+ new JSONWriter(out, indentSize).write(o);+ return toUTF8(out);+ }++ /**+ * @param indentSize The number of space characters to use as an indent. 0=newlines but no spaces,+ * -1=no indent at all.+ */+ public static String toJSONString(Object o, int indentSize) {+ return new String(toJSON(o, indentSize), StandardCharsets.UTF_8);+ }+public static String toJSONString(Object o) {return new String(toJSON(o), StandardCharsets.UTF_8);}diff --git a/solr/solrj/src/java/org/noggit/JSONWriter.java b/solr/solrj/src/java/org/noggit/JSONWriter.javaindex 349c1fecf02..9b1dd073c83 100644--- a/solr/solrj/src/java/org/noggit/JSONWriter.java+++ b/solr/solrj/src/java/org/noggit/JSONWriter.java@@ -31,6 +31,8 @@ public class JSONWriter {public void write(JSONWriter writer);}+ public static final int DEFAULT_INDENT = 2;+protected int level;protected int indent;protected final CharArr out;@@ -46,7 +48,7 @@ public class JSONWriter {}public JSONWriter(CharArr out) {- this(out, 2);+ this(out, DEFAULT_INDENT);}public void setIndentSize(int indentSize) {