Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-15277: Schema designer UI and supporting backend (#42)

Timothy Potteraab66e7bf12b
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 8f1d1cbc2fb..9d72c0dc20c 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -41,6 +41,8 @@ New Features
* SOLR-15300: Report collection and shard "health" state in CLUSTERSTATUS response. (ab, janhoy)
+* SOLR-15277: Schema Designer UI (Timothy Potter, Kiran Chitturi)
+
Improvements
----------------------
* LUCENE-8984: MoreLikeThis MLT is biased for uncommon fields (Andy Hind via Anshum Gupta)
diff --git a/solr/core/src/java/org/apache/solr/core/CoreContainer.java b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
index 9cfc4c5461b..d6801e443c5 100644
--- a/solr/core/src/java/org/apache/solr/core/CoreContainer.java
+++ b/solr/core/src/java/org/apache/solr/core/CoreContainer.java
@@ -97,6 +97,7 @@ import org.apache.solr.handler.ClusterAPI;
import org.apache.solr.handler.CollectionBackupsAPI;
import org.apache.solr.handler.CollectionsAPI;
import org.apache.solr.handler.RequestHandlerBase;
+import org.apache.solr.handler.designer.SchemaDesignerAPI;
import org.apache.solr.handler.SnapShooter;
import org.apache.solr.handler.admin.CollectionsHandler;
import org.apache.solr.handler.admin.ConfigSetsHandler;
@@ -782,6 +783,11 @@ public class CoreContainer {
containerHandlers.getApiBag().registerObject(clusterAPI);
containerHandlers.getApiBag().registerObject(clusterAPI.commands);
containerHandlers.getApiBag().registerObject(clusterAPI.configSetCommands);
+
+ if (isZooKeeperAware()) {
+ containerHandlers.getApiBag().registerObject(new SchemaDesignerAPI(this));
+ } // else Schema Designer not available in standalone (non-cloud) mode
+
/*
* HealthCheckHandler needs to be initialized before InfoHandler, since the later one will call CoreContainer.getHealthCheckHandler().
* We don't register the handler here because it'll be registered inside InfoHandler
diff --git a/solr/core/src/java/org/apache/solr/handler/designer/DefaultSampleDocumentsLoader.java b/solr/core/src/java/org/apache/solr/handler/designer/DefaultSampleDocumentsLoader.java
new file mode 100644
index 00000000000..c79e4ab231a
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/designer/DefaultSampleDocumentsLoader.java
@@ -0,0 +1,387 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.designer;
+
+import javax.xml.stream.XMLInputFactory;
+import javax.xml.stream.XMLStreamConstants;
+import javax.xml.stream.XMLStreamException;
+import javax.xml.stream.XMLStreamReader;
+import java.io.BufferedReader;
+import java.io.ByteArrayOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Reader;
+import java.io.StringReader;
+import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.SolrInputField;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.ContentStreamBase;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.handler.loader.CSVLoaderBase;
+import org.apache.solr.handler.loader.JsonLoader;
+import org.apache.solr.handler.loader.XMLLoader;
+import org.apache.solr.request.SolrQueryRequestBase;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.update.processor.UpdateRequestProcessor;
+import org.apache.solr.util.SafeXMLParsing;
+import org.noggit.JSONParser;
+import org.noggit.ObjectBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.w3c.dom.Document;
+import org.w3c.dom.Element;
+import org.xml.sax.SAXException;
+
+import static org.apache.solr.common.params.CommonParams.JSON_MIME;
+import static org.apache.solr.handler.loader.CSVLoaderBase.SEPARATOR;
+
+public class DefaultSampleDocumentsLoader implements SampleDocumentsLoader {
+ public static final String CSV_MULTI_VALUE_DELIM_PARAM = "csvMultiValueDelimiter";
+ private static final int MAX_STREAM_SIZE = (5 * 1024 * 1024);
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ public static byte[] streamAsBytes(final InputStream in) throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ byte[] buf = new byte[1024];
+ int r;
+ try (in) {
+ while ((r = in.read(buf)) != -1) baos.write(buf, 0, r);
+ }
+ return baos.toByteArray();
+ }
+
+ @Override
+ public SampleDocuments parseDocsFromStream(SolrParams params, ContentStream stream, final int maxDocsToLoad) throws IOException {
+ final String contentType = stream.getContentType();
+ if (contentType == null) {
+ return SampleDocuments.NONE;
+ }
+
+ if (params == null) {
+ params = new ModifiableSolrParams();
+ }
+
+ Long streamSize = stream.getSize();
+ if (streamSize != null && streamSize > MAX_STREAM_SIZE) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Sample is too big! " + MAX_STREAM_SIZE + " bytes is the max upload size for sample documents.");
+ }
+
+ String fileSource = "paste";
+ if ("file".equals(stream.getName())) {
+ fileSource = stream.getSourceInfo() != null ? stream.getSourceInfo() : "file";
+ }
+
+ byte[] uploadedBytes = streamAsBytes(stream.getStream());
+ // recheck the upload size in case the stream returned null for getSize
+ if (uploadedBytes.length > MAX_STREAM_SIZE) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Sample is too big! " + MAX_STREAM_SIZE + " bytes is the max upload size for sample documents.");
+ }
+ // use a byte stream for the parsers in case they need to re-parse using a different strategy
+ // e.g. JSON vs. JSON lines or different CSV strategies ...
+ ContentStreamBase.ByteArrayStream byteStream = new ContentStreamBase.ByteArrayStream(uploadedBytes, fileSource, contentType);
+ String charset = ContentStreamBase.getCharsetFromContentType(stream.getContentType());
+ if (charset == null) {
+ charset = ContentStreamBase.DEFAULT_CHARSET;
+ }
+
+ List<SolrInputDocument> docs = null;
+ if (stream.getSize() > 0) {
+ if (contentType.contains(JSON_MIME)) {
+ docs = loadJsonDocs(params, byteStream, maxDocsToLoad);
+ } else if (contentType.contains("text/xml") || contentType.contains("application/xml")) {
+ docs = loadXmlDocs(params, byteStream, maxDocsToLoad);
+ } else if (contentType.contains("text/csv") || contentType.contains("application/csv")) {
+ docs = loadCsvDocs(params, fileSource, uploadedBytes, charset, maxDocsToLoad);
+ } else if (contentType.contains("text/plain") || contentType.contains("application/octet-stream")) {
+ docs = loadJsonLines(params, byteStream, maxDocsToLoad);
+ } else {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, contentType + " not supported yet!");
+ }
+
+ if (docs != null && maxDocsToLoad > 0 && docs.size() > maxDocsToLoad) {
+ docs = docs.subList(0, maxDocsToLoad);
+ }
+ }
+
+ return new SampleDocuments(docs, contentType, fileSource);
+ }
+
+ protected List<SolrInputDocument> loadCsvDocs(SolrParams params, String source, byte[] streamBytes, String charset, final int maxDocsToLoad) throws IOException {
+ ContentStream stream;
+ if (params.get(SEPARATOR) == null) {
+ String csvStr = new String(streamBytes, charset);
+ char sep = detectTSV(csvStr);
+ ModifiableSolrParams modifiableSolrParams = new ModifiableSolrParams(params);
+ modifiableSolrParams.set(SEPARATOR, String.valueOf(sep));
+ params = modifiableSolrParams;
+ stream = new ContentStreamBase.StringStream(csvStr, "text/csv");
+ } else {
+ stream = new ContentStreamBase.ByteArrayStream(streamBytes, source, "text/csv");
+ }
+ return (new SampleCSVLoader(new CSVRequest(params), maxDocsToLoad)).loadDocs(stream);
+ }
+
+ @SuppressWarnings("unchecked")
+ protected List<SolrInputDocument> loadJsonLines(SolrParams params, ContentStreamBase.ByteArrayStream stream, final int maxDocsToLoad) throws IOException {
+ List<Map<String, Object>> docs = new LinkedList<>();
+ try (Reader r = stream.getReader()) {
+ BufferedReader br = new BufferedReader(r);
+ String line;
+ while ((line = br.readLine()) != null) {
+ line = line.trim();
+ if (!line.isEmpty() && line.startsWith("{") && line.endsWith("}")) {
+ Object jsonLine = ObjectBuilder.getVal(new JSONParser(line));
+ if (jsonLine instanceof Map) {
+ docs.add((Map<String, Object>) jsonLine);
+ }
+ }
+ if (maxDocsToLoad > 0 && docs.size() == maxDocsToLoad) {
+ break;
+ }
+ }
+ }
+
+ return docs.stream().map(JsonLoader::buildDoc).collect(Collectors.toList());
+ }
+
+ @SuppressWarnings("unchecked")
+ protected List<SolrInputDocument> loadJsonDocs(SolrParams params, ContentStreamBase.ByteArrayStream stream, final int maxDocsToLoad) throws IOException {
+ Object json;
+ try (Reader r = stream.getReader()) {
+ json = ObjectBuilder.getVal(new JSONParser(r));
+ }
+ if (json == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Expected at least 1 JSON doc in the request body!");
+ }
+
+ List<Map<String, Object>> docs;
+ if (json instanceof List) {
+ // list of docs
+ docs = (List<Map<String, Object>>) json;
+ } else if (json instanceof Map) {
+ // single doc ... see if this is a json lines file
+ boolean isJsonLines = false;
+ String charset = ContentStreamBase.getCharsetFromContentType(stream.getContentType());
+ String jsonStr = new String(streamAsBytes(stream.getStream()), charset != null ? charset : ContentStreamBase.DEFAULT_CHARSET);
+ String[] lines = jsonStr.split("\n");
+ if (lines.length > 1) {
+ for (String line : lines) {
+ line = line.trim();
+ if (!line.isEmpty() && line.startsWith("{") && line.endsWith("}")) {
+ isJsonLines = true;
+ break;
+ }
+ }
+ }
+ if (isJsonLines) {
+ docs = loadJsonLines(lines);
+ } else {
+ docs = Collections.singletonList((Map<String, Object>) json);
+ }
+ } else {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Expected one or more JSON docs in the request body!");
+ }
+ if (maxDocsToLoad > 0 && docs.size() > maxDocsToLoad) {
+ docs = docs.subList(0, maxDocsToLoad);
+ }
+ return docs.stream().map(JsonLoader::buildDoc).collect(Collectors.toList());
+ }
+
+ protected List<SolrInputDocument> loadXmlDocs(SolrParams params, ContentStreamBase.ByteArrayStream stream, final int maxDocsToLoad) throws IOException {
+ String xmlString = readInputAsString(stream.getStream()).trim();
+ List<SolrInputDocument> docs;
+ if (xmlString.contains("<add>") && xmlString.contains("<doc>")) {
+ XMLInputFactory inputFactory = XMLInputFactory.newInstance();
+ inputFactory.setProperty(XMLInputFactory.IS_SUPPORTING_EXTERNAL_ENTITIES, false);
+ inputFactory.setProperty(XMLInputFactory.SUPPORT_DTD, false);
+ XMLStreamReader parser = null;
+ try {
+ parser = inputFactory.createXMLStreamReader(new StringReader(xmlString));
+ docs = parseXmlDocs(parser, maxDocsToLoad);
+ } catch (XMLStreamException e) {
+ throw new IOException(e);
+ } finally {
+ if (parser != null) {
+ try {
+ parser.close();
+ } catch (XMLStreamException ignore) {
+ }
+ }
+ }
+ } else {
+ Document xmlDoc;
+ try {
+ xmlDoc = SafeXMLParsing.parseUntrustedXML(log, xmlString);
+ } catch (SAXException e) {
+ throw new IOException(e);
+ }
+ Element root = xmlDoc.getDocumentElement();
+ // TODO: support other types of XML here
+ throw new IOException("TODO: XML documents with root " + root.getTagName() + " not supported yet!");
+ }
+ return docs;
+ }
+
+ protected List<SolrInputDocument> parseXmlDocs(XMLStreamReader parser, final int maxDocsToLoad) throws XMLStreamException {
+ List<SolrInputDocument> docs = new LinkedList<>();
+ XMLLoader loader = new XMLLoader().init(null);
+ while (true) {
+ final int event;
+ try {
+ event = parser.next();
+ } catch (java.util.NoSuchElementException noSuchElementException) {
+ return docs;
+ }
+ switch (event) {
+ case XMLStreamConstants.END_DOCUMENT:
+ parser.close();
+ return docs;
+ case XMLStreamConstants.START_ELEMENT:
+ if ("doc".equals(parser.getLocalName())) {
+ SolrInputDocument doc = loader.readDoc(parser);
+ if (doc != null) {
+ docs.add(doc);
+
+ if (maxDocsToLoad > 0 && docs.size() >= maxDocsToLoad) {
+ parser.close();
+ return docs;
+ }
+ }
+ }
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ protected List<Map<String, Object>> loadJsonLines(String[] lines) throws IOException {
+ List<Map<String, Object>> docs = new ArrayList<>(lines.length);
+ for (String line : lines) {
+ line = line.trim();
+ if (!line.isEmpty() && line.startsWith("{") && line.endsWith("}")) {
+ Object jsonLine = ObjectBuilder.getVal(new JSONParser(line));
+ if (jsonLine instanceof Map) {
+ docs.add((Map<String, Object>) jsonLine);
+ }
+ }
+ }
+ return docs;
+ }
+
+ protected String readInputAsString(InputStream in) throws IOException {
+ return new String(streamAsBytes(in), StandardCharsets.UTF_8);
+ }
+
+ protected char detectTSV(String csvStr) {
+ char sep = ',';
+ int endOfFirstLine = csvStr.indexOf('\n');
+ if (endOfFirstLine != -1) {
+ int commas = 0;
+ int tabs = 0;
+ for (char value : csvStr.substring(0, endOfFirstLine).toCharArray()) {
+ if (value == ',') {
+ ++commas;
+ } else if (value == '\t') {
+ ++tabs;
+ }
+ }
+ if (tabs >= commas) {
+ sep = '\t';
+ }
+ }
+ return sep;
+ }
+
+ @Override
+ @SuppressWarnings({"rawtypes"})
+ public void init(NamedList args) {
+
+ }
+
+ private static class NoOpUpdateRequestProcessor extends UpdateRequestProcessor {
+ NoOpUpdateRequestProcessor() {
+ super(null);
+ }
+ }
+
+ private static class CSVRequest extends SolrQueryRequestBase {
+ CSVRequest(SolrParams params) {
+ super(null, params);
+ }
+ }
+
+ private static class SampleCSVLoader extends CSVLoaderBase {
+ List<SolrInputDocument> docs = new LinkedList<>();
+ CSVRequest req;
+ int maxDocsToLoad;
+ String multiValueDelimiter;
+
+ SampleCSVLoader(CSVRequest req, int maxDocsToLoad) {
+ super(req, new NoOpUpdateRequestProcessor());
+ this.req = req;
+ this.maxDocsToLoad = maxDocsToLoad;
+ this.multiValueDelimiter = req.getParams().get(CSV_MULTI_VALUE_DELIM_PARAM);
+ }
+
+ List<SolrInputDocument> loadDocs(ContentStream stream) throws IOException {
+ load(req, new SolrQueryResponse(), stream, processor);
+ return docs;
+ }
+
+ @Override
+ public void addDoc(int line, String[] vals) throws IOException {
+ if (maxDocsToLoad > 0 && docs.size() >= maxDocsToLoad) {
+ return; // just a short circuit, probably doesn't help that much
+ }
+
+ templateAdd.clear();
+ SolrInputDocument doc = new SolrInputDocument();
+ doAdd(line, vals, doc, templateAdd);
+ if (templateAdd.solrDoc != null) {
+ if (multiValueDelimiter != null) {
+ for (SolrInputField field : templateAdd.solrDoc.values()) {
+ if (field.getValueCount() == 1) {
+ Object value = field.getFirstValue();
+ if (value instanceof String) {
+ String[] splitValue = ((String) value).split(multiValueDelimiter);
+ if (splitValue.length > 1) {
+ field.setValue(Arrays.asList(splitValue));
+ }
+ }
+ }
+ }
+ }
+ docs.add(templateAdd.solrDoc);
+ }
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/handler/designer/DefaultSchemaSuggester.java b/solr/core/src/java/org/apache/solr/handler/designer/DefaultSchemaSuggester.java
new file mode 100644
index 00000000000..704556ea744
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/designer/DefaultSchemaSuggester.java
@@ -0,0 +1,449 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.designer;
+
+import java.io.IOException;
+import java.math.RoundingMode;
+import java.text.NumberFormat;
+import java.text.ParsePosition;
+import java.time.ZoneId;
+import java.time.ZoneOffset;
+import java.time.format.DateTimeFormatter;
+import java.time.format.DateTimeFormatterBuilder;
+import java.time.format.ResolverStyle;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import com.google.common.collect.Sets;
+import org.apache.commons.lang3.LocaleUtils;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.ManagedIndexSchema;
+import org.apache.solr.schema.NumberType;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.schema.TextField;
+import org.apache.solr.update.processor.ParseBooleanFieldUpdateProcessorFactory;
+import org.apache.solr.update.processor.ParseDateFieldUpdateProcessorFactory;
+import org.apache.solr.update.processor.ParseDoubleFieldUpdateProcessorFactory;
+import org.apache.solr.update.processor.ParseLongFieldUpdateProcessorFactory;
+
+import static org.apache.solr.common.params.CommonParams.VERSION_FIELD;
+import static org.apache.solr.update.processor.ParseDateFieldUpdateProcessorFactory.validateFormatter;
+
+// Just a quick hack to flush out the design, more intelligence is needed
+public class DefaultSchemaSuggester implements SchemaSuggester {
+
+ private static final List<String> DEFAULT_DATE_TIME_PATTERNS =
+ Arrays.asList("yyyy-MM-dd['T'[HH:mm[:ss[.SSS]][z", "yyyy-MM-dd['T'[HH:mm[:ss[,SSS]][z", "yyyy-MM-dd HH:mm[:ss[.SSS]][z", "yyyy-MM-dd HH:mm[:ss[,SSS]][z", "[EEE, ]dd MMM yyyy HH:mm[:ss] z", "EEEE, dd-MMM-yy HH:mm:ss z", "EEE MMM ppd HH:mm:ss [z ]yyyy");
+
+ private static final String FORMATS_PARAM = "format";
+ private static final String DEFAULT_TIME_ZONE_PARAM = "defaultTimeZone";
+ private static final String LOCALE_PARAM = "locale";
+ private static final String TRUE_VALUES_PARAM = "trueValue";
+ private static final String FALSE_VALUES_PARAM = "falseValue";
+ private static final String CASE_SENSITIVE_PARAM = "caseSensitive";
+
+ private static final String TYPE_CHANGE_ERROR = "Failed to parse all sample values as %s for changing type for field %s to %s";
+
+ // boolean parsing
+ private final Set<String> trueValues = new HashSet<>(Arrays.asList("true"));
+ private final Set<String> falseValues = new HashSet<>(Arrays.asList("false"));
+ private final List<DateTimeFormatter> dateTimeFormatters = new LinkedList<>();
+ private boolean caseSensitive = false;
+
+ @Override
+ public void validateTypeChange(SchemaField field, FieldType toType, List<SolrInputDocument> docs) throws IOException {
+ final NumberType toNumType = toType.getNumberType();
+ if (toNumType != null) {
+ validateNumericTypeChange(field, toType, docs, toNumType);
+ }
+ }
+
+ protected void validateNumericTypeChange(SchemaField field, FieldType toType, List<SolrInputDocument> docs, final NumberType toNumType) {
+ // desired type is numeric, make sure all the sample values are numbers
+ List<Object> fieldValues = docs.stream()
+ .map(d -> d.getFieldValue(field.getName()))
+ .filter(Objects::nonNull)
+ .flatMap(c -> (c instanceof Collection) ? ((Collection<?>) c).stream() : Stream.of(c))
+ .collect(Collectors.toList());
+ switch (toNumType) {
+ case DOUBLE:
+ case FLOAT:
+ if (isFloatOrDouble(fieldValues, Locale.ROOT) == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ String.format(Locale.ROOT, TYPE_CHANGE_ERROR, toNumType.name(), field.getName(), toType.getTypeName()));
+ }
+ break;
+ case LONG:
+ case INTEGER:
+ if (isIntOrLong(fieldValues, Locale.ROOT) == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ String.format(Locale.ROOT, TYPE_CHANGE_ERROR, toNumType.name(), field.getName(), toType.getTypeName()));
+ }
+ break;
+ case DATE:
+ if (!isDateTime(fieldValues)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ String.format(Locale.ROOT, TYPE_CHANGE_ERROR, toNumType.name(), field.getName(), toType.getTypeName()));
+ }
+ break;
+ }
+ }
+
+ @Override
+ public Optional<SchemaField> suggestField(String fieldName, List<Object> sampleValues, IndexSchema schema, List<String> langs) {
+
+ // start by looking at the fieldName and seeing if there is a dynamic field in the schema that already applies
+ if (schema.isDynamicField(fieldName)) {
+ return Optional.of(schema.getFieldOrNull(fieldName));
+ }
+
+ // TODO: use passed in langs
+ Locale locale = Locale.ROOT;
+
+ boolean isMV = isMultiValued(sampleValues);
+ String fieldTypeName = guessFieldType(fieldName, sampleValues, schema, isMV, locale);
+ FieldType fieldType = schema.getFieldTypeByName(fieldTypeName);
+ if (fieldType == null) {
+ // TODO: construct this field type on-the-fly ...
+ throw new IllegalStateException("FieldType '" + fieldTypeName + "' not found in the schema!");
+ }
+
+ Map<String, String> fieldProps = guessFieldProps(fieldName, fieldType, sampleValues, isMV, schema);
+ SchemaField schemaField = schema.newField(fieldName, fieldTypeName, fieldProps);
+ return Optional.of(schemaField);
+ }
+
+ @Override
+ public ManagedIndexSchema adaptExistingFieldToData(SchemaField schemaField, List<Object> sampleValues, ManagedIndexSchema schema) {
+ // Promote a single-valued to multi-valued if needed
+ if (!schemaField.multiValued() && isMultiValued(sampleValues)) {
+ // this existing field needs to be promoted to multi-valued
+ SimpleOrderedMap<Object> fieldProps = schemaField.getNamedPropertyValues(false);
+ fieldProps.add("multiValued", true);
+ fieldProps.remove("name");
+ fieldProps.remove("type");
+ schema = schema.replaceField(schemaField.getName(), schemaField.getType(), fieldProps.asShallowMap());
+ }
+ // TODO: other "healing" type operations here ... but we have to be careful about overriding explicit user changes
+ // such as a user making a text field a string field, we wouldn't want to revert that field back to text
+ return schema;
+ }
+
+ @Override
+ public Map<String, List<Object>> transposeDocs(List<SolrInputDocument> docs) {
+ Map<String, List<Object>> mapByField = new HashMap<>();
+ docs.forEach(doc -> doc.getFieldNames().forEach(f -> {
+ // skip the version field on incoming docs
+ if (!VERSION_FIELD.equals(f)) {
+ List<Object> values = mapByField.computeIfAbsent(f, k -> new LinkedList<>());
+ Collection<Object> fieldValues = doc.getFieldValues(f);
+ if (fieldValues != null && !fieldValues.isEmpty()) {
+ if (fieldValues.size() == 1) {
+ // flatten so every field doesn't end up multi-valued
+ values.add(fieldValues.iterator().next());
+ } else {
+ // truly multi-valued
+ values.add(fieldValues);
+ }
+ }
+ }
+ }));
+ return mapByField;
+ }
+
+ protected String guessFieldType(String fieldName, final List<Object> sampleValues, IndexSchema schema, boolean isMV, Locale locale) {
+ String type = null;
+
+ // flatten values to a single stream for easier analysis; also remove nulls
+ List<Object> flattened = sampleValues.stream()
+ .flatMap(c -> (c instanceof Collection) ? ((Collection<?>) c).stream() : Stream.of(c))
+ .filter(Objects::nonNull)
+ .collect(Collectors.toList());
+
+ if (isBoolean(flattened)) {
+ type = isMV ? "booleans" : "boolean";
+ } else {
+ String intType = isIntOrLong(flattened, locale);
+ if (intType != null) {
+ type = isMV ? intType + "s" : intType;
+ } else {
+ String floatType = isFloatOrDouble(flattened, locale);
+ if (floatType != null) {
+ type = isMV ? floatType + "s" : floatType;
+ }
+ }
+ }
+
+ if (type == null) {
+ if (isDateTime(flattened)) {
+ type = isMV ? "pdates" : "pdate";
+ } else if (isText(flattened)) {
+ type = "en".equals(locale.getLanguage()) ? "text_en" : "text_general";
+ }
+ }
+
+ // if we get here and haven't made a decision, it's a string
+ if (type == null) {
+ type = isMV ? "strings" : "string";
+ }
+
+ return type;
+ }
+
+ protected boolean isText(List<Object> values) {
+ if (values == null || values.isEmpty()) {
+ return false;
+ }
+
+ int maxLength = -1;
+ int maxTerms = -1;
+ for (Object next : values) {
+ if (!(next instanceof String)) {
+ return false;
+ }
+
+ String cs = (String) next;
+ int len = cs.length();
+ if (len > maxLength) {
+ maxLength = len;
+ }
+
+ String[] terms = cs.split("\\s+");
+ if (terms.length > maxTerms) {
+ maxTerms = terms.length;
+ }
+ }
+
+ // don't want to choose text for fields where string will do
+ // if most of the sample values are unique but only a few terms, then it's likely a text field
+ return (maxLength > 60 || maxTerms > 12 || (maxTerms > 4 && values.size() >= 10 && ((float) Sets.newHashSet(values).size() / values.size()) > 0.9f));
+ }
+
+ protected String isFloatOrDouble(List<Object> values, Locale locale) {
+ NumberFormat format = NumberFormat.getInstance(locale);
+ format.setParseIntegerOnly(false);
+ format.setRoundingMode(RoundingMode.CEILING);
+ //boolean isFloat = true;
+ for (Object next : values) {
+ Object parsed = ParseDoubleFieldUpdateProcessorFactory.parsePossibleDouble(next, format);
+ if (parsed == null) {
+ // not a double ...
+ return null;
+ }
+
+ /*
+ Tried to be clever and pick pfloat if double precision is not needed, but the ParseDoubleFieldUpdateProcessorFactory
+ doesn't work with pfloat, so you don't get any locale sensitive parsing in the URP chain, so pdouble it is ...
+
+ Number num = (Number) parsed;
+ String str = num.toString();
+ int dotAt = str.indexOf('.');
+ if (dotAt != -1) {
+ String scalePart = str.substring(dotAt + 1);
+ if (scalePart.length() > 2) {
+ isFloat = false;
+ }
+ }
+ */
+ }
+
+ return "pdouble";
+ }
+
+ protected boolean isBoolean(List<Object> values) {
+ for (Object next : values) {
+ Object parsed = ParseBooleanFieldUpdateProcessorFactory.parsePossibleBoolean(next, caseSensitive, trueValues, falseValues);
+ if (parsed == null) {
+ return false;
+ }
+ }
+ // all values are booleans
+ return true;
+ }
+
+ protected String isIntOrLong(List<Object> values, Locale locale) {
+ NumberFormat format = NumberFormat.getInstance(locale);
+ format.setParseIntegerOnly(true);
+ long maxLong = Long.MIN_VALUE;
+ for (Object next : values) {
+ Object parsed = ParseLongFieldUpdateProcessorFactory.parsePossibleLong(next, format);
+ if (parsed == null) {
+ // not a long ...
+ return null;
+ } else {
+ long parsedLong = ((Number) parsed).longValue();
+ if (parsedLong > maxLong) {
+ maxLong = parsedLong;
+ }
+ }
+ }
+
+ // if all values are less than some smallish threshold, then it's likely this field holds small numbers
+ // but be very conservative here as it's simply an optimization and we can always fall back to long
+ return maxLong < 10000 ? "pint" : "plong";
+ }
+
+ protected boolean isDateTime(List<Object> values) {
+ if (dateTimeFormatters.isEmpty()) {
+ return false;
+ }
+
+ for (Object next : values) {
+ Object parsedDate = ParseDateFieldUpdateProcessorFactory.parsePossibleDate(next, dateTimeFormatters, new ParsePosition(0));
+ if (parsedDate == null) {
+ // not a date value
+ return false;
+ }
+ }
+ return true;
+ }
+
+ public boolean isMultiValued(String name, List<SolrInputDocument> docs) {
+ Map<String, List<Object>> transposed = transposeDocs(docs);
+ List<Object> sampleValues = transposed.get(name);
+ return sampleValues != null && isMultiValued(sampleValues);
+ }
+
+ protected boolean isMultiValued(final List<Object> sampleValues) {
+ for (Object next : sampleValues) {
+ if (next instanceof Collection) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ protected Map<String, String> guessFieldProps(String fieldName, FieldType fieldType, List<Object> sampleValues, boolean isMV, IndexSchema schema) {
+ Map<String, String> props = new HashMap<>();
+ props.put("indexed", "true");
+
+ if (isMV && !fieldType.isMultiValued()) {
+ props.put("multiValued", "true"); // override the mv setting on the type
+ }
+
+ boolean docValues = true;
+ if (fieldType instanceof TextField) {
+ docValues = false;
+ } else {
+ // Not sure if this field supports docValues, so try creating a SchemaField
+ Map<String, String> tmpProps = new HashMap<>(props);
+ tmpProps.put("docValues", "true"); // to test if docValues is supported
+ try {
+ fieldType.checkSchemaField(schema.newField(fieldName, fieldType.getTypeName(), tmpProps));
+ } catch (SolrException solrException) {
+ docValues = false;
+ }
+ }
+
+ props.put("docValues", String.valueOf(docValues));
+
+ if (!docValues) {
+ props.put("stored", "true");
+ } else {
+ props.put("stored", "false");
+ props.put("useDocValuesAsStored", "true");
+ }
+
+ return props;
+ }
+
+ @Override
+ @SuppressWarnings({"rawtypes"})
+ public void init(NamedList args) {
+ initDateTimeFormatters(args);
+ initBooleanParsing(args);
+ }
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ protected void initDateTimeFormatters(NamedList args) {
+
+ Locale locale = Locale.US;
+ String localeParam = (String) args.remove(LOCALE_PARAM);
+ if (null != localeParam) {
+ locale = LocaleUtils.toLocale(localeParam);
+ }
+
+ ZoneId defaultTimeZone = ZoneOffset.UTC;
+ Object defaultTimeZoneParam = args.remove(DEFAULT_TIME_ZONE_PARAM);
+ if (null != defaultTimeZoneParam) {
+ defaultTimeZone = ZoneId.of(defaultTimeZoneParam.toString());
+ }
+
+ Collection<String> dateTimePatterns = args.removeConfigArgs(FORMATS_PARAM);
+ if (dateTimePatterns == null || dateTimePatterns.isEmpty()) {
+ dateTimePatterns = DEFAULT_DATE_TIME_PATTERNS;
+ }
+
+ for (String pattern : dateTimePatterns) {
+ DateTimeFormatter formatter = new DateTimeFormatterBuilder().parseLenient().parseCaseInsensitive()
+ .appendPattern(pattern).toFormatter(locale).withResolverStyle(ResolverStyle.LENIENT).withZone(defaultTimeZone);
+ validateFormatter(formatter);
+ dateTimeFormatters.add(formatter);
+ }
+ }
+
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ protected void initBooleanParsing(NamedList args) {
+ Object caseSensitiveParam = args.remove(CASE_SENSITIVE_PARAM);
+ if (null != caseSensitiveParam) {
+ if (caseSensitiveParam instanceof Boolean) {
+ caseSensitive = (Boolean) caseSensitiveParam;
+ } else {
+ caseSensitive = Boolean.parseBoolean(caseSensitiveParam.toString());
+ }
+ }
+
+ Collection<String> trueValuesParam = args.removeConfigArgs(TRUE_VALUES_PARAM);
+ if (!trueValuesParam.isEmpty()) {
+ trueValues.clear();
+ for (String trueVal : trueValuesParam) {
+ trueValues.add(caseSensitive ? trueVal : trueVal.toLowerCase(Locale.ROOT));
+ }
+ }
+
+ Collection<String> falseValuesParam = args.removeConfigArgs(FALSE_VALUES_PARAM);
+ if (!falseValuesParam.isEmpty()) {
+ falseValues.clear();
+ for (String val : falseValuesParam) {
+ final String falseVal = caseSensitive ? val : val.toLowerCase(Locale.ROOT);
+ if (trueValues.contains(falseVal)) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Param '" + FALSE_VALUES_PARAM + "' contains a value also in param '" + TRUE_VALUES_PARAM
+ + "': '" + val + "'");
+ }
+ falseValues.add(falseVal);
+ }
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/handler/designer/ManagedSchemaDiff.java b/solr/core/src/java/org/apache/solr/handler/designer/ManagedSchemaDiff.java
new file mode 100644
index 00000000000..4fbf3e49a92
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/designer/ManagedSchemaDiff.java
@@ -0,0 +1,280 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.designer;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+import java.util.stream.Stream;
+
+import com.google.common.collect.MapDifference;
+import com.google.common.collect.Maps;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.ManagedIndexSchema;
+import org.apache.solr.schema.SchemaField;
+
+/**
+ * Utility methods for comparing managed index schemas
+ */
+public class ManagedSchemaDiff {
+
+ private static final String UPDATED_KEY_STRING = "updated";
+ private static final String ADDED_KEY_STRING = "added";
+ private static final String REMOVED_KEY_STRING = "removed";
+
+ private static final String FIELDS_KEY_STRING = "fields";
+ private static final String FIELD_TYPES_KEY_STRING = "fieldTypes";
+ private static final String DYNAMIC_FIELDS_KEY_STRING = "dynamicFields";
+ private static final String COPY_FIELDS_KEY_STRING = "copyFields";
+
+ /**
+ * Compute difference between two managed schemas. The returned map consists of changed, new, removed
+ * elements in fields, field types, dynamic fields and copy fields between input schemas.
+ *
+ * <pre> Output format when rendered to json will look like below:
+ * {@code
+ * {
+ * "fields": {
+ * "updated": {...},
+ * "added": {...},
+ * "removed": {...}
+ * },
+ * "fieldTypes": {
+ * "updated": {...},
+ * "added": {...},
+ * "removed": {...}
+ * },
+ * "dynamicFields": {
+ * "updated": {...},
+ * "added": {...},
+ * "removed": {...}
+ * },
+ * "copyFields: {
+ * "new": [...],
+ * "old": [...]
+ * }
+ * }
+ * }
+ * </pre>
+ *
+ * @param oldSchema instance of {@link ManagedIndexSchema}
+ * @param newSchema instance of {@link ManagedIndexSchema}
+ * @return the difference between two schemas
+ */
+ public static Map<String, Object> diff(ManagedIndexSchema oldSchema, ManagedIndexSchema newSchema) {
+ Map<String, Object> diff = new HashMap<>();
+
+ Map<String, Object> fieldsDiff = diff(mapFieldsToPropertyValues(oldSchema.getFields()), mapFieldsToPropertyValues(newSchema.getFields()));
+ Map<String, Object> fieldTypesDiff = diff(mapFieldTypesToPropValues(oldSchema.getFieldTypes()), mapFieldTypesToPropValues(newSchema.getFieldTypes()));
+ Map<String, Object> dynamicFieldDiff = diff(mapDynamicFieldToPropValues(oldSchema.getDynamicFields()), mapDynamicFieldToPropValues(newSchema.getDynamicFields()));
+ Map<String, Object> copyFieldDiff = diff(getCopyFieldList(oldSchema), getCopyFieldList(newSchema));
+
+ if (!fieldsDiff.isEmpty()) {
+ diff.put(FIELDS_KEY_STRING, fieldsDiff);
+ }
+ if (!fieldTypesDiff.isEmpty()) {
+ diff.put(FIELD_TYPES_KEY_STRING, fieldTypesDiff);
+ }
+ if (!dynamicFieldDiff.isEmpty()) {
+ diff.put(DYNAMIC_FIELDS_KEY_STRING, dynamicFieldDiff);
+ }
+ if (!copyFieldDiff.isEmpty()) {
+ diff.put(COPY_FIELDS_KEY_STRING, copyFieldDiff);
+ }
+
+ return diff;
+ }
+
+ /**
+ * Compute difference between two map objects with {@link SimpleOrderedMap} as values.
+ *
+ * <pre> Example of the output format when rendered to json
+ * {@code
+ * {
+ * "updated": {
+ * "stringField": [
+ * {
+ * "docValues": "false"
+ * },
+ * {
+ * "docValues": "true"
+ * }
+ * },
+ * "added": {
+ * "newstringfield: {
+ * "name": "newstringfield",
+ * "type": "string",
+ * .....
+ * }
+ * },
+ * "removed": {
+ * "oldstringfield": {
+ * "name": "oldstringfield",
+ * "type": "string",
+ * .....
+ * }
+ * }
+ * }
+ * }
+ * </pre>
+ *
+ * @param map1 instance of Map with {@link SimpleOrderedMap} elements
+ * @param map2 instance of Map with {@link SimpleOrderedMap} elements
+ * @return the difference between two Map
+ */
+ protected static Map<String, Object> diff(
+ Map<String, SimpleOrderedMap<Object>> map1,
+ Map<String, SimpleOrderedMap<Object>> map2) {
+ Map<String, List<Map<String, Object>>> changedValues = new HashMap<>();
+ Map<String, SimpleOrderedMap<Object>> newValues = new HashMap<>();
+ Map<String, SimpleOrderedMap<Object>> removedValues = new HashMap<>();
+ for (String fieldName : map1.keySet()) {
+ if (map2.containsKey(fieldName)) {
+ SimpleOrderedMap<Object> oldPropValues = map1.get(fieldName);
+ SimpleOrderedMap<Object> newPropValues = map2.get(fieldName);
+ if (!oldPropValues.equals(newPropValues)) {
+ List<Map<String, Object>> mapDiff = getMapDifference(oldPropValues, newPropValues);
+ if (!mapDiff.isEmpty()) {
+ changedValues.put(fieldName, mapDiff);
+ }
+ }
+ } else {
+ removedValues.put(fieldName, map1.get(fieldName));
+ }
+ }
+
+ for (String fieldName : map2.keySet()) {
+ if (!map1.containsKey(fieldName)) {
+ newValues.put(fieldName, map2.get(fieldName));
+ }
+ }
+
+ Map<String, Object> mapDiff = new HashMap<>();
+ if (!changedValues.isEmpty()) {
+ mapDiff.put(UPDATED_KEY_STRING, changedValues);
+ }
+ if (!newValues.isEmpty()) {
+ mapDiff.put(ADDED_KEY_STRING, newValues);
+ }
+ if (!removedValues.isEmpty()) {
+ mapDiff.put(REMOVED_KEY_STRING, removedValues);
+ }
+
+ return mapDiff;
+ }
+
+ /**
+ * Compute difference between two {@link SimpleOrderedMap} instances
+ *
+ * <pre> Output format example when rendered to json
+ * {@code
+ * [
+ * {
+ * "stored": false,
+ * "type": "string",
+ * "multiValued": false
+ * },
+ * {
+ * "stored": true,
+ * "type": "strings",
+ * "multiValued": true
+ * }
+ * ]
+ * }
+ * </pre>
+ *
+ * @param simpleOrderedMap1 Map to treat as "left" map
+ * @param simpleOrderedMap2 Map to treat as "right" map
+ * @return List containing the left diff and right diff
+ */
+ @SuppressWarnings("unchecked")
+ private static List<Map<String, Object>> getMapDifference(
+ SimpleOrderedMap<Object> simpleOrderedMap1,
+ SimpleOrderedMap<Object> simpleOrderedMap2) {
+ Map<String, Object> map1 = simpleOrderedMap1.toMap(new HashMap<>());
+ Map<String, Object> map2 = simpleOrderedMap2.toMap(new HashMap<>());
+ Map<String, MapDifference.ValueDifference<Object>> mapDiff = Maps.difference(map1, map2).entriesDiffering();
+ if (mapDiff.isEmpty()) {
+ return Collections.emptyList();
+ }
+ Map<String, Object> leftMapDiff =
+ mapDiff.entrySet().stream()
+ .map(entry -> Map.entry(entry.getKey(), entry.getValue().leftValue()))
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+ Map<String, Object> rightMapDiff =
+ mapDiff.entrySet().stream()
+ .map(entry -> Map.entry(entry.getKey(), entry.getValue().rightValue()))
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+ return Arrays.asList(leftMapDiff, rightMapDiff);
+ }
+
+ protected static Map<String, Object> diff(List<SimpleOrderedMap<Object>> list1, List<SimpleOrderedMap<Object>> list2) {
+ List<SimpleOrderedMap<Object>> oldList = new ArrayList<>(); // ordered map changed in list1 compared to list2
+ List<SimpleOrderedMap<Object>> newList = new ArrayList<>(); // ordered map changed in list2 compared to list1
+
+ list1.forEach(som -> {
+ if (!list2.contains(som)) {
+ oldList.add(som);
+ }
+ });
+
+ list2.forEach(som -> {
+ if (!list1.contains(som)) {
+ newList.add(som);
+ }
+ });
+
+ Map<String, Object> mapDiff = new HashMap<>();
+ if (!oldList.isEmpty()) {
+ mapDiff.put("old", oldList);
+ }
+ if (!newList.isEmpty()) {
+ mapDiff.put("new", newList);
+ }
+ return mapDiff;
+ }
+
+ protected static Map<String, SimpleOrderedMap<Object>> mapFieldsToPropertyValues(Map<String, SchemaField> fields) {
+ Map<String, SimpleOrderedMap<Object>> propValueMap = new HashMap<>();
+ fields.forEach((k, v) -> propValueMap.put(k, v.getNamedPropertyValues(true)));
+ return propValueMap;
+ }
+
+ protected static Map<String, SimpleOrderedMap<Object>> mapFieldTypesToPropValues(Map<String, FieldType> fieldTypes) {
+ Map<String, SimpleOrderedMap<Object>> propValueMap = new HashMap<>();
+ fieldTypes.forEach((k, v) -> propValueMap.put(k, v.getNamedPropertyValues(true)));
+ return propValueMap;
+ }
+
+ protected static Map<String, SimpleOrderedMap<Object>> mapDynamicFieldToPropValues(IndexSchema.DynamicField[] dynamicFields) {
+ return
+ Stream.of(dynamicFields)
+ .map(df -> Map.entry(df.getPrototype().getName(), df.getPrototype().getNamedPropertyValues(true)))
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+ }
+
+ protected static List<SimpleOrderedMap<Object>> getCopyFieldList(ManagedIndexSchema indexSchema) {
+ return indexSchema.getCopyFieldProperties(false, null, null);
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/handler/designer/SampleDocuments.java b/solr/core/src/java/org/apache/solr/handler/designer/SampleDocuments.java
new file mode 100644
index 00000000000..ee3cec82f3f
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/designer/SampleDocuments.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.designer;
+
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Objects;
+import java.util.Set;
+import java.util.stream.Collectors;
+
+import org.apache.solr.common.SolrInputDocument;
+
+import static org.apache.solr.common.params.CommonParams.JSON_MIME;
+
+public class SampleDocuments {
+ public static final SampleDocuments NONE = new SampleDocuments(null, null, null);
+ public final String contentType;
+ public final String fileSource;
+ public List<SolrInputDocument> parsed;
+
+ public SampleDocuments(List<SolrInputDocument> parsed, String contentType, String fileSource) {
+ this.parsed = parsed != null ? parsed : new LinkedList<>(); // needs to be mutable
+ this.contentType = contentType;
+ this.fileSource = fileSource;
+ }
+
+ public String getSource() {
+ return fileSource != null ? fileSource : "paste";
+ }
+
+ private boolean isTextContentType() {
+ if (contentType == null) {
+ return false;
+ }
+ return contentType.contains(JSON_MIME) || contentType.startsWith("text/") || contentType.contains("application/xml");
+ }
+
+ public List<SolrInputDocument> appendDocs(String idFieldName, List<SolrInputDocument> add, int maxDocsToLoad) {
+ if (add != null && !add.isEmpty()) {
+ final Set<Object> ids =
+ parsed.stream().map(doc -> doc.getFieldValue(idFieldName)).filter(Objects::nonNull).collect(Collectors.toSet());
+ final List<SolrInputDocument> toAdd = add.stream().filter(doc -> {
+ Object id = doc.getFieldValue(idFieldName);
+ return id != null && !ids.contains(id); // doc has ID and it's not already in the set
+ }).collect(Collectors.toList());
+ parsed.addAll(toAdd);
+ if (maxDocsToLoad > 0 && parsed.size() > maxDocsToLoad) {
+ parsed = parsed.subList(0, maxDocsToLoad);
+ }
+ }
+ return parsed;
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/handler/designer/SampleDocumentsLoader.java b/solr/core/src/java/org/apache/solr/handler/designer/SampleDocumentsLoader.java
new file mode 100644
index 00000000000..0e0ba1cc800
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/designer/SampleDocumentsLoader.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.designer;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Locale;
+import java.util.UUID;
+
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.handler.designer.SampleDocuments;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.schema.StrField;
+import org.apache.solr.util.plugin.NamedListInitializedPlugin;
+
+public interface SampleDocumentsLoader extends NamedListInitializedPlugin {
+ SampleDocuments parseDocsFromStream(SolrParams params, ContentStream stream, int maxDocsToLoad) throws IOException;
+
+ /**
+ * Ensure every sample document has a unique ID, but only applies if the unique key field is a string.
+ */
+ default boolean ensureUniqueKey(final SchemaField idField, List<SolrInputDocument> docs) {
+ boolean updatedDocs = false;
+ // if the unique key field is a string, we can supply a UUID if needed, otherwise must come from the user.
+ if (StrField.class.equals(idField.getType().getClass())) {
+ String idFieldName = idField.getName();
+ for (SolrInputDocument d : docs) {
+ if (d.getFieldValue(idFieldName) == null) {
+ d.setField(idFieldName, UUID.randomUUID().toString().toLowerCase(Locale.ROOT));
+ updatedDocs = true;
+ }
+ }
+ }
+ return updatedDocs;
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerAPI.java b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerAPI.java
new file mode 100644
index 00000000000..bec0966b6cc
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerAPI.java
@@ -0,0 +1,1221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.designer;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.Reader;
+import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Iterator;
+import java.util.LinkedHashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Locale;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.TimeUnit;
+import java.util.stream.Collectors;
+
+import org.apache.solr.api.EndPoint;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.cloud.ZkConfigSetService;
+import org.apache.solr.cloud.ZkSolrResourceLoader;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.SolrInputField;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.ZkMaintenanceUtils;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.ContentStreamBase;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrConfig;
+import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.RawResponseWriter;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.schema.ManagedIndexSchema;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.util.RTimer;
+import org.apache.zookeeper.KeeperException;
+import org.noggit.JSONParser;
+import org.noggit.ObjectBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.client.solrj.SolrRequest.METHOD.GET;
+import static org.apache.solr.client.solrj.SolrRequest.METHOD.POST;
+import static org.apache.solr.client.solrj.SolrRequest.METHOD.PUT;
+import static org.apache.solr.common.StringUtils.isEmpty;
+import static org.apache.solr.common.params.CommonParams.JSON_MIME;
+import static org.apache.solr.handler.admin.ConfigSetsHandler.DEFAULT_CONFIGSET_NAME;
+import static org.apache.solr.schema.ManagedIndexSchemaFactory.DEFAULT_MANAGED_SCHEMA_RESOURCE_NAME;
+import static org.apache.solr.security.PermissionNameProvider.Name.CONFIG_EDIT_PERM;
+import static org.apache.solr.security.PermissionNameProvider.Name.CONFIG_READ_PERM;
+
+/**
+ * All V2 APIs that have a prefix of /api/schema-designer/
+ */
+public class SchemaDesignerAPI implements SchemaDesignerConstants {
+
+ private static final Set<String> excludeConfigSetNames = new HashSet<>(Arrays.asList(DEFAULT_CONFIGSET_NAME, BLOB_STORE_ID));
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private final CoreContainer coreContainer;
+ private final SchemaSuggester schemaSuggester;
+ private final SampleDocumentsLoader sampleDocLoader;
+ private final SchemaDesignerSettingsDAO settingsDAO;
+ private final SchemaDesignerConfigSetHelper configSetHelper;
+ private final Map<String, Integer> indexedVersion = new ConcurrentHashMap<>();
+
+ public SchemaDesignerAPI(CoreContainer coreContainer) {
+ this(coreContainer, SchemaDesignerAPI.newSchemaSuggester(coreContainer), SchemaDesignerAPI.newSampleDocumentsLoader(coreContainer));
+ }
+
+ SchemaDesignerAPI(CoreContainer coreContainer, SchemaSuggester schemaSuggester, SampleDocumentsLoader sampleDocLoader) {
+ this.coreContainer = coreContainer;
+ this.schemaSuggester = schemaSuggester;
+ this.sampleDocLoader = sampleDocLoader;
+ this.settingsDAO = new SchemaDesignerSettingsDAO(coreContainer);
+ this.configSetHelper = new SchemaDesignerConfigSetHelper(this.coreContainer, this.schemaSuggester);
+ }
+
+ public static SchemaSuggester newSchemaSuggester(CoreContainer coreContainer) {
+ DefaultSchemaSuggester suggester = new DefaultSchemaSuggester();
+ suggester.init(new NamedList<>());
+ return suggester;
+ }
+
+ public static SampleDocumentsLoader newSampleDocumentsLoader(CoreContainer coreContainer) {
+ SampleDocumentsLoader loader = new DefaultSampleDocumentsLoader();
+ loader.init(new NamedList<>());
+ return loader;
+ }
+
+ static String getConfigSetZkPath(final String configSet) {
+ return getConfigSetZkPath(configSet, null);
+ }
+
+ static String getConfigSetZkPath(final String configSet, final String childNode) {
+ String path = ZkConfigSetService.CONFIGS_ZKNODE + "/" + configSet;
+ if (childNode != null) {
+ path += "/" + childNode;
+ }
+ return path;
+ }
+
+ static String getMutableId(final String configSet) {
+ return DESIGNER_PREFIX + configSet;
+ }
+
+ @EndPoint(method = GET, path = "/schema-designer/info", permission = CONFIG_READ_PERM)
+ @SuppressWarnings("unchecked")
+ public void getInfo(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {
+ final String configSet = getRequiredParam(CONFIG_SET_PARAM, req);
+
+ Map<String, Object> responseMap = new HashMap<>();
+ responseMap.put(CONFIG_SET_PARAM, configSet);
+ boolean exists = configExists(configSet);
+ responseMap.put("published", exists);
+
+ // mutable config may not exist yet as this is just an info check to gather some basic info the UI needs
+ String mutableId = getMutableId(configSet);
+
+ // tricky: if there's a mutable config, prefer the settings from that first but fallback to the original if not found
+ SolrConfig srcConfig = exists ? configSetHelper.loadSolrConfig(configSet) : null;
+ SolrConfig solrConfig = configExists(mutableId) ? configSetHelper.loadSolrConfig(mutableId) : srcConfig;
+ addSettingsToResponse(settingsDAO.getSettings(solrConfig), responseMap);
+
+ responseMap.put(SCHEMA_VERSION_PARAM, configSetHelper.getCurrentSchemaVersion(mutableId));
+ responseMap.put("collections", exists ? configSetHelper.listCollectionsForConfig(configSet) : Collections.emptyList());
+
+ // don't fail if loading sample docs fails
+ try {
+ responseMap.put("numDocs", configSetHelper.getStoredSampleDocs(configSet).size());
+ } catch (Exception exc) {
+ log.warn("Failed to load sample docs from blob store for {}", configSet, exc);
+ }
+
+ rsp.getValues().addAll(responseMap);
+ }
+
+ @EndPoint(method = POST, path = "/schema-designer/prep", permission = CONFIG_EDIT_PERM)
+ @SuppressWarnings("unchecked")
+ public void prepNewSchema(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException, SolrServerException {
+ final String configSet = getRequiredParam(CONFIG_SET_PARAM, req);
+ validateNewConfigSetName(configSet);
+
+ final String copyFrom = req.getParams().get(COPY_FROM_PARAM, DEFAULT_CONFIGSET_NAME);
+
+ SchemaDesignerSettings settings = getMutableSchemaForConfigSet(configSet, -1, copyFrom);
+ ManagedIndexSchema schema = settings.getSchema();
+ String mutableId = getMutableId(configSet);
+
+ // make sure the temp collection for this analysis exists
+ if (!zkStateReader().getClusterState().hasCollection(mutableId)) {
+ indexedVersion.remove(mutableId);
+ configSetHelper.createCollection(mutableId, mutableId);
+ }
+
+ settingsDAO.persistIfChanged(mutableId, settings);
+
+ rsp.getValues().addAll(buildResponse(configSet, schema, settings, null));
+ }
+
+ @EndPoint(method = PUT, path = "/schema-designer/cleanup", permission = CONFIG_EDIT_PERM)
+ public void cleanupTemp(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException, SolrServerException {
+ cleanupTemp(getRequiredParam(CONFIG_SET_PARAM, req));
+ }
+
+ @EndPoint(method = GET, path = "/schema-designer/file", permission = CONFIG_READ_PERM)
+ @SuppressWarnings("unchecked")
+ public void getFileContents(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {
+ final String configSet = getRequiredParam(CONFIG_SET_PARAM, req);
+ final String file = getRequiredParam("file", req);
+ String filePath = getConfigSetZkPath(getMutableId(configSet), file);
+ byte[] data;
+ try {
+ data = zkStateReader().getZkClient().getData(filePath, null, null, true);
+ } catch (KeeperException | InterruptedException e) {
+ throw new IOException("Error reading file: " + filePath, SolrZkClient.checkInterrupted(e));
+ }
+ String stringData = data != null && data.length > 0 ? new String(data, StandardCharsets.UTF_8) : "";
+ rsp.getValues().addAll(Collections.singletonMap(file, stringData));
+ }
+
+ @EndPoint(method = POST, path = "/schema-designer/file", permission = CONFIG_EDIT_PERM)
+ @SuppressWarnings("unchecked")
+ public void updateFileContents(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException, SolrServerException {
+ final String configSet = getRequiredParam(CONFIG_SET_PARAM, req);
+ final String file = getRequiredParam("file", req);
+
+ String mutableId = getMutableId(configSet);
+ String zkPath = getConfigSetZkPath(mutableId, file);
+ if (!pathExistsInZk(zkPath)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "File '" + file + "' not found in configSet: " + configSet);
+ }
+
+ byte[] data = DefaultSampleDocumentsLoader.streamAsBytes(extractSingleContentStream(req, true).getStream());
+ Exception updateFileError = null;
+ if (SOLR_CONFIG_XML.equals(file)) {
+ // verify the updated solrconfig.xml is valid before saving to ZK (to avoid things blowing up later)
+ try {
+ InMemoryResourceLoader loader = new InMemoryResourceLoader(coreContainer, mutableId, SOLR_CONFIG_XML, data);
+ SolrConfig.readFromResourceLoader(loader, SOLR_CONFIG_XML, true, null);
+ } catch (Exception exc) {
+ updateFileError = exc;
+ }
+ }
+
+ if (updateFileError != null) {
+ // solrconfig.xml update failed, but haven't impacted the configSet yet, so just return the error directly
+ Throwable causedBy = SolrException.getRootCause(updateFileError);
+ Map<String, Object> response = new HashMap<>();
+ response.put("updateFileError", causedBy.getMessage());
+ response.put(file, new String(data, StandardCharsets.UTF_8));
+ rsp.getValues().addAll(response);
+ return;
+ }
+
+ // apply the update and reload the temp collection / re-index sample docs
+ SolrZkClient zkClient = zkStateReader().getZkClient();
+ try {
+ zkClient.setData(zkPath, data, true);
+ } catch (KeeperException | InterruptedException e) {
+ throw new IOException("Failed to save data in ZK at path: " + zkPath, SolrZkClient.checkInterrupted(e));
+ }
+
+ configSetHelper.reloadTempCollection(mutableId, false);
+
+ ManagedIndexSchema schema = loadLatestSchema(mutableId);
+ Map<Object, Throwable> errorsDuringIndexing = null;
+ SolrException solrExc = null;
+ List<SolrInputDocument> docs = configSetHelper.getStoredSampleDocs(configSet);
+ String[] analysisErrorHolder = new String[1];
+ if (!docs.isEmpty()) {
+ String idField = schema.getUniqueKeyField().getName();
+ try {
+ errorsDuringIndexing =
+ indexSampleDocsWithRebuildOnAnalysisError(idField, docs, mutableId, true, analysisErrorHolder);
+ } catch (SolrException exc) {
+ solrExc = exc;
+ }
+ }
+
+ Map<String, Object> response = buildResponse(configSet, schema, null, docs);
+
+ if (analysisErrorHolder[0] != null) {
+ response.put(ANALYSIS_ERROR, analysisErrorHolder[0]);
+ }
+
+ addErrorToResponse(mutableId, solrExc, errorsDuringIndexing, response,
+ "Failed to re-index sample documents after update to the " + file + " file");
+
+ rsp.getValues().addAll(response);
+ }
+
+ @EndPoint(method = GET, path = "/schema-designer/sample", permission = CONFIG_READ_PERM)
+ @SuppressWarnings("unchecked")
+ public void getSampleValue(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {
+ final String configSet = getRequiredParam(CONFIG_SET_PARAM, req);
+ final String fieldName = getRequiredParam(FIELD_PARAM, req);
+ final String idField = getRequiredParam(UNIQUE_KEY_FIELD_PARAM, req);
+ String docId = req.getParams().get(DOC_ID_PARAM);
+
+ final List<SolrInputDocument> docs = configSetHelper.getStoredSampleDocs(configSet);
+ String textValue = null;
+ if (isEmpty(docId)) {
+ // no doc ID from client ... find the first doc with a non-empty string value for fieldName
+ Optional<SolrInputDocument> doc = docs.stream()
+ .filter(d -> d.getField(fieldName) != null && d.getField(fieldName).getFirstValue() != null && !d.getField(fieldName).getFirstValue().toString().isEmpty())
+ .findFirst();
+ if (doc.isPresent()) {
+ docId = doc.get().getFieldValue(idField).toString();
+ textValue = doc.get().getField(fieldName).getFirstValue().toString();
+ }
+ } else {
+ final String idFilter = docId;
+ Optional<SolrInputDocument> doc = docs.stream().filter(d -> idFilter.equals(d.getFieldValue(idField))).findFirst();
+ if (doc.isPresent()) {
+ SolrInputField field = doc.get().getField(fieldName);
+ textValue = field != null && field.getFirstValue() != null ? field.getFirstValue().toString() : "";
+ }
+ }
+
+ if (textValue != null) {
+ Map<String, Object> analysis = configSetHelper.analyzeField(configSet, fieldName, textValue);
+ rsp.getValues().addAll(Map.of(idField, docId, fieldName, textValue, "analysis", analysis));
+ }
+ }
+
+ @EndPoint(method = GET, path = "/schema-designer/collectionsForConfig", permission = CONFIG_READ_PERM)
+ @SuppressWarnings("unchecked")
+ public void listCollectionsForConfig(SolrQueryRequest req, SolrQueryResponse rsp) {
+ final String configSet = getRequiredParam(CONFIG_SET_PARAM, req);
+ rsp.getValues().addAll(Collections.singletonMap("collections", configSetHelper.listCollectionsForConfig(configSet)));
+ }
+
+ @EndPoint(method = GET, path = "/schema-designer/configs", permission = CONFIG_READ_PERM)
+ @SuppressWarnings("unchecked")
+ public void listConfigs(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {
+ rsp.getValues().addAll(Collections.singletonMap("configSets", listEnabledConfigs()));
+ }
+
+ protected Map<String, Boolean> listEnabledConfigs() throws IOException {
+ List<String> configsInZk = configSetHelper.listConfigsInZk();
+ final Map<String, Boolean> configs = configsInZk.stream()
+ .filter(c -> !excludeConfigSetNames.contains(c) && !c.startsWith(DESIGNER_PREFIX))
+ .collect(Collectors.toMap(c -> c, c -> !settingsDAO.isDesignerDisabled(c)));
+
+ // add the in-progress but drop the _designer prefix
+ configsInZk.stream().filter(c -> c.startsWith(DESIGNER_PREFIX))
+ .map(c -> c.substring(DESIGNER_PREFIX.length()))
+ .forEach(c -> configs.putIfAbsent(c, true));
+
+ return configs;
+ }
+
+ @EndPoint(method = GET, path = "/schema-designer/download/*", permission = CONFIG_READ_PERM)
+ public void downloadConfig(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {
+ final String configSet = getRequiredParam(CONFIG_SET_PARAM, req);
+ String mutableId = getMutableId(configSet);
+
+ // find the configSet to download
+ SolrZkClient zkClient = zkStateReader().getZkClient();
+ String configId = mutableId;
+ try {
+ if (!zkClient.exists(getConfigSetZkPath(mutableId, null), true)) {
+ if (zkClient.exists(getConfigSetZkPath(configSet, null), true)) {
+ configId = configSet;
+ } else {
+ throw new SolrException(SolrException.ErrorCode.NOT_FOUND, "ConfigSet " + configSet + " not found!");
+ }
+ }
+ } catch (KeeperException | InterruptedException e) {
+ throw new IOException("Error reading config from ZK", SolrZkClient.checkInterrupted(e));
+ }
+
+ ContentStreamBase content =
+ new ContentStreamBase.ByteArrayStream(
+ configSetHelper.downloadAndZipConfigSet(configId), configSet + ".zip", "application/zip");
+ rsp.add(RawResponseWriter.CONTENT, content);
+ }
+
+ @EndPoint(method = POST, path = "/schema-designer/add", permission = CONFIG_EDIT_PERM)
+ @SuppressWarnings("unchecked")
+ public void addSchemaObject(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException, SolrServerException {
+ final String configSet = getRequiredParam(CONFIG_SET_PARAM, req);
+ final String mutableId = checkMutable(configSet, req);
+
+ Map<String, Object> addJson = readJsonFromRequest(req);
+ log.info("Adding new schema object from JSON: {}", addJson);
+
+ String objectName = configSetHelper.addSchemaObject(configSet, addJson);
+ String action = addJson.keySet().iterator().next();
+
+ ManagedIndexSchema schema = loadLatestSchema(mutableId);
+ Map<String, Object> response = buildResponse(configSet, schema, null, configSetHelper.getStoredSampleDocs(configSet));
+ response.put(action, objectName);
+ rsp.getValues().addAll(response);
+ }
+
+ @EndPoint(method = PUT, path = "/schema-designer/update", permission = CONFIG_EDIT_PERM)
+ @SuppressWarnings("unchecked")
+ public void updateSchemaObject(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException, SolrServerException {
+ final String configSet = getRequiredParam(CONFIG_SET_PARAM, req);
+ final String mutableId = checkMutable(configSet, req);
+
+ // Updated field definition is in the request body as JSON
+ Map<String, Object> updateField = readJsonFromRequest(req);
+ String name = (String) updateField.get("name");
+ if (isEmpty(name)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Invalid update request! JSON payload is missing the required name property: " + updateField);
+ }
+ log.info("Updating schema object: configSet={}, mutableId={}, name={}, JSON={}", configSet, mutableId, name, updateField);
+
+ SchemaDesignerSettings settings = getMutableSchemaForConfigSet(configSet, -1, null);
+ ManagedIndexSchema schemaBeforeUpdate = settings.getSchema();
+
+ Map<String, Object> updateResult = configSetHelper.updateSchemaObject(configSet, updateField, schemaBeforeUpdate);
+ SolrException solrExc = (SolrException) updateResult.get("solrExc");
+ String updateError = (String) updateResult.get(UPDATE_ERROR);
+ String updateType = (String) updateResult.get("updateType");
+ boolean needsRebuild = (boolean) updateResult.get("rebuild");
+
+ // re-index the docs if no error to this point
+ final ManagedIndexSchema schema = loadLatestSchema(mutableId);
+ List<SolrInputDocument> docs = configSetHelper.getStoredSampleDocs(configSet);
+ Map<Object, Throwable> errorsDuringIndexing = null;
+ String[] analysisErrorHolder = new String[1];
+ if (solrExc == null && !docs.isEmpty()) {
+ try {
+ errorsDuringIndexing =
+ indexSampleDocsWithRebuildOnAnalysisError(schema.getUniqueKeyField().getName(), docs, mutableId, false, analysisErrorHolder);
+ } catch (SolrException exc) {
+ solrExc = exc;
+ updateError = "Failed to re-index sample documents after update to the " + name + " " + updateType + " due to: " + solrExc.getMessage();
+ }
+ }
+
+ Map<String, Object> response = buildResponse(configSet, schema, settings, docs);
+ response.put("updateType", updateType);
+ if (FIELD_PARAM.equals(updateType)) {
+ response.put(updateType, fieldToMap(schema.getField(name), schema));
+ } else if ("type".equals(updateType)) {
+ response.put(updateType, schema.getFieldTypeByName(name).getNamedPropertyValues(true));
+ }
+
+ if (analysisErrorHolder[0] != null) {
+ response.put(ANALYSIS_ERROR, analysisErrorHolder[0]);
+ }
+
+ addErrorToResponse(mutableId, solrExc, errorsDuringIndexing, response, updateError);
+
+ response.put("rebuild", needsRebuild);
+ rsp.getValues().addAll(response);
+ }
+
+ @EndPoint(method = PUT, path = "/schema-designer/publish", permission = CONFIG_EDIT_PERM)
+ @SuppressWarnings("unchecked")
+ public void publish(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException, SolrServerException {
+ final String configSet = getRequiredParam(CONFIG_SET_PARAM, req);
+ final String mutableId = checkMutable(configSet, req);
+
+ // verify the configSet we're going to apply changes to has not changed since being loaded for editing by the schema designer
+ SchemaDesignerSettings settings = settingsDAO.getSettings(mutableId);
+ final Optional<Integer> publishedVersion = settings.getPublishedVersion();
+ if (publishedVersion.isPresent()) {
+ int currentVersionOfSrc = configSetHelper.getCurrentSchemaVersion(configSet);
+ if (publishedVersion.get() != currentVersionOfSrc) {
+ throw new SolrException(SolrException.ErrorCode.CONFLICT,
+ "Version mismatch for " + configSet + "! Expected version " + publishedVersion.get() + " but current is " + currentVersionOfSrc +
+ "; another user may have changed the published schema while you were making edits. " +
+ "Publishing your changes would result in losing the edits from the other user.");
+ }
+ }
+
+ String newCollection = req.getParams().get(NEW_COLLECTION_PARAM);
+ if (!isEmpty(newCollection) && zkStateReader().getClusterState().hasCollection(newCollection)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Collection '" + newCollection + "' already exists!");
+ }
+
+ if (configExists(configSet)) {
+ SolrZkClient zkClient = coreContainer.getZkController().getZkClient();
+ try {
+ zkClient.zkTransfer(getConfigSetZkPath(mutableId), true, getConfigSetZkPath(configSet), true, true);
+ } catch (KeeperException | InterruptedException e) {
+ throw new IOException("Failed to copy config set: " + mutableId, SolrZkClient.checkInterrupted(e));
+ }
+ } else {
+ copyConfig(mutableId, configSet);
+ }
+
+ boolean reloadCollections = req.getParams().getBool(RELOAD_COLLECTIONS_PARAM, false);
+ if (reloadCollections) {
+ log.debug("Reloading collections after update to configSet: {}", configSet);
+ List<String> collectionsForConfig = configSetHelper.listCollectionsForConfig(configSet);
+ CloudSolrClient csc = cloudClient();
+ for (String next : collectionsForConfig) {
+ CollectionAdminRequest.reloadCollection(next).processAsync(csc);
+ }
+ }
+
+ // create new collection
+ Map<Object, Throwable> errorsDuringIndexing = null;
+ if (!isEmpty(newCollection)) {
+ int numShards = req.getParams().getInt("numShards", 1);
+ int rf = req.getParams().getInt("replicationFactor", 1);
+ configSetHelper.createCollection(newCollection, configSet, numShards, rf);
+ if (req.getParams().getBool(INDEX_TO_COLLECTION_PARAM, false)) {
+ List<SolrInputDocument> docs = configSetHelper.getStoredSampleDocs(configSet);
+ if (!docs.isEmpty()) {
+ ManagedIndexSchema schema = loadLatestSchema(mutableId);
+ errorsDuringIndexing = indexSampleDocs(schema.getUniqueKeyField().getName(), docs, newCollection, true);
+ }
+ }
+ }
+
+ if (req.getParams().getBool(CLEANUP_TEMP_PARAM, true)) {
+ cleanupTemp(configSet);
+ }
+
+ settings.setDisabled(req.getParams().getBool(DISABLE_DESIGNER_PARAM, false));
+ settingsDAO.persistIfChanged(configSet, settings);
+
+ Map<String, Object> response = new HashMap<>();
+ response.put(CONFIG_SET_PARAM, configSet);
+ response.put(SCHEMA_VERSION_PARAM, configSetHelper.getCurrentSchemaVersion(configSet));
+ if (!isEmpty(newCollection)) {
+ response.put(NEW_COLLECTION_PARAM, newCollection);
+ }
+
+ addErrorToResponse(newCollection, null, errorsDuringIndexing, response, null);
+
+ rsp.getValues().addAll(response);
+ }
+
+ @EndPoint(method = POST, path = "/schema-designer/analyze", permission = CONFIG_EDIT_PERM)
+ @SuppressWarnings("unchecked")
+ public void analyze(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException, SolrServerException {
+ final int schemaVersion = req.getParams().getInt(SCHEMA_VERSION_PARAM, -1);
+ final String configSet = getRequiredParam(CONFIG_SET_PARAM, req);
+
+ // don't let the user edit the _default configSet with the designer (for now)
+ if (DEFAULT_CONFIGSET_NAME.equals(configSet)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "'" + DEFAULT_CONFIGSET_NAME + "' is a reserved configSet name! Please choose a different name.");
+ }
+ validateNewConfigSetName(configSet);
+
+ // Get the sample documents to analyze, preferring those in the request but falling back to previously stored
+ SampleDocuments sampleDocuments = loadSampleDocuments(req, configSet);
+
+ // Get a mutable "temp" schema either from the specified copy source or configSet if it already exists.
+ String copyFrom = configExists(configSet) ? configSet
+ : req.getParams().get(COPY_FROM_PARAM, DEFAULT_CONFIGSET_NAME);
+
+ String mutableId = getMutableId(configSet);
+
+ // holds additional settings needed by the designer to maintain state
+ SchemaDesignerSettings settings = getMutableSchemaForConfigSet(configSet, schemaVersion, copyFrom);
+ ManagedIndexSchema schema = settings.getSchema();
+
+ String uniqueKeyFieldParam = req.getParams().get(UNIQUE_KEY_FIELD_PARAM);
+ if (!isEmpty(uniqueKeyFieldParam)) {
+ String uniqueKeyField = schema.getUniqueKeyField() != null ? schema.getUniqueKeyField().getName() : null;
+ if (!uniqueKeyFieldParam.equals(uniqueKeyField)) {
+ // The Schema API doesn't support changing the ID field so would have to use XML directly
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Changing the unique key field not supported yet!");
+ }
+ }
+
+ boolean langsUpdated = false;
+ String[] languages = req.getParams().getParams(LANGUAGES_PARAM);
+ List<String> langs;
+ if (languages != null) {
+ langs = languages.length == 0 || (languages.length == 1 && "*".equals(languages[0])) ? Collections.emptyList() : Arrays.asList(languages);
+ if (!langs.equals(settings.getLanguages())) {
+ settings.setLanguages(langs);
+ langsUpdated = true;
+ }
+ } else {
+ // nothing from client, go with what's stored in the settings ...
+ langs = settings.getLanguages();
+ }
+
+ boolean dynamicUpdated = false;
+ Boolean enableDynamicFields = req.getParams().getBool(ENABLE_DYNAMIC_FIELDS_PARAM);
+ if (enableDynamicFields != null && enableDynamicFields != settings.dynamicFieldsEnabled()) {
+ settings.setDynamicFieldsEnabled(enableDynamicFields);
+ dynamicUpdated = true;
+ }
+
+ if (langsUpdated) {
+ schema = configSetHelper.syncLanguageSpecificObjectsAndFiles(mutableId, schema, langs, settings.dynamicFieldsEnabled(), copyFrom);
+ }
+
+ if (dynamicUpdated) {
+ if (!enableDynamicFields) {
+ schema = configSetHelper.removeDynamicFields(schema);
+ } else {
+ schema = configSetHelper.restoreDynamicFields(schema, langs, copyFrom);
+ }
+ }
+
+ List<SolrInputDocument> docs = sampleDocuments.parsed;
+ if (!docs.isEmpty()) {
+ if (sampleDocLoader.ensureUniqueKey(schema.getUniqueKeyField(), docs)) {
+ configSetHelper.storeSampleDocs(configSet, docs);
+ }
+ schema = analyzeInputDocs(schemaSuggester.transposeDocs(docs), schema, langs);
+ }
+
+ // persist the updated schema
+ schema.persistManagedSchema(false);
+
+ Boolean enableFieldGuessing = req.getParams().getBool(ENABLE_FIELD_GUESSING_PARAM);
+ if (enableFieldGuessing != null && enableFieldGuessing != settings.fieldGuessingEnabled()) {
+ settings.setFieldGuessingEnabled(enableFieldGuessing);
+ }
+
+ // make sure the temp collection for this analysis exists
+ if (!zkStateReader().getClusterState().hasCollection(mutableId)) {
+ configSetHelper.createCollection(mutableId, mutableId);
+ indexedVersion.remove(mutableId);
+ } else {
+ // already created in the prep step ... reload it to pull in the updated schema
+ CollectionAdminRequest.reloadCollection(mutableId).process(cloudClient());
+ }
+
+ // nested docs
+ Boolean enableNestedDocs = req.getParams().getBool(ENABLE_NESTED_DOCS_PARAM);
+ if (enableNestedDocs != null && enableNestedDocs != settings.nestedDocsEnabled()) {
+ settings.setNestedDocsEnabled(enableNestedDocs);
+ configSetHelper.toggleNestedDocsFields(schema, enableNestedDocs);
+ }
+
+ // index the sample docs using the suggested schema
+ String[] analysisErrorHolder = new String[1];
+ Map<Object, Throwable> errorsDuringIndexing = null;
+ if (!docs.isEmpty()) {
+ String idField = schema.getUniqueKeyField().getName();
+ errorsDuringIndexing =
+ indexSampleDocsWithRebuildOnAnalysisError(idField, docs, mutableId, false, analysisErrorHolder);
+ }
+
+ if (settingsDAO.persistIfChanged(mutableId, settings)) {
+ CollectionAdminRequest.reloadCollection(mutableId).process(cloudClient());
+ }
+
+ Map<String, Object> response = buildResponse(configSet, loadLatestSchema(mutableId), settings, docs);
+ response.put("sampleSource", sampleDocuments.getSource());
+ if (analysisErrorHolder[0] != null) {
+ response.put(ANALYSIS_ERROR, analysisErrorHolder[0]);
+ }
+ addErrorToResponse(mutableId, null, errorsDuringIndexing, response, null);
+ rsp.getValues().addAll(response);
+ }
+
+ @EndPoint(method = GET, path = "/schema-designer/query", permission = CONFIG_READ_PERM)
+ @SuppressWarnings("unchecked")
+ public void query(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException, SolrServerException {
+ final String configSet = getRequiredParam(CONFIG_SET_PARAM, req);
+ String mutableId = getMutableId(configSet);
+ if (!configExists(mutableId)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ mutableId + " configSet not found! Are you sure " + configSet + " was being edited by the schema designer?");
+ }
+
+ if (!zkStateReader().getClusterState().hasCollection(mutableId)) {
+ indexedVersion.remove(mutableId);
+ configSetHelper.createCollection(mutableId, mutableId);
+ }
+
+ // only re-index if current state of test collection is not up-to-date
+ int currentVersion = configSetHelper.getCurrentSchemaVersion(mutableId);
+ Integer version = indexedVersion.get(mutableId);
+ Map<Object, Throwable> errorsDuringIndexing = null;
+ if (version == null || version != currentVersion) {
+ log.debug("Schema for collection {} is stale ({} != {}), need to re-index sample docs", mutableId, version, currentVersion);
+ List<SolrInputDocument> docs = configSetHelper.getStoredSampleDocs(configSet);
+ ManagedIndexSchema schema = loadLatestSchema(mutableId);
+ errorsDuringIndexing =
+ indexSampleDocsWithRebuildOnAnalysisError(schema.getUniqueKeyField().getName(), docs, mutableId, true, null);
+ // the version changes when you index (due to field guessing URP)
+ currentVersion = configSetHelper.getCurrentSchemaVersion(mutableId);
+ indexedVersion.put(mutableId, currentVersion);
+ }
+
+ if (errorsDuringIndexing != null) {
+ Map<String, Object> response = new HashMap<>();
+ rsp.setException(new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Failed to re-index sample documents after schema updated."));
+ response.put(ERROR_DETAILS, errorsDuringIndexing);
+ rsp.getValues().addAll(response);
+ return;
+ }
+
+ // execute the user's query against the temp collection
+ QueryResponse qr = cloudClient().query(mutableId, req.getParams());
+ rsp.getValues().addAll(qr.getResponse());
+ }
+
+ /**
+ * Return the diff of designer schema with the source schema (either previously published or the copyFrom).
+ */
+ @EndPoint(method = GET,
+ path = "/schema-designer/diff",
+ permission = CONFIG_READ_PERM)
+ @SuppressWarnings("unchecked")
+ public void getSchemaDiff(SolrQueryRequest req, SolrQueryResponse rsp) throws IOException {
+ final String configSet = getRequiredParam(CONFIG_SET_PARAM, req);
+
+ SchemaDesignerSettings settings = getMutableSchemaForConfigSet(configSet, -1, null);
+ // diff the published if found, else use the original source schema
+ String sourceSchema = configExists(configSet) ? configSet : settings.getCopyFrom();
+ Map<String, Object> response = new HashMap<>();
+ response.put("diff", ManagedSchemaDiff.diff(loadLatestSchema(sourceSchema), settings.getSchema()));
+ response.put("diff-source", sourceSchema);
+ addSettingsToResponse(settings, response);
+ rsp.getValues().addAll(response);
+ }
+
+ protected SampleDocuments loadSampleDocuments(SolrQueryRequest req, String configSet) throws IOException {
+ List<SolrInputDocument> docs = null;
+ ContentStream stream = extractSingleContentStream(req, false);
+ SampleDocuments sampleDocs = null;
+ if (stream != null && stream.getContentType() != null) {
+ sampleDocs = sampleDocLoader.parseDocsFromStream(req.getParams(), stream, MAX_SAMPLE_DOCS);
+ docs = sampleDocs.parsed;
+ if (!docs.isEmpty()) {
+ // user posted in some docs, if there are already docs stored in the blob store, then add these to the existing set
+ List<SolrInputDocument> stored = configSetHelper.getStoredSampleDocs(configSet);
+ if (!stored.isEmpty()) {
+ // keep the docs in the request as newest
+ ManagedIndexSchema latestSchema = loadLatestSchema(getMutableId(configSet));
+ docs = sampleDocs.appendDocs(latestSchema.getUniqueKeyField().getName(), stored, MAX_SAMPLE_DOCS);
+ }
+
+ // store in the blob store so we always have access to these docs
+ configSetHelper.storeSampleDocs(configSet, docs);
+ }
+ }
+
+ if (docs == null || docs.isEmpty()) {
+ // no sample docs in the request ... find in blob store (or fail if no docs previously stored)
+ docs = configSetHelper.getStoredSampleDocs(configSet);
+
+ // no docs? but if this schema has already been published, it's OK, we can skip the docs part
+ if (docs.isEmpty() && !configExists(configSet)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "No sample documents provided for analyzing schema! Only CSV/TSV, XML, JSON, and JSON lines supported.");
+ }
+
+ sampleDocs = new SampleDocuments(docs, "", "blob");
+ }
+
+ return sampleDocs;
+ }
+
+ protected ManagedIndexSchema analyzeInputDocs(final Map<String, List<Object>> docs, ManagedIndexSchema schema, List<String> langs) {
+ // collect the fields to add ... adding all fields at once is faster than one-at-a-time
+ List<SchemaField> fieldsToAdd = new LinkedList<>();
+ for (String field : docs.keySet()) {
+ List<Object> sampleValues = docs.getOrDefault(field, Collections.emptyList());
+
+ // Collapse all whitespace in fields to a single underscore
+ String normalizedField = field.trim().replaceAll("\\s+", "_");
+ if (schema.hasExplicitField(normalizedField)) {
+ SchemaField existing = schema.getField(normalizedField);
+ schema = schemaSuggester.adaptExistingFieldToData(existing, sampleValues, schema);
+ continue;
+ }
+
+ Optional<SchemaField> maybeSchemaField = schemaSuggester.suggestField(normalizedField, sampleValues, schema, langs);
+ maybeSchemaField.ifPresent(fieldsToAdd::add);
+ }
+
+ if (!fieldsToAdd.isEmpty()) {
+ schema = (ManagedIndexSchema) schema.addFields(fieldsToAdd);
+ }
+
+ return schema;
+ }
+
+ protected String getManagedSchemaZkPath(final String configSet) {
+ return getConfigSetZkPath(configSet, DEFAULT_MANAGED_SCHEMA_RESOURCE_NAME);
+ }
+
+ protected SchemaDesignerSettings getMutableSchemaForConfigSet(final String configSet, final int schemaVersion, String copyFrom) throws IOException {
+ // The designer works with mutable config sets stored in a "temp" znode in ZK instead of the "live" configSet
+ final String mutableId = getMutableId(configSet);
+
+ ManagedIndexSchema schema;
+
+ // create new from the built-in "_default" configSet
+ int publishedVersion = -1;
+ boolean isNew = false;
+ if (!configExists(mutableId)) {
+
+ // are they opening a temp of an existing?
+ if (configExists(configSet)) {
+ if (settingsDAO.isDesignerDisabled(configSet)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Schema '" + configSet + "' is locked for edits by the schema designer!");
+ }
+ publishedVersion = configSetHelper.getCurrentSchemaVersion(configSet);
+ // ignore the copyFrom as we're making a mutable temp copy of an already published configSet
+ copyConfig(configSet, mutableId);
+ copyFrom = null;
+ } else {
+ copyConfig(copyFrom, mutableId);
+ }
+ isNew = true;
+ }
+
+ SolrConfig solrConfig = configSetHelper.loadSolrConfig(mutableId);
+ schema = configSetHelper.loadLatestSchema(solrConfig);
+ if (!isNew) {
+ // schema is not new, so the provided version must match, otherwise, we're trying to edit dirty data
+ configSetHelper.checkSchemaVersion(mutableId, schemaVersion, schema.getSchemaZkVersion());
+ }
+
+ SchemaDesignerSettings settings = settingsDAO.getSettings(solrConfig);
+ if (isNew) {
+ if (!configSet.equals(copyFrom)) {
+ settings.setDisabled(false);
+ }
+
+ // remember where this new one came from
+ if (copyFrom != null) {
+ settings.setCopyFrom(copyFrom);
+ }
+
+ if (publishedVersion != -1) {
+ // keep track of the version of the configSet the mutable is derived from in case another user
+ // changes the derived from schema before we publish the mutable on top of it
+ settings.setPublishedVersion(publishedVersion);
+ }
+
+ if (!settings.nestedDocsEnabled()) {
+ schema = configSetHelper.deleteNestedDocsFieldsIfNeeded(schema, false);
+ }
+
+ if (!settings.dynamicFieldsEnabled()) {
+ schema = configSetHelper.removeDynamicFields(schema);
+ }
+
+ schema.persistManagedSchema(false);
+ }
+
+ settings.setSchema(schema);
+
+ return settings;
+ }
+
+ ManagedIndexSchema loadLatestSchema(String configSet) {
+ return configSetHelper.loadLatestSchema(configSet);
+ }
+
+ protected ContentStream extractSingleContentStream(final SolrQueryRequest req, boolean required) {
+ Iterable<ContentStream> streams = req.getContentStreams();
+ Iterator<ContentStream> iter = streams != null ? streams.iterator() : null;
+ ContentStream stream = iter != null && iter.hasNext() ? iter.next() : null;
+ if (required && stream == null)
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "No JSON content found in the request body!");
+
+ return stream;
+ }
+
+ protected CloudSolrClient cloudClient() {
+ return coreContainer.getSolrClientCache().getCloudSolrClient(coreContainer.getZkController().getZkServerAddress());
+ }
+
+ protected ZkStateReader zkStateReader() {
+ return coreContainer.getZkController().getZkStateReader();
+ }
+
+ protected Map<Object, Throwable> indexSampleDocsWithRebuildOnAnalysisError(String idField,
+ List<SolrInputDocument> docs,
+ final String collectionName,
+ boolean asBatch,
+ String[] analysisErrorHolder) throws IOException, SolrServerException {
+ Map<Object, Throwable> results;
+ try {
+ results = indexSampleDocs(idField, docs, collectionName, asBatch);
+ } catch (IllegalArgumentException analysisExc) {
+ String errMsg = SolrException.getRootCause(analysisExc).getMessage();
+ if (analysisErrorHolder != null) {
+ analysisErrorHolder[0] = errMsg;
+ }
+ log.warn("Rebuilding temp collection {} after low-level Lucene indexing issue: {}", collectionName, errMsg);
+ configSetHelper.reloadTempCollection(collectionName, true);
+ results = indexSampleDocs(idField, docs, collectionName, asBatch);
+ log.info("Re-index sample docs into {} after rebuild due to {} succeeded; results: {}", collectionName, errMsg, results);
+ }
+ return results;
+ }
+
+ protected Map<Object, Throwable> indexSampleDocs(String idField,
+ List<SolrInputDocument> docs,
+ final String collectionName,
+ boolean asBatch) throws IOException, SolrServerException {
+
+ Map<Object, Throwable> errorsDuringIndexing = new LinkedHashMap<>();
+
+ RTimer timer = new RTimer();
+
+ // load sample docs from blob store
+ CloudSolrClient cloudSolrClient = cloudClient();
+ cloudSolrClient.deleteByQuery(collectionName, "*:*", 1);
+ cloudSolrClient.optimize(collectionName, true, true, 1);
+
+ final int commitWithin = 100;
+ final int numDocs = docs.size();
+ int numAdded = 0;
+ if (asBatch) {
+ cloudSolrClient.add(collectionName, docs, commitWithin);
+ numAdded = docs.size();
+ } else {
+ int maxErrors = Math.min(100, Math.round(numDocs / 2f));
+ for (SolrInputDocument next : docs) {
+ try {
+ cloudSolrClient.add(collectionName, next, commitWithin);
+ ++numAdded;
+ } catch (Exception exc) {
+ Throwable rootCause = SolrException.getRootCause(exc);
+ if (String.valueOf(rootCause.getMessage()).contains("possible analysis error")) {
+ throw new IllegalArgumentException(rootCause);
+ } else {
+ Object docId = next.getFieldValue(idField);
+ if (docId == null) {
+ throw exc;
+ }
+ errorsDuringIndexing.put(docId, rootCause);
+
+ // break if there are a lot of errors in indexing as something is very wrong if so ...
+ if (errorsDuringIndexing.size() > 20 && errorsDuringIndexing.size() >= maxErrors) {
+ break;
+ }
+ }
+ }
+ }
+ }
+
+ cloudSolrClient.commit(collectionName, true, true, true);
+
+ if (!errorsDuringIndexing.isEmpty()) {
+ return errorsDuringIndexing;
+ }
+
+ long numFound = waitToSeeSampleDocs(collectionName, numAdded);
+ double tookMs = timer.getTime();
+ log.debug("Indexed {} docs into collection {}, took {} ms", numFound, collectionName, tookMs);
+
+ return !errorsDuringIndexing.isEmpty() ? errorsDuringIndexing : null;
+ }
+
+ protected long waitToSeeSampleDocs(String collectionName, long numAdded) throws IOException, SolrServerException {
+ CloudSolrClient cloudSolrClient = cloudClient();
+ SolrQuery query = new SolrQuery("*:*");
+ query.setRows(0);
+ QueryResponse queryResponse = cloudSolrClient.query(collectionName, query);
+ long numFound = queryResponse.getResults().getNumFound();
+ if (numFound < numAdded) {
+ // wait up to 5 seconds for this to occur
+ final long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(5);
+ do {
+ cloudSolrClient.commit(collectionName, true, true, true);
+ queryResponse = cloudSolrClient.query(collectionName, query);
+ numFound = queryResponse.getResults().getNumFound();
+ if (numFound >= numAdded) {
+ break;
+ }
+ try {
+ Thread.sleep(200); // little pause to avoid flooding the server with requests in this loop
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ break;
+ }
+ } while (System.nanoTime() < deadline);
+
+ if (numFound < numAdded) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR,
+ "Failed to index " + numAdded + " sample docs into temp collection: " + collectionName);
+ }
+ }
+ return numFound;
+ }
+
+ protected Map<String, Object> buildResponse(String configSet,
+ final ManagedIndexSchema schema,
+ SchemaDesignerSettings settings,
+ List<SolrInputDocument> docs) throws IOException {
+ String mutableId = getMutableId(configSet);
+ int currentVersion = configSetHelper.getCurrentSchemaVersion(mutableId);
+ indexedVersion.put(mutableId, currentVersion);
+
+ // response is a map of data structures to support the schema designer
+ Map<String, Object> response = new HashMap<>();
+
+ DocCollection coll = zkStateReader().getCollection(mutableId);
+ if (coll.getActiveSlicesArr().length > 0) {
+ String coreName = coll.getActiveSlicesArr()[0].getLeader().getCoreName();
+ response.put("core", coreName);
+ }
+
+ response.put(UNIQUE_KEY_FIELD_PARAM, schema.getUniqueKeyField().getName());
+
+ response.put(CONFIG_SET_PARAM, configSet);
+ // important: pass the designer the current schema zk version for MVCC
+ response.put(SCHEMA_VERSION_PARAM, currentVersion);
+ response.put(TEMP_COLLECTION_PARAM, mutableId);
+ response.put("collectionsForConfig", configSetHelper.listCollectionsForConfig(configSet));
+ // Guess at a schema for each field found in the sample docs
+ // Collect all fields across all docs with mapping to values
+ response.put("fields", schema.getFields().values().stream()
+ .map(f -> fieldToMap(f, schema))
+ .sorted(Comparator.comparing(map -> ((String) map.get("name"))))
+ .collect(Collectors.toList()));
+
+ if (settings == null) {
+ settings = settingsDAO.getSettings(mutableId);
+ }
+ addSettingsToResponse(settings, response);
+
+ response.put("dynamicFields", Arrays.stream(schema.getDynamicFieldPrototypes())
+ .map(e -> e.getNamedPropertyValues(true))
+ .sorted(Comparator.comparing(map -> ((String) map.get("name"))))
+ .collect(Collectors.toList()));
+
+ response.put("fieldTypes", schema.getFieldTypes().values().stream()
+ .map(fieldType -> fieldType.getNamedPropertyValues(true))
+ .sorted(Comparator.comparing(map -> ((String) map.get("name"))))
+ .collect(Collectors.toList()));
+
+ // files
+ SolrZkClient zkClient = zkStateReader().getZkClient();
+ String configPathInZk = getConfigSetZkPath(mutableId);
+ final Set<String> files = new HashSet<>();
+ try {
+ ZkMaintenanceUtils.traverseZkTree(zkClient, configPathInZk, ZkMaintenanceUtils.VISIT_ORDER.VISIT_POST, files::add);
+ } catch (KeeperException | InterruptedException e) {
+ throw new IOException("Failed to traverse files under: " + configPathInZk, SolrZkClient.checkInterrupted(e));
+ }
+ files.remove(configPathInZk);
+
+ final String prefix = configPathInZk + "/";
+ final int prefixLen = prefix.length();
+ Set<String> stripPrefix = files.stream().map(f -> f.startsWith(prefix) ? f.substring(prefixLen) : f).collect(Collectors.toSet());
+ stripPrefix.remove(DEFAULT_MANAGED_SCHEMA_RESOURCE_NAME);
+ stripPrefix.remove("lang");
+ stripPrefix.remove(CONFIGOVERLAY_JSON); // treat this file as private
+
+ List<String> sortedFiles = new ArrayList<>(stripPrefix);
+ Collections.sort(sortedFiles);
+ response.put("files", sortedFiles);
+
+ // info about the sample docs
+ if (docs != null) {
+ final String uniqueKeyField = schema.getUniqueKeyField().getName();
+ response.put("docIds", docs.stream()
+ .map(d -> (String) d.getFieldValue(uniqueKeyField))
+ .filter(Objects::nonNull)
+ .limit(100)
+ .collect(Collectors.toList()));
+ }
+
+ response.put("numDocs", docs != null ? docs.size() : -1);
+
+ return response;
+ }
+
+ protected void addErrorToResponse(String collection,
+ SolrException solrExc,
+ Map<Object, Throwable> errorsDuringIndexing,
+ Map<String, Object> response,
+ String updateError) {
+
+ if (solrExc == null && (errorsDuringIndexing == null || errorsDuringIndexing.isEmpty())) {
+ return; // no errors
+ }
+
+ if (updateError != null) {
+ response.put(UPDATE_ERROR, updateError);
+ }
+
+ if (solrExc != null) {
+ response.put("updateErrorCode", solrExc.code());
+ response.putIfAbsent(UPDATE_ERROR, solrExc.getMessage());
+ }
+
+ response.putIfAbsent(UPDATE_ERROR, "Index sample documents into " + collection + " failed!");
+ response.putIfAbsent("updateErrorCode", 400);
+ if (errorsDuringIndexing != null) {
+ response.put(ERROR_DETAILS, errorsDuringIndexing);
+ }
+ }
+
+ protected SimpleOrderedMap<Object> fieldToMap(SchemaField f, ManagedIndexSchema schema) {
+ SimpleOrderedMap<Object> map = f.getNamedPropertyValues(true);
+
+ // add the copy field destination field names
+ List<String> copyFieldNames =
+ schema.getCopyFieldsList((String) map.get("name")).stream().map(c -> c.getDestination().getName()).collect(Collectors.toList());
+ map.add("copyDest", String.join(",", copyFieldNames));
+
+ return map;
+ }
+
+ @SuppressWarnings("unchecked")
+ protected Map<String, Object> readJsonFromRequest(SolrQueryRequest req) throws IOException {
+ ContentStream stream = extractSingleContentStream(req, true);
+ String contentType = stream.getContentType();
+ if (isEmpty(contentType) || !contentType.toLowerCase(Locale.ROOT).contains(JSON_MIME)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Expected JSON in request!");
+ }
+ final Object json;
+ try (Reader reader = stream.getReader()) {
+ json = ObjectBuilder.getVal(new JSONParser(reader));
+ }
+ return (Map<String, Object>) json;
+ }
+
+ protected void addSettingsToResponse(SchemaDesignerSettings settings, final Map<String, Object> response) {
+ response.put(LANGUAGES_PARAM, settings.getLanguages());
+ response.put(ENABLE_FIELD_GUESSING_PARAM, settings.fieldGuessingEnabled());
+ response.put(ENABLE_DYNAMIC_FIELDS_PARAM, settings.dynamicFieldsEnabled());
+ response.put(ENABLE_NESTED_DOCS_PARAM, settings.nestedDocsEnabled());
+ response.put(DISABLED, settings.isDisabled());
+ Optional<Integer> publishedVersion = settings.getPublishedVersion();
+ publishedVersion.ifPresent(version -> response.put(PUBLISHED_VERSION, version));
+ String copyFrom = settings.getCopyFrom();
+ if (copyFrom != null) {
+ response.put(COPY_FROM_PARAM, copyFrom);
+ }
+ }
+
+ protected String checkMutable(String configSet, SolrQueryRequest req) throws IOException {
+ // an apply just copies over the temp config to the "live" location
+ String mutableId = getMutableId(configSet);
+ if (!configExists(mutableId)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ mutableId + " configSet not found! Are you sure " + configSet + " was being edited by the schema designer?");
+ }
+
+ // check the versions agree
+ configSetHelper.checkSchemaVersion(mutableId, requireSchemaVersionFromClient(req), -1);
+
+ return mutableId;
+ }
+
+ protected int requireSchemaVersionFromClient(SolrQueryRequest req) {
+ final int schemaVersion = req.getParams().getInt(SCHEMA_VERSION_PARAM, -1);
+ if (schemaVersion == -1) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ SCHEMA_VERSION_PARAM + " is a required parameter for the " + req.getPath() + " endpoint");
+ }
+ return schemaVersion;
+ }
+
+ protected String getRequiredParam(final String param, final SolrQueryRequest req) {
+ final String paramValue = req.getParams().get(param);
+ if (isEmpty(paramValue)) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ param + " is a required parameter for the " + req.getPath() + " endpoint!");
+ }
+ return paramValue;
+ }
+
+ protected void cleanupTemp(String configSet) throws IOException, SolrServerException {
+ String mutableId = getMutableId(configSet);
+ indexedVersion.remove(mutableId);
+ CollectionAdminRequest.deleteCollection(mutableId).process(cloudClient());
+ configSetHelper.deleteStoredSampleDocs(configSet);
+ deleteConfig(mutableId);
+ }
+
+ private boolean configExists(String configSet) throws IOException {
+ return coreContainer.getConfigSetService().checkConfigExists(configSet);
+ }
+
+ private void deleteConfig(String configSet) throws IOException {
+ coreContainer.getConfigSetService().deleteConfig(configSet);
+ }
+
+ private void copyConfig(String from, String to) throws IOException {
+ coreContainer.getConfigSetService().copyConfig(from, to);
+ }
+
+ private void validateNewConfigSetName(String configSet) {
+ if (configSet.length() > 50) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Schema name should be 50 characters or less");
+ }
+ if (configSet.contains(" ") || configSet.contains("/")) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Schema name should not contain spaces or forward slash");
+ }
+ }
+
+ private boolean pathExistsInZk(final String zkPath) throws IOException {
+ SolrZkClient zkClient = zkStateReader().getZkClient();
+ try {
+ return zkClient.exists(zkPath, true);
+ } catch (KeeperException | InterruptedException e) {
+ throw new IOException("Failed to check if path exists: " + zkPath, SolrZkClient.checkInterrupted(e));
+ }
+ }
+
+ private static class InMemoryResourceLoader extends SolrResourceLoader {
+ String resource;
+ byte[] data;
+ ZkSolrResourceLoader delegate;
+
+ public InMemoryResourceLoader(CoreContainer cc, String configSet, String resource, byte[] data) {
+ super(cc.getResourceLoader().getInstancePath(), cc.getResourceLoader().getClassLoader());
+
+ final SolrResourceLoader resourceLoader = cc.getResourceLoader();
+ this.delegate = new ZkSolrResourceLoader(resourceLoader.getInstancePath(), configSet, resourceLoader.getClassLoader(), cc.getZkController());
+ this.resource = resource;
+ this.data = data;
+ }
+
+ @Override
+ public InputStream openResource(String res) throws IOException {
+ return (this.resource.equals(res)) ? new ByteArrayInputStream(data) : delegate.openResource(res);
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java
new file mode 100644
index 00000000000..2b768b61c0b
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConfigSetHelper.java
@@ -0,0 +1,1061 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.designer;
+
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.net.URI;
+import java.net.URISyntaxException;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+import java.util.Set;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+import java.util.stream.Collectors;
+import java.util.zip.ZipEntry;
+import java.util.zip.ZipOutputStream;
+
+import com.google.common.collect.Sets;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.FilenameUtils;
+import org.apache.http.HttpResponse;
+import org.apache.http.HttpStatus;
+import org.apache.http.client.methods.HttpGet;
+import org.apache.http.client.methods.HttpPost;
+import org.apache.http.client.utils.URIBuilder;
+import org.apache.http.client.utils.URLEncodedUtils;
+import org.apache.http.entity.ByteArrayEntity;
+import org.apache.http.util.EntityUtils;
+import org.apache.solr.client.solrj.SolrResponse;
+import org.apache.solr.client.solrj.SolrServerException;
+import org.apache.solr.client.solrj.impl.CloudSolrClient;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.request.schema.FieldTypeDefinition;
+import org.apache.solr.client.solrj.request.schema.SchemaRequest;
+import org.apache.solr.client.solrj.response.schema.SchemaResponse;
+import org.apache.solr.cloud.ZkConfigSetService;
+import org.apache.solr.cloud.ZkSolrResourceLoader;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.cloud.ClusterState;
+import org.apache.solr.common.cloud.DocCollection;
+import org.apache.solr.common.cloud.Replica;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.cloud.UrlScheme;
+import org.apache.solr.common.cloud.ZkMaintenanceUtils;
+import org.apache.solr.common.cloud.ZkStateReader;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.common.util.Utils;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrConfig;
+import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.handler.admin.CollectionsHandler;
+import org.apache.solr.schema.CopyField;
+import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.ManagedIndexSchema;
+import org.apache.solr.schema.ManagedIndexSchemaFactory;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.schema.TextField;
+import org.apache.solr.util.RTimer;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.common.params.CommonParams.VERSION_FIELD;
+import static org.apache.solr.common.util.Utils.fromJSONString;
+import static org.apache.solr.common.util.Utils.toJavabin;
+import static org.apache.solr.handler.admin.ConfigSetsHandler.DEFAULT_CONFIGSET_NAME;
+import static org.apache.solr.handler.designer.SchemaDesignerAPI.getConfigSetZkPath;
+import static org.apache.solr.handler.designer.SchemaDesignerAPI.getMutableId;
+import static org.apache.solr.schema.IndexSchema.NEST_PATH_FIELD_NAME;
+import static org.apache.solr.schema.IndexSchema.ROOT_FIELD_NAME;
+import static org.apache.solr.schema.ManagedIndexSchemaFactory.DEFAULT_MANAGED_SCHEMA_RESOURCE_NAME;
+
+class SchemaDesignerConfigSetHelper implements SchemaDesignerConstants {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private static final Set<String> removeFieldProps = new HashSet<>(Arrays.asList("href", "id", "copyDest"));
+ private static final List<String> includeLangIds = Arrays.asList("ws", "general", "rev", "sort");
+ private static final String ZNODE_PATH_DELIM = "/";
+ private static final String MULTIVALUED = "multiValued";
+ private static final int TEXT_PREFIX_LEN = "text_".length();
+
+
+ private final CoreContainer cc;
+ private final SchemaSuggester schemaSuggester;
+
+ SchemaDesignerConfigSetHelper(CoreContainer cc, SchemaSuggester schemaSuggester) {
+ this.cc = cc;
+ this.schemaSuggester = schemaSuggester;
+ }
+
+ @SuppressWarnings("unchecked")
+ Map<String, Object> analyzeField(String configSet, String fieldName, String fieldText) throws IOException {
+ final String mutableId = getMutableId(configSet);
+ final URI uri;
+ try {
+ uri = collectionApiEndpoint(mutableId, "analysis", "field")
+ .setParameter(CommonParams.WT, CommonParams.JSON)
+ .setParameter("analysis.showmatch", "true")
+ .setParameter("analysis.fieldname", fieldName)
+ .setParameter("analysis.fieldvalue", "POST")
+ .build();
+ } catch (URISyntaxException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
+ }
+
+ Map<String, Object> analysis = Collections.emptyMap();
+ HttpPost httpPost = new HttpPost(uri);
+ httpPost.setHeader("Content-Type", "text/plain");
+ httpPost.setEntity(new ByteArrayEntity(fieldText.getBytes(StandardCharsets.UTF_8)));
+ try {
+ HttpResponse resp = cloudClient().getHttpClient().execute(httpPost);
+ int statusCode = resp.getStatusLine().getStatusCode();
+ if (statusCode != HttpStatus.SC_OK) {
+ throw new SolrException(SolrException.ErrorCode.getErrorCode(statusCode),
+ EntityUtils.toString(resp.getEntity(), StandardCharsets.UTF_8));
+ }
+
+ Map<String, Object> response = (Map<String, Object>) fromJSONString(EntityUtils.toString(resp.getEntity(), StandardCharsets.UTF_8));
+ if (response != null) {
+ analysis = (Map<String, Object>) response.get("analysis");
+ }
+ } finally {
+ httpPost.releaseConnection();
+ }
+
+ return analysis;
+ }
+
+ List<String> listCollectionsForConfig(String configSet) {
+ final List<String> collections = new LinkedList<>();
+ Map<String, ClusterState.CollectionRef> states = zkStateReader().getClusterState().getCollectionStates();
+ for (Map.Entry<String, ClusterState.CollectionRef> e : states.entrySet()) {
+ final String coll = e.getKey();
+ if (coll.startsWith(DESIGNER_PREFIX)) {
+ continue; // ignore temp
+ }
+
+ try {
+ if (configSet.equals(zkStateReader().readConfigName(coll)) && e.getValue().get() != null) {
+ collections.add(coll);
+ }
+ } catch (Exception exc) {
+ log.warn("Failed to get config name for {}", coll, exc);
+ }
+ }
+ return collections;
+ }
+
+ @SuppressWarnings("unchecked")
+ public String addSchemaObject(String configSet, Map<String, Object> addJson) throws IOException, SolrServerException {
+ String mutableId = getMutableId(configSet);
+ SchemaRequest.Update addAction;
+ String action;
+ String objectName = null;
+ if (addJson.containsKey("add-field")) {
+ action = "add-field";
+ Map<String, Object> fieldAttrs = (Map<String, Object>) addJson.get(action);
+ objectName = (String) fieldAttrs.get("name");
+ addAction = new SchemaRequest.AddField(fieldAttrs);
+ } else if (addJson.containsKey("add-dynamic-field")) {
+ action = "add-dynamic-field";
+ Map<String, Object> fieldAttrs = (Map<String, Object>) addJson.get(action);
+ objectName = (String) fieldAttrs.get("name");
+ addAction = new SchemaRequest.AddDynamicField(fieldAttrs);
+ } else if (addJson.containsKey("add-copy-field")) {
+ action = "add-copy-field";
+ Map<String, Object> map = (Map<String, Object>) addJson.get(action);
+ Object dest = map.get("dest");
+ List<String> destFields = null;
+ if (dest instanceof String) {
+ destFields = Collections.singletonList((String) dest);
+ } else if (dest instanceof List) {
+ destFields = (List<String>) dest;
+ } else if (dest instanceof Collection) {
+ Collection<String> destColl = (Collection<String>) dest;
+ destFields = new ArrayList<>(destColl);
+ }
+ addAction = new SchemaRequest.AddCopyField((String) map.get("source"), destFields);
+ } else if (addJson.containsKey("add-field-type")) {
+ action = "add-field-type";
+ Map<String, Object> fieldAttrs = (Map<String, Object>) addJson.get(action);
+ objectName = (String) fieldAttrs.get("name");
+ FieldTypeDefinition ftDef = new FieldTypeDefinition();
+ ftDef.setAttributes(fieldAttrs);
+ addAction = new SchemaRequest.AddFieldType(ftDef);
+ } else {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, "Unsupported action in request body! " + addJson);
+ }
+
+ // Using the SchemaAPI vs. working on the schema directly because SchemaField.create methods are package protected
+ log.info("Sending {} request for configSet {}: {}", action, mutableId, addJson);
+ SchemaResponse.UpdateResponse schemaResponse = addAction.process(cloudClient(), mutableId);
+ Exception exc = schemaResponse.getException();
+ if (exc instanceof SolrException) {
+ throw (SolrException) exc;
+ } else if (schemaResponse.getStatus() != 0) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, exc);
+ }
+
+ return objectName;
+ }
+
+ void reloadTempCollection(String mutableId, boolean delete) throws IOException, SolrServerException {
+ if (delete) {
+ log.debug("Deleting and re-creating existing collection {} after schema update", mutableId);
+ CollectionAdminRequest.deleteCollection(mutableId).process(cloudClient());
+ try {
+ zkStateReader().waitForState(mutableId, 30, TimeUnit.SECONDS, Objects::isNull);
+ } catch (InterruptedException | TimeoutException e) {
+ throw new IOException("Failed to see deleted collection " + mutableId + " reflected in cluster state", SolrZkClient.checkInterrupted(e));
+ }
+ createCollection(mutableId, mutableId);
+ log.debug("Deleted and re-created existing collection: {}", mutableId);
+ } else {
+ CollectionAdminRequest.reloadCollection(mutableId).process(cloudClient());
+ log.debug("Reloaded existing collection: {}", mutableId);
+ }
+ }
+
+ Map<String, Object> updateSchemaObject(String configSet, Map<String, Object> updateJson, ManagedIndexSchema schemaBeforeUpdate) throws IOException, SolrServerException {
+ String name = (String) updateJson.get("name");
+ String mutableId = getMutableId(configSet);
+
+ boolean needsRebuild = false;
+
+ SolrException solrExc = null;
+ String updateType;
+ String updateError = null;
+
+ if (updateJson.get("type") != null) {
+ updateType = schemaBeforeUpdate.isDynamicField(name) ? "dynamicField" : "field";
+ try {
+ needsRebuild = updateField(configSet, updateJson, schemaBeforeUpdate);
+ } catch (SolrException exc) {
+ if (exc.code() != 400) {
+ throw exc;
+ }
+ solrExc = exc;
+ updateError = solrExc.getMessage() + " Previous settings will be restored.";
+ }
+ } else {
+ updateType = "type";
+ needsRebuild = updateFieldType(configSet, name, updateJson, schemaBeforeUpdate);
+ }
+
+ // the update may have required a full rebuild of the index, otherwise, it's just a reload / re-index sample
+ reloadTempCollection(mutableId, needsRebuild);
+
+ Map<String, Object> results = new HashMap<>();
+ results.put("rebuild", needsRebuild);
+ results.put("updateType", updateType);
+ if (updateError != null) {
+ results.put("updateError", updateError);
+ }
+ if (solrExc != null) {
+ results.put("solrExc", solrExc);
+ }
+ return results;
+ }
+
+ protected boolean updateFieldType(String configSet, String typeName, Map<String, Object> updateJson, ManagedIndexSchema schemaBeforeUpdate) {
+ boolean needsRebuild = false;
+
+ Map<String, Object> typeAttrs = updateJson.entrySet().stream()
+ .filter(e -> !removeFieldProps.contains(e.getKey()))
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+
+ FieldType fieldType = schemaBeforeUpdate.getFieldTypeByName(typeName);
+
+ // this is a field type
+ Object multiValued = typeAttrs.get(MULTIVALUED);
+ if (typeHasMultiValuedChange(multiValued, fieldType)) {
+ needsRebuild = true;
+ log.warn("Re-building the temp collection for {} after type {} updated to multi-valued {}", configSet, typeName, multiValued);
+ }
+
+ // nice, the json for this field looks like
+ // "synonymQueryStyle": "org.apache.solr.parser.SolrQueryParserBase$SynonymQueryStyle:AS_SAME_TERM"
+ if (typeAttrs.get("synonymQueryStyle") instanceof String) {
+ String synonymQueryStyle = (String) typeAttrs.get("synonymQueryStyle");
+ if (synonymQueryStyle.lastIndexOf(':') != -1) {
+ typeAttrs.put("synonymQueryStyle", synonymQueryStyle.substring(synonymQueryStyle.lastIndexOf(':') + 1));
+ }
+ }
+
+ ManagedIndexSchema updatedSchema =
+ schemaBeforeUpdate.replaceFieldType(fieldType.getTypeName(), (String) typeAttrs.get("class"), typeAttrs);
+ updatedSchema.persistManagedSchema(false);
+
+ return needsRebuild;
+ }
+
+ boolean updateField(String configSet, Map<String, Object> updateField, ManagedIndexSchema schemaBeforeUpdate) throws IOException, SolrServerException {
+ String mutableId = getMutableId(configSet);
+
+ String name = (String) updateField.get("name");
+ String type = (String) updateField.get("type");
+ String copyDest = (String) updateField.get("copyDest");
+ Map<String, Object> fieldAttributes = updateField.entrySet().stream()
+ .filter(e -> !removeFieldProps.contains(e.getKey()))
+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue));
+
+ boolean needsRebuild = false;
+
+ SchemaField schemaField = schemaBeforeUpdate.getField(name);
+ boolean isDynamic = schemaBeforeUpdate.isDynamicField(name);
+ String currentType = schemaField.getType().getTypeName();
+
+ SimpleOrderedMap<Object> fromTypeProps;
+ if (type.equals(currentType)) {
+ // no type change, so just pull the current type's props (with defaults) as we'll use these
+ // to determine which props get explicitly overridden on the field
+ fromTypeProps = schemaBeforeUpdate.getFieldTypeByName(currentType).getNamedPropertyValues(true);
+ } else {
+ // validate type change
+ FieldType newType = schemaBeforeUpdate.getFieldTypeByName(type);
+ if (newType == null) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Invalid update request for field " + name + "! Field type " + type + " doesn't exist!");
+ }
+ validateTypeChange(configSet, schemaField, newType);
+
+ // type change looks valid
+ fromTypeProps = newType.getNamedPropertyValues(true);
+ }
+
+ // the diff holds all the explicit properties not inherited from the type
+ Map<String, Object> diff = new HashMap<>();
+ for (Map.Entry<String, Object> e : fieldAttributes.entrySet()) {
+ String attr = e.getKey();
+ Object attrValue = e.getValue();
+ if ("name".equals(attr) || "type".equals(attr)) {
+ continue; // we don't want these in the diff map
+ }
+
+ if ("required".equals(attr)) {
+ diff.put(attr, attrValue != null ? attrValue : false);
+ } else {
+ Object fromType = fromTypeProps.get(attr);
+ if (fromType == null || !fromType.equals(attrValue)) {
+ diff.put(attr, attrValue);
+ }
+ }
+ }
+
+ // detect if they're trying to copy multi-valued fields into a single-valued field
+ Object multiValued = diff.get(MULTIVALUED);
+ if (multiValued == null) {
+ // mv not overridden explicitly, but we need the actual value, which will come from the new type (if that changed) or the current field
+ multiValued = type.equals(currentType) ? schemaField.multiValued() : schemaBeforeUpdate.getFieldTypeByName(type).isMultiValued();
+ }
+
+ if (!isDynamic && Boolean.FALSE.equals(multiValued)) {
+ // make sure there are no mv source fields if this is a copy dest
+ for (String src : schemaBeforeUpdate.getCopySources(name)) {
+ SchemaField srcField = schemaBeforeUpdate.getField(src);
+ if (srcField.multiValued()) {
+ log.warn("Cannot change multi-valued field {} to single-valued because it is a copy field destination for multi-valued field {}", name, src);
+ multiValued = Boolean.TRUE;
+ diff.put(MULTIVALUED, multiValued);
+ break;
+ }
+ }
+ }
+
+ if (Boolean.FALSE.equals(multiValued) && schemaField.multiValued()) {
+ // changing from multi- to single value ... verify the data agrees ...
+ validateMultiValuedChange(configSet, schemaField, Boolean.FALSE);
+ }
+
+ // switch from single-valued to multi-valued requires a full rebuild
+ // See SOLR-12185 ... if we're switching from single to multi-valued, then it's a big operation
+ if (fieldHasMultiValuedChange(multiValued, schemaField)) {
+ needsRebuild = true;
+ log.warn("Need to rebuild the temp collection for {} after field {} updated to multi-valued {}", configSet, name, multiValued);
+ }
+
+ if (!needsRebuild) {
+ // check term vectors too
+ Boolean storeTermVector = (Boolean) fieldAttributes.getOrDefault("termVectors", Boolean.FALSE);
+ if (schemaField.storeTermVector() != storeTermVector) {
+ // cannot change termVectors w/o a full-rebuild
+ needsRebuild = true;
+ }
+ }
+
+ log.info("For {}, replacing field {} with attributes: {}", configSet, name, diff);
+ final FieldType fieldType = schemaBeforeUpdate.getFieldTypeByName(type);
+ ManagedIndexSchema updatedSchema = isDynamic ? schemaBeforeUpdate.replaceDynamicField(name, fieldType, diff)
+ : schemaBeforeUpdate.replaceField(name, fieldType, diff);
+
+ // persist the change before applying the copy-field updates
+ updatedSchema.persistManagedSchema(false);
+
+ if (!isDynamic) {
+ applyCopyFieldUpdates(mutableId, copyDest, name, updatedSchema);
+ }
+
+ return needsRebuild;
+ }
+
+ protected void validateMultiValuedChange(String configSet, SchemaField field, Boolean multiValued) throws IOException {
+ List<SolrInputDocument> docs = getStoredSampleDocs(configSet);
+ if (!docs.isEmpty()) {
+ boolean isMV = schemaSuggester.isMultiValued(field.getName(), docs);
+ if (isMV && !multiValued) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Cannot change field " + field.getName() + " to single-valued as some sample docs have multiple values!");
+ }
+ }
+ }
+
+ protected void validateTypeChange(String configSet, SchemaField field, FieldType toType) throws IOException {
+ if ("_version_".equals(field.getName())) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST,
+ "Cannot change type of the _version_ field; it must be a plong.");
+ }
+ List<SolrInputDocument> docs = getStoredSampleDocs(configSet);
+ if (!docs.isEmpty()) {
+ schemaSuggester.validateTypeChange(field, toType, docs);
+ }
+ }
+
+ void deleteStoredSampleDocs(String configSet) {
+ try {
+ cloudClient().deleteByQuery(BLOB_STORE_ID, "id:" + configSet + "_sample/*", 10);
+ } catch (IOException | SolrServerException exc) {
+ log.warn("Failed to delete sample docs from blob store for {}", configSet, exc);
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ List<SolrInputDocument> getStoredSampleDocs(final String configSet) throws IOException {
+ List<SolrInputDocument> docs = null;
+
+ final URI uri;
+ try {
+ uri = collectionApiEndpoint(BLOB_STORE_ID, "blob", configSet + "_sample")
+ .setParameter(CommonParams.WT, "filestream")
+ .build();
+ } catch (URISyntaxException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
+ }
+
+ HttpGet httpGet = new HttpGet(uri);
+ try {
+ HttpResponse entity = cloudClient().getHttpClient().execute(httpGet);
+ int statusCode = entity.getStatusLine().getStatusCode();
+ if (statusCode == HttpStatus.SC_OK) {
+ byte[] bytes = DefaultSampleDocumentsLoader.streamAsBytes(entity.getEntity().getContent());
+ if (bytes.length > 0) {
+ docs = (List<SolrInputDocument>) Utils.fromJavabin(bytes);
+ }
+ } else if (statusCode != HttpStatus.SC_NOT_FOUND) {
+ byte[] bytes = DefaultSampleDocumentsLoader.streamAsBytes(entity.getEntity().getContent());
+ throw new IOException("Failed to lookup stored docs for " + configSet + " due to: " + new String(bytes, StandardCharsets.UTF_8));
+ } // else not found is ok
+ } finally {
+ httpGet.releaseConnection();
+ }
+ return docs != null ? docs : Collections.emptyList();
+ }
+
+ void storeSampleDocs(final String configSet, List<SolrInputDocument> docs) throws IOException {
+ docs.forEach(d -> d.removeField(VERSION_FIELD)); // remove _version_ field before storing ...
+ postDataToBlobStore(cloudClient(), configSet + "_sample",
+ DefaultSampleDocumentsLoader.streamAsBytes(toJavabin(docs)));
+ }
+
+ protected void postDataToBlobStore(CloudSolrClient cloudClient, String blobName, byte[] bytes) throws IOException {
+ final URI uri;
+ try {
+ uri = collectionApiEndpoint(BLOB_STORE_ID, "blob", blobName).build();
+ } catch (URISyntaxException e) {
+ throw new SolrException(SolrException.ErrorCode.BAD_REQUEST, e);
+ }
+
+ HttpPost httpPost = new HttpPost(uri);
+ try {
+ httpPost.setHeader("Content-Type", "application/octet-stream");
+ httpPost.setEntity(new ByteArrayEntity(bytes));
+ HttpResponse resp = cloudClient.getHttpClient().execute(httpPost);
+ int statusCode = resp.getStatusLine().getStatusCode();
+ if (statusCode != HttpStatus.SC_OK) {
+ throw new SolrException(SolrException.ErrorCode.getErrorCode(statusCode),
+ EntityUtils.toString(resp.getEntity(), StandardCharsets.UTF_8));
+ }
+ } finally {
+ httpPost.releaseConnection();
+ }
+ }
+
+ private String getBaseUrl(final String collection) {
+ String baseUrl = null;
+ try {
+ Set<String> liveNodes = zkStateReader().getClusterState().getLiveNodes();
+ DocCollection docColl = zkStateReader().getCollection(collection);
+ if (docColl != null && !liveNodes.isEmpty()) {
+ Optional<Replica> maybeActive = docColl.getReplicas().stream().filter(r -> r.isActive(liveNodes)).findAny();
+ if (maybeActive.isPresent()) {
+ baseUrl = maybeActive.get().getBaseUrl();
+ }
+ }
+ } catch (Exception exc) {
+ log.warn("Failed to lookup base URL for collection {}", collection, exc);
+ }
+
+ if (baseUrl == null) {
+ baseUrl = UrlScheme.INSTANCE.getBaseUrlForNodeName(cc.getZkController().getNodeName());
+ }
+
+ return baseUrl;
+ }
+
+ private URIBuilder collectionApiEndpoint(final String collection, final String... morePathSegments) throws URISyntaxException {
+ URI baseUrl = new URI(getBaseUrl(collection));
+ // build up a list of path segments including any path in the base URL, collection, and additional segments provided by caller
+ List<String> path = new ArrayList<>(URLEncodedUtils.parsePathSegments(baseUrl.getPath()));
+ path.add(collection);
+ if (morePathSegments != null && morePathSegments.length > 0) {
+ path.addAll(Arrays.asList(morePathSegments));
+ }
+ return new URIBuilder(baseUrl).setPathSegments(path);
+ }
+
+ protected String getManagedSchemaZkPath(final String configSet) {
+ return getConfigSetZkPath(configSet, DEFAULT_MANAGED_SCHEMA_RESOURCE_NAME);
+ }
+
+ ManagedIndexSchema toggleNestedDocsFields(ManagedIndexSchema schema, boolean enabled) {
+ return enabled ? enableNestedDocsFields(schema, true) : deleteNestedDocsFieldsIfNeeded(schema, true);
+ }
+
+ ManagedIndexSchema enableNestedDocsFields(ManagedIndexSchema schema, boolean persist) {
+ boolean madeChanges = false;
+
+ if (!schema.hasExplicitField(ROOT_FIELD_NAME)) {
+ Map<String, Object> fieldAttrs = Map.of("docValues", false, "indexed", true, "stored", false);
+ schema = (ManagedIndexSchema) schema.addField(schema.newField(ROOT_FIELD_NAME, "string", fieldAttrs), false);
+ madeChanges = true;
+ }
+
+ if (!schema.hasExplicitField(NEST_PATH_FIELD_NAME)) {
+ schema = (ManagedIndexSchema) schema.addField(schema.newField(NEST_PATH_FIELD_NAME, NEST_PATH_FIELD_NAME, Collections.emptyMap()), false);
+ madeChanges = true;
+ }
+
+ if (madeChanges && persist) {
+ schema.persistManagedSchema(false);
+ }
+
+ return schema;
+ }
+
+ ManagedIndexSchema deleteNestedDocsFieldsIfNeeded(ManagedIndexSchema schema, boolean persist) {
+ List<String> toDelete = new LinkedList<>();
+ if (schema.hasExplicitField(ROOT_FIELD_NAME)) {
+ toDelete.add(ROOT_FIELD_NAME);
+ }
+ if (schema.hasExplicitField(NEST_PATH_FIELD_NAME)) {
+ toDelete.add(NEST_PATH_FIELD_NAME);
+ }
+ if (!toDelete.isEmpty()) {
+ schema = schema.deleteFields(toDelete);
+ if (persist) {
+ schema.persistManagedSchema(false);
+ }
+ }
+ return schema;
+ }
+
+ SolrConfig loadSolrConfig(String configSet) {
+ SolrResourceLoader resourceLoader = cc.getResourceLoader();
+ ZkSolrResourceLoader zkLoader =
+ new ZkSolrResourceLoader(resourceLoader.getInstancePath(), configSet, resourceLoader.getClassLoader(), cc.getZkController());
+ return SolrConfig.readFromResourceLoader(zkLoader, SOLR_CONFIG_XML, true, null);
+ }
+
+ ManagedIndexSchema loadLatestSchema(String configSet) {
+ return loadLatestSchema(loadSolrConfig(configSet));
+ }
+
+ ManagedIndexSchema loadLatestSchema(SolrConfig solrConfig) {
+ ManagedIndexSchemaFactory factory = new ManagedIndexSchemaFactory();
+ factory.init(new NamedList<>());
+ return factory.create(DEFAULT_MANAGED_SCHEMA_RESOURCE_NAME, solrConfig, null);
+ }
+
+ int getCurrentSchemaVersion(final String configSet) throws IOException {
+ int currentVersion = -1;
+ final String path = getManagedSchemaZkPath(configSet);
+ try {
+ Stat stat = cc.getZkController().getZkClient().exists(path, null, true);
+ if (stat != null) {
+ currentVersion = stat.getVersion();
+ }
+ } catch (KeeperException.NoNodeException notExists) {
+ // safe to ignore
+ } catch (KeeperException | InterruptedException e) {
+ throw new IOException("Error getting version for schema: " + configSet, SolrZkClient.checkInterrupted(e));
+ }
+ return currentVersion;
+ }
+
+ void createCollection(final String collection, final String configSet) throws IOException, SolrServerException {
+ createCollection(collection, configSet, 1, 1);
+ }
+
+ void createCollection(final String collection, final String configSet, int numShards, int numReplicas) throws IOException, SolrServerException {
+ RTimer timer = new RTimer();
+ SolrResponse rsp =
+ CollectionAdminRequest.createCollection(collection, configSet, numShards, numReplicas).process(cloudClient());
+ try {
+ CollectionsHandler.waitForActiveCollection(collection, cc, rsp);
+ } catch (KeeperException | InterruptedException e) {
+ throw new IOException("Failed waiting for new collection " + collection + " to reach the active state",
+ SolrZkClient.checkInterrupted(e));
+ }
+ double tookMs = timer.getTime();
+ log.debug("Took {} ms to create new collection {} with configSet {}", tookMs, collection, configSet);
+ }
+
+ protected CloudSolrClient cloudClient() {
+ return cc.getSolrClientCache().getCloudSolrClient(cc.getZkController().getZkServerAddress());
+ }
+
+ protected ZkStateReader zkStateReader() {
+ return cc.getZkController().getZkStateReader();
+ }
+
+ boolean applyCopyFieldUpdates(String mutableId, String copyDest, String fieldName, ManagedIndexSchema schema) throws IOException, SolrServerException {
+ boolean updated = false;
+
+ if (copyDest == null || copyDest.trim().isEmpty()) {
+ // delete all the copy field directives for this field
+ List<CopyField> copyFieldsList = schema.getCopyFieldsList(fieldName);
+ if (!copyFieldsList.isEmpty()) {
+ List<String> dests = copyFieldsList.stream().map(cf -> cf.getDestination().getName()).collect(Collectors.toList());
+ SchemaRequest.DeleteCopyField delAction = new SchemaRequest.DeleteCopyField(fieldName, dests);
+ SchemaResponse.UpdateResponse schemaResponse = delAction.process(cloudClient(), mutableId);
+ if (schemaResponse.getStatus() != 0) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, schemaResponse.getException());
+ }
+ updated = true;
+ }
+ } else {
+ SchemaField field = schema.getField(fieldName);
+ Set<String> desired = new HashSet<>();
+ for (String dest : copyDest.trim().split(",")) {
+ String toAdd = dest.trim();
+ if (toAdd.equals(fieldName)) {
+ continue; // cannot copy to self
+ }
+
+ // make sure the field exists and is multi-valued if this field is
+ SchemaField toAddField = schema.getFieldOrNull(toAdd);
+ if (toAddField != null) {
+ if (!field.multiValued() || toAddField.multiValued()) {
+ desired.add(toAdd);
+ } else {
+ log.warn("Skipping copy-field dest {} for {} because it is not multi-valued!", toAdd, fieldName);
+ }
+ } else {
+ log.warn("Skipping copy-field dest {} for {} because it doesn't exist!", toAdd, fieldName);
+ }
+ }
+ Set<String> existing = schema.getCopyFieldsList(fieldName).stream().map(cf -> cf.getDestination().getName()).collect(Collectors.toSet());
+ Set<String> add = Sets.difference(desired, existing);
+ if (!add.isEmpty()) {
+ SchemaRequest.AddCopyField addAction = new SchemaRequest.AddCopyField(fieldName, new ArrayList<>(add));
+ SchemaResponse.UpdateResponse schemaResponse = addAction.process(cloudClient(), mutableId);
+ if (schemaResponse.getStatus() != 0) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, schemaResponse.getException());
+ }
+ updated = true;
+ } // no additions ...
+
+ Set<String> del = Sets.difference(existing, desired);
+ if (!del.isEmpty()) {
+ SchemaRequest.DeleteCopyField delAction = new SchemaRequest.DeleteCopyField(fieldName, new ArrayList<>(del));
+ SchemaResponse.UpdateResponse schemaResponse = delAction.process(cloudClient(), mutableId);
+ if (schemaResponse.getStatus() != 0) {
+ throw new SolrException(SolrException.ErrorCode.SERVER_ERROR, schemaResponse.getException());
+ }
+ updated = true;
+ } // no deletions ...
+ }
+
+ return updated;
+ }
+
+ protected boolean fieldHasMultiValuedChange(Object multiValued, SchemaField schemaField) {
+ return (multiValued == null ||
+ (Boolean.TRUE.equals(multiValued) && !schemaField.multiValued()) ||
+ (Boolean.FALSE.equals(multiValued) && schemaField.multiValued()));
+ }
+
+ protected boolean typeHasMultiValuedChange(Object multiValued, FieldType fieldType) {
+ return (multiValued == null ||
+ (Boolean.TRUE.equals(multiValued) && !fieldType.isMultiValued()) ||
+ (Boolean.FALSE.equals(multiValued) && fieldType.isMultiValued()));
+ }
+
+ ManagedIndexSchema syncLanguageSpecificObjectsAndFiles(String configSet, ManagedIndexSchema schema, List<String> langs, boolean dynamicEnabled, String copyFrom) throws IOException {
+ if (!langs.isEmpty()) {
+ // there's a subset of languages applied, so remove all the other langs
+ schema = removeLanguageSpecificObjectsAndFiles(configSet, schema, langs);
+ }
+
+ // now restore any missing types / files for the languages we need, optionally adding back dynamic fields too
+ schema = restoreLanguageSpecificObjectsAndFiles(configSet, schema, langs, dynamicEnabled, copyFrom);
+
+ schema.persistManagedSchema(false);
+ return schema;
+ }
+
+ protected ManagedIndexSchema removeLanguageSpecificObjectsAndFiles(String configSet, ManagedIndexSchema schema, List<String> langs) throws IOException {
+ final Set<String> languages = new HashSet<>(includeLangIds);
+ languages.addAll(langs);
+
+ final Set<String> usedTypes =
+ schema.getFields().values().stream().map(f -> f.getType().getTypeName()).collect(Collectors.toSet());
+
+ final Set<String> usedLangs =
+ schema.getFields().values().stream().filter(f -> isTextType(f.getType()))
+ .map(f -> f.getType().getTypeName().substring(TEXT_PREFIX_LEN)).collect(Collectors.toSet());
+
+ // don't remove types / files for langs that are explicitly being used by a field
+ languages.addAll(usedLangs);
+
+ Map<String, FieldType> types = schema.getFieldTypes();
+ final Set<String> toRemove = types.values().stream()
+ .filter(this::isTextType)
+ .filter(t -> !languages.contains(t.getTypeName().substring(TEXT_PREFIX_LEN)))
+ .map(FieldType::getTypeName)
+ .filter(t -> !usedTypes.contains(t)) // not explicitly used by a field
+ .collect(Collectors.toSet());
+
+ // find dynamic fields that refer to the types we're removing ...
+ List<String> toRemoveDF = Arrays.stream(schema.getDynamicFields())
+ .filter(df -> toRemove.contains(df.getPrototype().getType().getTypeName()))
+ .map(df -> df.getPrototype().getName())
+ .collect(Collectors.toList());
+
+ schema = schema.deleteDynamicFields(toRemoveDF);
+ schema = schema.deleteFieldTypes(toRemove);
+
+ SolrZkClient zkClient = cc.getZkController().getZkClient();
+ final String configPathInZk = ZkConfigSetService.CONFIGS_ZKNODE + ZNODE_PATH_DELIM + configSet;
+ final Set<String> toRemoveFiles = new HashSet<>();
+ final Set<String> langExt = languages.stream().map(l -> "_" + l).collect(Collectors.toSet());
+ try {
+ ZkMaintenanceUtils.traverseZkTree(zkClient, configPathInZk, ZkMaintenanceUtils.VISIT_ORDER.VISIT_POST, path -> {
+ if (!isMatchingLangOrNonLangFile(path, langExt)) toRemoveFiles.add(path);
+ });
+ } catch (KeeperException.NoNodeException nne) {
+ // no-op
+ } catch (KeeperException | InterruptedException e) {
+ throw new IOException("Failed to traverse znode path: " + configPathInZk, SolrZkClient.checkInterrupted(e));
+ }
+
+ for (String path : toRemoveFiles) {
+ try {
+ zkClient.delete(path, -1, false);
+ } catch (KeeperException.NoNodeException nne) {
+ // no-op
+ } catch (KeeperException | InterruptedException e) {
+ throw new IOException("Failed to delete znode: " + path, SolrZkClient.checkInterrupted(e));
+ }
+ }
+
+ return schema;
+ }
+
+ protected ManagedIndexSchema restoreLanguageSpecificObjectsAndFiles(String configSet,
+ ManagedIndexSchema schema,
+ List<String> langs,
+ boolean dynamicEnabled,
+ String copyFrom) throws IOException {
+ // pull the dynamic fields from the copyFrom schema
+ ManagedIndexSchema copyFromSchema = loadLatestSchema(copyFrom);
+
+ final Set<String> langSet = new HashSet<>(includeLangIds);
+ langSet.addAll(langs);
+
+ boolean restoreAllLangs = langs.isEmpty();
+
+ final Set<String> langFilesToRestore = new HashSet<>();
+
+ // Restore missing files
+ SolrZkClient zkClient = zkStateReader().getZkClient();
+ String configPathInZk = ZkConfigSetService.CONFIGS_ZKNODE + ZNODE_PATH_DELIM + copyFrom;
+ final Set<String> langExt = langSet.stream().map(l -> "_" + l).collect(Collectors.toSet());
+ try {
+ ZkMaintenanceUtils.traverseZkTree(zkClient, configPathInZk, ZkMaintenanceUtils.VISIT_ORDER.VISIT_POST, path -> {
+ if (path.endsWith(".txt")) {
+ if (restoreAllLangs) {
+ langFilesToRestore.add(path);
+ return;
+ }
+
+ final String pathWoExt = path.substring(0, path.length() - 4);
+ for (String lang : langExt) {
+ if (pathWoExt.endsWith(lang)) {
+ langFilesToRestore.add(path);
+ break;
+ }
+ }
+ }
+ });
+ } catch (KeeperException.NoNodeException nne) {
+ // no-op
+ } catch (KeeperException | InterruptedException e) {
+ throw new IOException("Failed to traverse znode path: " + configPathInZk, SolrZkClient.checkInterrupted(e));
+ }
+
+ if (!langFilesToRestore.isEmpty()) {
+ final String replacePathDir = "/" + configSet;
+ final String origPathDir = "/" + copyFrom;
+ for (String path : langFilesToRestore) {
+ String copyToPath = path.replace(origPathDir, replacePathDir);
+ try {
+ if (!zkClient.exists(copyToPath, true)) {
+ zkClient.makePath(copyToPath, false, true);
+ zkClient.setData(copyToPath, zkClient.getData(path, null, null, true), true);
+ }
+ } catch (KeeperException | InterruptedException e) {
+ throw new IOException("Failed to restore file at znode path: " + copyToPath, SolrZkClient.checkInterrupted(e));
+ }
+ }
+ }
+
+ // Restore field types
+ final Map<String, FieldType> existingTypes = schema.getFieldTypes();
+ List<FieldType> addTypes = copyFromSchema.getFieldTypes().values().stream()
+ .filter(t -> isLangTextType(t, restoreAllLangs ? null : langSet) && !existingTypes.containsKey(t.getTypeName()))
+ .collect(Collectors.toList());
+ if (!addTypes.isEmpty()) {
+ schema = schema.addFieldTypes(addTypes, false);
+ }
+
+ if (dynamicEnabled) {
+ // restore language specific dynamic fields
+ final Set<String> existingDynFields = Arrays.stream(schema.getDynamicFieldPrototypes())
+ .map(SchemaField::getName)
+ .collect(Collectors.toSet());
+
+ final Set<String> langFieldTypeNames = schema.getFieldTypes().values().stream()
+ .filter(t -> isLangTextType(t, restoreAllLangs ? null : langSet))
+ .map(FieldType::getTypeName)
+ .collect(Collectors.toSet());
+
+ List<SchemaField> addDynFields = Arrays.stream(copyFromSchema.getDynamicFields())
+ .filter(df -> langFieldTypeNames.contains(df.getPrototype().getType().getTypeName()))
+ .filter(df -> !existingDynFields.contains(df.getPrototype().getName()))
+ .map(IndexSchema.DynamicField::getPrototype)
+ .collect(Collectors.toList());
+ if (!addDynFields.isEmpty()) {
+ schema = schema.addDynamicFields(addDynFields, null, false);
+ }
+ } else {
+ schema = removeDynamicFields(schema);
+ }
+
+ return schema;
+ }
+
+ private boolean isMatchingLangOrNonLangFile(final String path, final Set<String> langs) {
+ if (!path.endsWith(".txt"))
+ return true; // not a .txt file, always include
+
+ int slashAt = path.lastIndexOf('/');
+ String fileName = slashAt != -1 ? path.substring(slashAt + 1) : "";
+ if (!fileName.contains("_"))
+ return true; // looking for file names like stopwords_en.txt, not a match, so skip it
+
+ // remove the .txt extension
+ final String pathWoExt = fileName.substring(0, fileName.length() - 4);
+ for (String lang : langs) {
+ if (pathWoExt.endsWith(lang)) {
+ return true;
+ }
+ }
+
+ // if we fall thru to here, then the file should be excluded
+ return false;
+ }
+
+ private boolean isTextType(final FieldType t) {
+ return t.getTypeName().startsWith("text_") && TextField.class.equals(t.getClass());
+ }
+
+ private boolean isLangTextType(final FieldType t, final Set<String> langSet) {
+ return isTextType(t) && (langSet == null || langSet.contains(t.getTypeName().substring("text_".length())));
+ }
+
+ protected ManagedIndexSchema removeDynamicFields(ManagedIndexSchema schema) {
+ List<String> dynamicFieldNames =
+ Arrays.stream(schema.getDynamicFields()).map(f -> f.getPrototype().getName()).collect(Collectors.toList());
+ if (!dynamicFieldNames.isEmpty()) {
+ schema = schema.deleteDynamicFields(dynamicFieldNames);
+ }
+ return schema;
+ }
+
+ protected ManagedIndexSchema restoreDynamicFields(ManagedIndexSchema schema, List<String> langs, String copyFrom) {
+ // pull the dynamic fields from the copyFrom schema
+ ManagedIndexSchema copyFromSchema = loadLatestSchema(copyFrom);
+ IndexSchema.DynamicField[] dynamicFields = copyFromSchema.getDynamicFields();
+ if (dynamicFields.length == 0 && !DEFAULT_CONFIGSET_NAME.equals(copyFrom)) {
+ copyFromSchema = loadLatestSchema(DEFAULT_CONFIGSET_NAME);
+ dynamicFields = copyFromSchema.getDynamicFields();
+ }
+
+ if (dynamicFields.length == 0) {
+ return schema;
+ }
+
+ final Set<String> existingDFNames =
+ Arrays.stream(schema.getDynamicFields()).map(df -> df.getPrototype().getName()).collect(Collectors.toSet());
+ List<SchemaField> toAdd = Arrays.stream(dynamicFields)
+ .filter(df -> !existingDFNames.contains(df.getPrototype().getName()))
+ .map(IndexSchema.DynamicField::getPrototype)
+ .collect(Collectors.toList());
+
+ // only restore language specific dynamic fields that match our langSet
+ if (!langs.isEmpty()) {
+ final Set<String> langSet = new HashSet<>(includeLangIds);
+ langSet.addAll(langs);
+ toAdd = toAdd.stream()
+ .filter(df -> !df.getName().startsWith("*_txt_") || langSet.contains(df.getName().substring("*_txt_".length())))
+ .collect(Collectors.toList());
+ }
+
+ if (!toAdd.isEmpty()) {
+ // grab any field types that need to be re-added
+ final Map<String, FieldType> fieldTypes = schema.getFieldTypes();
+ List<FieldType> addTypes = toAdd.stream()
+ .map(SchemaField::getType)
+ .filter(t -> !fieldTypes.containsKey(t.getTypeName()))
+ .collect(Collectors.toList());
+ if (!addTypes.isEmpty()) {
+ schema = schema.addFieldTypes(addTypes, false);
+ }
+
+ schema = schema.addDynamicFields(toAdd, null, true);
+ }
+
+ return schema;
+ }
+
+ void checkSchemaVersion(String configSet, final int versionInRequest, int currentVersion) throws IOException {
+ if (versionInRequest < 0) {
+ return; // don't enforce the version check
+ }
+
+ if (currentVersion == -1) {
+ currentVersion = getCurrentSchemaVersion(configSet);
+ }
+
+ if (currentVersion != versionInRequest) {
+ if (configSet.startsWith(DESIGNER_PREFIX)) {
+ configSet = configSet.substring(DESIGNER_PREFIX.length());
+ }
+ throw new SolrException(SolrException.ErrorCode.CONFLICT,
+ "Your schema version " + versionInRequest + " for " + configSet + " is out-of-date; current version is: " + currentVersion +
+ ". Perhaps another user also updated the schema while you were editing it? You'll need to retry your update after the schema is refreshed.");
+ }
+ }
+
+ List<String> listConfigsInZk() throws IOException {
+ return cc.getConfigSetService().listConfigs();
+ }
+
+ byte[] downloadAndZipConfigSet(String configId) throws IOException {
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ Path tmpDirectory = Files.createTempDirectory("schema-designer-" + FilenameUtils.getName(configId));
+ File tmpDir = tmpDirectory.toFile();
+ try {
+ cc.getConfigSetService().downloadConfig(configId, tmpDirectory);
+ try (ZipOutputStream zipOut = new ZipOutputStream(baos)) {
+ zipIt(tmpDir, "", zipOut);
+ }
+ } finally {
+ FileUtils.deleteDirectory(tmpDir);
+ }
+ return baos.toByteArray();
+ }
+
+ protected void zipIt(File f, String fileName, ZipOutputStream zipOut) throws IOException {
+ if (f.isHidden()) {
+ return;
+ }
+
+ if (f.isDirectory()) {
+ String dirPrefix = "";
+ if (fileName.endsWith("/")) {
+ zipOut.putNextEntry(new ZipEntry(fileName));
+ zipOut.closeEntry();
+ dirPrefix = fileName;
+ } else if (!fileName.isEmpty()) {
+ dirPrefix = fileName + "/";
+ zipOut.putNextEntry(new ZipEntry(dirPrefix));
+ zipOut.closeEntry();
+ }
+ File[] files = f.listFiles();
+ if (files != null) {
+ for (File child : files) {
+ zipIt(child, dirPrefix + child.getName(), zipOut);
+ }
+ }
+ return;
+ }
+
+ byte[] bytes = new byte[1024];
+ int r;
+ try (FileInputStream fis = new FileInputStream(f)) {
+ ZipEntry zipEntry = new ZipEntry(fileName);
+ zipOut.putNextEntry(zipEntry);
+ while ((r = fis.read(bytes)) >= 0) {
+ zipOut.write(bytes, 0, r);
+ }
+ }
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConstants.java b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConstants.java
new file mode 100644
index 00000000000..b6afdc66152
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerConstants.java
@@ -0,0 +1,50 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.designer;
+
+public interface SchemaDesignerConstants {
+ String CONFIG_SET_PARAM = "configSet";
+ String COPY_FROM_PARAM = "copyFrom";
+ String SCHEMA_VERSION_PARAM = "schemaVersion";
+ String RELOAD_COLLECTIONS_PARAM = "reloadCollections";
+ String INDEX_TO_COLLECTION_PARAM = "indexToCollection";
+ String NEW_COLLECTION_PARAM = "newCollection";
+ String CLEANUP_TEMP_PARAM = "cleanupTemp";
+ String ENABLE_DYNAMIC_FIELDS_PARAM = "enableDynamicFields";
+ String ENABLE_FIELD_GUESSING_PARAM = "enableFieldGuessing";
+ String ENABLE_NESTED_DOCS_PARAM = "enableNestedDocs";
+ String TEMP_COLLECTION_PARAM = "tempCollection";
+ String PUBLISHED_VERSION = "publishedVersion";
+ String DISABLE_DESIGNER_PARAM = "disableDesigner";
+ String DISABLED = "disabled";
+ String DOC_ID_PARAM = "docId";
+ String FIELD_PARAM = "field";
+ String UNIQUE_KEY_FIELD_PARAM = "uniqueKeyField";
+ String AUTO_CREATE_FIELDS = "update.autoCreateFields";
+ String SOLR_CONFIG_XML = "solrconfig.xml";
+ String DESIGNER_KEY = "_designer.";
+ String LANGUAGES_PARAM = "languages";
+ String CONFIGOVERLAY_JSON = "configoverlay.json";
+ String BLOB_STORE_ID = ".system";
+ String UPDATE_ERROR = "updateError";
+ String ANALYSIS_ERROR = "analysisError";
+ String ERROR_DETAILS = "errorDetails";
+
+ String DESIGNER_PREFIX = "._designer_";
+ int MAX_SAMPLE_DOCS = 1000;
+}
diff --git a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerSettings.java b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerSettings.java
new file mode 100644
index 00000000000..756a26d6ddd
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerSettings.java
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.designer;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Objects;
+import java.util.Optional;
+
+import org.apache.solr.schema.ManagedIndexSchema;
+
+class SchemaDesignerSettings implements SchemaDesignerConstants {
+
+ private String copyFrom;
+ private boolean isDisabled;
+ private List<String> languages;
+ private boolean dynamicFieldsEnabled;
+ private boolean nestedDocsEnabled;
+ private boolean fieldGuessingEnabled;
+ private Integer publishedVersion;
+ private ManagedIndexSchema schema;
+
+ @SuppressWarnings("unchecked")
+ SchemaDesignerSettings(Map<String, Object> stored) {
+ this.isDisabled = (Boolean) stored.getOrDefault(DESIGNER_KEY + DISABLED, false);
+ this.publishedVersion = null;
+ this.copyFrom = (String) stored.get(DESIGNER_KEY + COPY_FROM_PARAM);
+ this.languages = (List<String>) stored.getOrDefault(DESIGNER_KEY + LANGUAGES_PARAM, Collections.emptyList());
+ this.dynamicFieldsEnabled = (Boolean) stored.getOrDefault(DESIGNER_KEY + ENABLE_DYNAMIC_FIELDS_PARAM, true);
+ this.nestedDocsEnabled = (Boolean) stored.getOrDefault(DESIGNER_KEY + ENABLE_NESTED_DOCS_PARAM, false);
+ this.fieldGuessingEnabled = (Boolean) stored.getOrDefault(AUTO_CREATE_FIELDS, true);
+ }
+
+ Map<String, Object> toMap() {
+ Map<String, Object> map = new HashMap<>();
+ map.put(DESIGNER_KEY + DISABLED, isDisabled);
+ map.put(DESIGNER_KEY + LANGUAGES_PARAM, languages);
+ map.put(DESIGNER_KEY + ENABLE_DYNAMIC_FIELDS_PARAM, dynamicFieldsEnabled);
+ map.put(DESIGNER_KEY + ENABLE_NESTED_DOCS_PARAM, nestedDocsEnabled);
+ map.put(AUTO_CREATE_FIELDS, fieldGuessingEnabled);
+ if (copyFrom != null) {
+ map.put(DESIGNER_KEY + COPY_FROM_PARAM, copyFrom);
+ }
+ if (publishedVersion != null) {
+ map.put(DESIGNER_KEY + PUBLISHED_VERSION, publishedVersion);
+ }
+ return map;
+ }
+
+ public ManagedIndexSchema getSchema() {
+ return schema;
+ }
+
+ public void setSchema(ManagedIndexSchema schema) {
+ this.schema = schema;
+ }
+
+ public Optional<Integer> getPublishedVersion() {
+ return Optional.ofNullable(publishedVersion);
+ }
+
+ public void setPublishedVersion(int publishedVersion) {
+ this.publishedVersion = publishedVersion;
+ }
+
+ public String getCopyFrom() {
+ return copyFrom;
+ }
+
+ public void setCopyFrom(String copyFrom) {
+ this.copyFrom = copyFrom;
+ }
+
+ public boolean isDisabled() {
+ return isDisabled;
+ }
+
+ public void setDisabled(boolean isDisabled) {
+ this.isDisabled = isDisabled;
+ }
+
+ public List<String> getLanguages() {
+ return languages;
+ }
+
+ public void setLanguages(List<String> langs) {
+ this.languages = langs != null ? langs : Collections.emptyList();
+ }
+
+ public boolean dynamicFieldsEnabled() {
+ return dynamicFieldsEnabled;
+ }
+
+ public void setDynamicFieldsEnabled(boolean enabled) {
+ this.dynamicFieldsEnabled = enabled;
+ }
+
+ public boolean nestedDocsEnabled() {
+ return nestedDocsEnabled;
+ }
+
+ public void setNestedDocsEnabled(boolean enabled) {
+ this.nestedDocsEnabled = enabled;
+ }
+
+ public boolean fieldGuessingEnabled() {
+ return fieldGuessingEnabled;
+ }
+
+ public void setFieldGuessingEnabled(boolean enabled) {
+ this.fieldGuessingEnabled = enabled;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ SchemaDesignerSettings that = (SchemaDesignerSettings) o;
+ return isDisabled == that.isDisabled &&
+ dynamicFieldsEnabled == that.dynamicFieldsEnabled &&
+ nestedDocsEnabled == that.nestedDocsEnabled &&
+ fieldGuessingEnabled == that.fieldGuessingEnabled &&
+ Objects.equals(copyFrom, that.copyFrom) &&
+ languages.equals(that.languages) &&
+ Objects.equals(publishedVersion, that.publishedVersion);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(copyFrom, isDisabled, languages, dynamicFieldsEnabled, nestedDocsEnabled, fieldGuessingEnabled, publishedVersion);
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerSettingsDAO.java b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerSettingsDAO.java
new file mode 100644
index 00000000000..975c1b68b2c
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/designer/SchemaDesignerSettingsDAO.java
@@ -0,0 +1,174 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.designer;
+
+import java.io.IOException;
+import java.lang.invoke.MethodHandles;
+import java.nio.charset.StandardCharsets;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.solr.cloud.ZkController;
+import org.apache.solr.cloud.ZkSolrResourceLoader;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.core.ConfigOverlay;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.PluginInfo;
+import org.apache.solr.core.SolrConfig;
+import org.apache.solr.core.SolrResourceLoader;
+import org.apache.solr.update.processor.UpdateRequestProcessorChain;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+import org.noggit.JSONParser;
+import org.noggit.ObjectBuilder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import static org.apache.solr.handler.designer.SchemaDesignerAPI.getConfigSetZkPath;
+
+class SchemaDesignerSettingsDAO implements SchemaDesignerConstants {
+
+ private static final Logger log = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass());
+
+ private final CoreContainer cc;
+
+ SchemaDesignerSettingsDAO(CoreContainer cc) {
+ this.cc = cc;
+ }
+
+ SchemaDesignerSettings getSettings(String configSet) {
+ SolrConfig solrConfig =
+ SolrConfig.readFromResourceLoader(zkLoaderForConfigSet(configSet), SOLR_CONFIG_XML, true, null);
+ return getSettings(solrConfig);
+ }
+
+ SchemaDesignerSettings getSettings(SolrConfig solrConfig) {
+ Map<String, Object> map = new HashMap<>();
+
+ boolean isFieldGuessingEnabled = true;
+ if (solrConfig != null) {
+ ConfigOverlay overlay = solrConfig.getOverlay();
+ Map<String, Object> userProps = overlay != null ? overlay.getUserProps() : null;
+ if (userProps != null) {
+ map.putAll(userProps);
+ }
+ isFieldGuessingEnabled = isFieldGuessingEnabled(solrConfig);
+ }
+
+ map.putIfAbsent(AUTO_CREATE_FIELDS, isFieldGuessingEnabled);
+ map.putIfAbsent(DESIGNER_KEY + LANGUAGES_PARAM, Collections.emptyList());
+ map.putIfAbsent(DESIGNER_KEY + ENABLE_DYNAMIC_FIELDS_PARAM, true);
+ map.putIfAbsent(DESIGNER_KEY + ENABLE_NESTED_DOCS_PARAM, false);
+
+ return new SchemaDesignerSettings(map);
+ }
+
+ boolean persistIfChanged(String configSet, SchemaDesignerSettings settings) throws IOException {
+ boolean changed = false;
+
+ ConfigOverlay overlay = getConfigOverlay(configSet);
+ Map<String, Object> storedUserProps = overlay != null ? overlay.getUserProps() : Collections.emptyMap();
+ for (Map.Entry<String, Object> e : settings.toMap().entrySet()) {
+ String key = e.getKey();
+ Object propValue = e.getValue();
+ if (propValue != null && !propValue.equals(storedUserProps.get(key))) {
+ if (overlay == null) overlay = new ConfigOverlay(null, -1);
+ overlay = overlay.setUserProperty(key, propValue);
+ changed = true;
+ }
+ }
+
+ if (changed) {
+ ZkController.persistConfigResourceToZooKeeper(zkLoaderForConfigSet(configSet), overlay.getZnodeVersion(),
+ ConfigOverlay.RESOURCE_NAME, overlay.toByteArray(), true);
+ }
+
+ return changed;
+ }
+
+ boolean isDesignerDisabled(String configSet) {
+ // filter out any configs that don't want to be edited by the Schema Designer
+ // this allows users to lock down specific configs from being edited by the designer
+ ConfigOverlay overlay = null;
+ try {
+ overlay = getConfigOverlay(configSet);
+ } catch (IOException exc) {
+ log.error("Failed to get config overlay for {}", configSet, exc);
+ }
+ Map<String, Object> userProps = overlay != null ? overlay.getUserProps() : Collections.emptyMap();
+ return (boolean) userProps.getOrDefault(DESIGNER_KEY + DISABLED, false);
+ }
+
+ @SuppressWarnings("unchecked")
+ private ConfigOverlay getConfigOverlay(String config) throws IOException {
+ ConfigOverlay overlay = null;
+ String path = getConfigSetZkPath(config, CONFIGOVERLAY_JSON);
+ byte[] data = null;
+ Stat stat = new Stat();
+ try {
+ data = cc.getZkController().getZkStateReader().getZkClient().getData(path, null, stat, true);
+ } catch (KeeperException.NoNodeException nne) {
+ // ignore
+ } catch (KeeperException | InterruptedException e) {
+ throw new IOException("Error reading path: " + path, SolrZkClient.checkInterrupted(e));
+ }
+ if (data != null && data.length > 0) {
+ Map<String, Object> json =
+ (Map<String, Object>) ObjectBuilder.getVal(new JSONParser(new String(data, StandardCharsets.UTF_8)));
+ overlay = new ConfigOverlay(json, stat.getVersion());
+ }
+ return overlay;
+ }
+
+ private boolean isFieldGuessingEnabled(final SolrConfig solrConfig) {
+ if (!hasFieldGuessingURPChain(solrConfig)) {
+ return false; // no URP chain, so can't be enabled
+ }
+
+ boolean isEnabled = true;
+ ConfigOverlay overlay = solrConfig.getOverlay();
+ if (overlay != null) {
+ Map<String, Object> userProps = overlay.getUserProps();
+ if (userProps != null) {
+ isEnabled = (Boolean) userProps.getOrDefault(AUTO_CREATE_FIELDS, true);
+ }
+ }
+ return isEnabled;
+ }
+
+ private boolean hasFieldGuessingURPChain(final SolrConfig solrConfig) {
+ boolean hasPlugin = false;
+ List<PluginInfo> plugins = solrConfig.getPluginInfos(UpdateRequestProcessorChain.class.getName());
+ if (plugins != null) {
+ for (PluginInfo next : plugins) {
+ if ("add-unknown-fields-to-the-schema".equals(next.name)) {
+ hasPlugin = true;
+ break;
+ }
+ }
+ }
+ return hasPlugin;
+ }
+
+ private ZkSolrResourceLoader zkLoaderForConfigSet(final String configSet) {
+ SolrResourceLoader loader = cc.getResourceLoader();
+ return new ZkSolrResourceLoader(loader.getInstancePath(), configSet, loader.getClassLoader(), cc.getZkController());
+ }
+}
diff --git a/solr/core/src/java/org/apache/solr/handler/designer/SchemaSuggester.java b/solr/core/src/java/org/apache/solr/handler/designer/SchemaSuggester.java
new file mode 100644
index 00000000000..ca70486a228
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/designer/SchemaSuggester.java
@@ -0,0 +1,42 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.designer;
+
+import java.io.IOException;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.IndexSchema;
+import org.apache.solr.schema.ManagedIndexSchema;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.util.plugin.NamedListInitializedPlugin;
+
+public interface SchemaSuggester extends NamedListInitializedPlugin {
+ Optional<SchemaField> suggestField(String fieldName, List<Object> sampleValues, IndexSchema schema, List<String> langs);
+
+ ManagedIndexSchema adaptExistingFieldToData(SchemaField schemaField, List<Object> sampleValues, ManagedIndexSchema schema);
+
+ Map<String, List<Object>> transposeDocs(List<SolrInputDocument> docs);
+
+ void validateTypeChange(SchemaField field, FieldType toType, List<SolrInputDocument> docs) throws IOException;
+
+ boolean isMultiValued(String name, List<SolrInputDocument> docs);
+}
diff --git a/solr/core/src/java/org/apache/solr/handler/designer/package-info.java b/solr/core/src/java/org/apache/solr/handler/designer/package-info.java
new file mode 100644
index 00000000000..30be4e281bf
--- /dev/null
+++ b/solr/core/src/java/org/apache/solr/handler/designer/package-info.java
@@ -0,0 +1,26 @@
+/*
+ * This software was produced for the U. S. Government
+ * under Contract No. W15P7T-11-C-F600, and is
+ * subject to the Rights in Noncommercial Computer Software
+ * and Noncommercial Computer Software Documentation
+ * Clause 252.227-7014 (JUN 1995)
+ *
+ * Copyright 2013 The MITRE Corporation. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * The {@link org.apache.solr.handler.designer.SchemaDesignerAPI} and supporting classes.
+ */
+package org.apache.solr.handler.designer;
diff --git a/solr/core/src/java/org/apache/solr/handler/loader/CSVLoader.java b/solr/core/src/java/org/apache/solr/handler/loader/CSVLoader.java
index 11866a44643..6e2fb8bd7de 100644
--- a/solr/core/src/java/org/apache/solr/handler/loader/CSVLoader.java
+++ b/solr/core/src/java/org/apache/solr/handler/loader/CSVLoader.java
@@ -37,7 +37,7 @@ class SingleThreadedCSVLoader extends CSVLoaderBase {
}
@Override
- void addDoc(int line, String[] vals) throws IOException {
+ public void addDoc(int line, String[] vals) throws IOException {
templateAdd.clear();
SolrInputDocument doc = new SolrInputDocument();
doAdd(line, vals, doc, templateAdd);
diff --git a/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java b/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java
index f5bc2c8a0a4..2cc766211e3 100644
--- a/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java
+++ b/solr/core/src/java/org/apache/solr/handler/loader/CSVLoaderBase.java
@@ -37,7 +37,7 @@ import java.util.HashMap;
import java.util.Iterator;
import java.io.*;
-abstract class CSVLoaderBase extends ContentStreamLoader {
+public abstract class CSVLoaderBase extends ContentStreamLoader {
public static final String SEPARATOR="separator";
public static final String FIELDNAMES="fieldnames";
public static final String HEADER="header";
@@ -59,7 +59,7 @@ abstract class CSVLoaderBase extends ContentStreamLoader {
final SolrParams params;
final CSVStrategy strategy;
- final UpdateRequestProcessor processor;
+ protected final UpdateRequestProcessor processor;
// hashmap to save any literal fields and their values
HashMap <String, String> literals;
@@ -71,7 +71,7 @@ abstract class CSVLoaderBase extends ContentStreamLoader {
int skipLines; // number of lines to skip at start of file
- final AddUpdateCommand templateAdd;
+ protected final AddUpdateCommand templateAdd;
@@ -157,7 +157,7 @@ abstract class CSVLoaderBase extends ContentStreamLoader {
String errHeader="CSVLoader:";
- CSVLoaderBase(SolrQueryRequest req, UpdateRequestProcessor processor) {
+ protected CSVLoaderBase(SolrQueryRequest req, UpdateRequestProcessor processor) {
this.processor = processor;
this.params = req.getParams();
this.literals = new HashMap<>();
@@ -368,10 +368,10 @@ abstract class CSVLoaderBase extends ContentStreamLoader {
}
/** called for each line of values (document) */
- abstract void addDoc(int line, String[] vals) throws IOException;
+ public abstract void addDoc(int line, String[] vals) throws IOException;
/** this must be MT safe... may be called concurrently from multiple threads. */
- void doAdd(int line, String[] vals, SolrInputDocument doc, AddUpdateCommand template) throws IOException {
+ protected void doAdd(int line, String[] vals, SolrInputDocument doc, AddUpdateCommand template) throws IOException {
// the line number is passed for error reporting in MT mode as well as for optional rowId.
// first, create the lucene document
for (int i=0; i<vals.length; i++) {
diff --git a/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java b/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java
index 2964834244e..6791f0a84f8 100644
--- a/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java
+++ b/solr/core/src/java/org/apache/solr/handler/loader/JsonLoader.java
@@ -84,6 +84,41 @@ public class JsonLoader extends ContentStreamLoader {
new SingleThreadedJsonLoader(req, rsp, processor).load(req, rsp, stream, processor);
}
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public static SolrInputDocument buildDoc(Map<String, Object> m) {
+ SolrInputDocument result = new SolrInputDocument();
+ for (Map.Entry<String, Object> e : m.entrySet()) {
+ if (mapEntryIsChildDoc(e.getValue())) { // parse child documents
+ if (e.getValue() instanceof List) {
+ List value = (List) e.getValue();
+ for (Object o : value) {
+ if (o instanceof Map) {
+ // retain the value as a list, even if the list contains a single value.
+ if(!result.containsKey(e.getKey())) {
+ result.setField(e.getKey(), new ArrayList<>(1));
+ }
+ result.addField(e.getKey(), buildDoc((Map) o));
+ }
+ }
+ } else if (e.getValue() instanceof Map) {
+ result.addField(e.getKey(), buildDoc((Map) e.getValue()));
+ }
+ } else {
+ result.setField(e.getKey(), e.getValue());
+ }
+ }
+ return result;
+ }
+
+ private static boolean mapEntryIsChildDoc(Object val) {
+ if(val instanceof List) {
+ @SuppressWarnings({"rawtypes"})
+ List listVal = (List) val;
+ if (listVal.size() == 0) return false;
+ return listVal.get(0) instanceof Map;
+ }
+ return val instanceof Map;
+ }
static class SingleThreadedJsonLoader extends ContentStreamLoader {
@@ -247,32 +282,6 @@ public class JsonLoader extends ContentStreamLoader {
});
}
- @SuppressWarnings({"unchecked", "rawtypes"})
- private SolrInputDocument buildDoc(Map<String, Object> m) {
- SolrInputDocument result = new SolrInputDocument();
- for (Map.Entry<String, Object> e : m.entrySet()) {
- if (mapEntryIsChildDoc(e.getValue())) { // parse child documents
- if (e.getValue() instanceof List) {
- List value = (List) e.getValue();
- for (Object o : value) {
- if (o instanceof Map) {
- // retain the value as a list, even if the list contains a single value.
- if(!result.containsKey(e.getKey())) {
- result.setField(e.getKey(), new ArrayList<>(1));
- }
- result.addField(e.getKey(), buildDoc((Map) o));
- }
- }
- } else if (e.getValue() instanceof Map) {
- result.addField(e.getKey(), buildDoc((Map) e.getValue()));
- }
- } else {
- result.setField(e.getKey(), e.getValue());
- }
- }
- return result;
- }
-
@SuppressWarnings({"unchecked", "rawtypes"})
private Map<String, Object> getDocMap(Map<String, Object> record, JSONParser parser, String srcField, boolean mapUniqueKeyOnly) {
Map result = record;
@@ -625,15 +634,6 @@ public class JsonLoader extends ContentStreamLoader {
return extendedFieldValue.containsKey(req.getSchema().getUniqueKeyField().getName());
}
- private boolean mapEntryIsChildDoc(Object val) {
- if(val instanceof List) {
- @SuppressWarnings({"rawtypes"})
- List listVal = (List) val;
- if (listVal.size() == 0) return false;
- return listVal.get(0) instanceof Map;
- }
- return val instanceof Map;
- }
}
@SuppressWarnings({"unchecked", "rawtypes"})
diff --git a/solr/core/src/java/org/apache/solr/update/processor/ParseBooleanFieldUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/ParseBooleanFieldUpdateProcessorFactory.java
index ea1863f0bb7..cc25612f484 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/ParseBooleanFieldUpdateProcessorFactory.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/ParseBooleanFieldUpdateProcessorFactory.java
@@ -73,8 +73,8 @@ public class ParseBooleanFieldUpdateProcessorFactory extends FieldMutatingUpdate
private static final String FALSE_VALUES_PARAM = "falseValue";
private static final String CASE_SENSITIVE_PARAM = "caseSensitive";
- private Set<String> trueValues = new HashSet<>(Arrays.asList(new String[] { "true" }));
- private Set<String> falseValues = new HashSet<>(Arrays.asList(new String[] { "false" }));
+ private final Set<String> trueValues = new HashSet<>(Arrays.asList("true"));
+ private final Set<String> falseValues = new HashSet<>(Arrays.asList("false"));
private boolean caseSensitive = false;
@Override
@@ -84,20 +84,8 @@ public class ParseBooleanFieldUpdateProcessorFactory extends FieldMutatingUpdate
return new AllValuesOrNoneFieldMutatingUpdateProcessor(getSelector(), next) {
@Override
protected Object mutateValue(Object srcVal) {
- if (srcVal instanceof CharSequence) {
- String stringVal = caseSensitive ? srcVal.toString() : srcVal.toString().toLowerCase(Locale.ROOT);
- if (trueValues.contains(stringVal)) {
- return true;
- } else if (falseValues.contains(stringVal)) {
- return false;
- } else {
- return SKIP_FIELD_VALUE_LIST_SINGLETON;
- }
- }
- if (srcVal instanceof Boolean) {
- return srcVal;
- }
- return SKIP_FIELD_VALUE_LIST_SINGLETON;
+ Object parsed = parsePossibleBoolean(srcVal, caseSensitive, trueValues, falseValues);
+ return parsed != null ? parsed : SKIP_FIELD_VALUE_LIST_SINGLETON;
}
};
}
@@ -109,7 +97,7 @@ public class ParseBooleanFieldUpdateProcessorFactory extends FieldMutatingUpdate
if (caseSensitiveParam instanceof Boolean) {
caseSensitive = (Boolean)caseSensitiveParam;
} else {
- caseSensitive = Boolean.valueOf(caseSensitiveParam.toString());
+ caseSensitive = Boolean.parseBoolean(caseSensitiveParam.toString());
}
}
@@ -152,4 +140,21 @@ public class ParseBooleanFieldUpdateProcessorFactory extends FieldMutatingUpdate
return (null == type) || (type instanceof BoolField);
};
}
+
+ public static Object parsePossibleBoolean(Object srcVal, boolean caseSensitive, Set<String> trueValues, Set<String> falseValues) {
+ if (srcVal instanceof CharSequence) {
+ String stringVal = caseSensitive ? srcVal.toString() : srcVal.toString().toLowerCase(Locale.ROOT);
+ if (trueValues.contains(stringVal)) {
+ return true;
+ } else if (falseValues.contains(stringVal)) {
+ return false;
+ } else {
+ return null;
+ }
+ }
+ if (srcVal instanceof Boolean) {
+ return srcVal;
+ }
+ return null;
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/update/processor/ParseDateFieldUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/ParseDateFieldUpdateProcessorFactory.java
index bb5a88cc772..b31d429c63b 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/ParseDateFieldUpdateProcessorFactory.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/ParseDateFieldUpdateProcessorFactory.java
@@ -131,36 +131,8 @@ public class ParseDateFieldUpdateProcessorFactory extends FieldMutatingUpdatePro
@Override
protected Object mutateValue(Object srcVal) {
- if (srcVal instanceof CharSequence) {
- String srcStringVal = srcVal.toString();
- // trim single quotes around date if present
- // see issue #5279 (Apache HttpClient)
- int stringValLen = srcStringVal.length();
- if (stringValLen > 1
- && srcStringVal.startsWith("'")
- && srcStringVal.endsWith("'")
- ) {
- srcStringVal = srcStringVal.substring(1, stringValLen - 1);
- }
-
- for (Map.Entry<String,DateTimeFormatter> format : formats.entrySet()) {
- DateTimeFormatter parser = format.getValue();
- try {
- return Date.from(parseInstant(parser, srcStringVal, parsePosition));
- } catch (DateTimeParseException e) {
- if (log.isDebugEnabled()) {
- log.debug("value '{}' is not parseable with format '{}'",
- new Object[]{srcStringVal, format.getKey()});
- }
- }
- }
- log.debug("value '{}' was not parsed by any configured format, thus was not mutated", srcStringVal);
- return SKIP_FIELD_VALUE_LIST_SINGLETON;
- }
- if (srcVal instanceof Date) {
- return srcVal;
- }
- return SKIP_FIELD_VALUE_LIST_SINGLETON;
+ Object parsed = parsePossibleDate(srcVal, formats.values(), parsePosition);
+ return parsed != null ? parsed : SKIP_FIELD_VALUE_LIST_SINGLETON;
}
};
}
@@ -211,6 +183,37 @@ public class ParseDateFieldUpdateProcessorFactory extends FieldMutatingUpdatePro
};
}
+ public static Object parsePossibleDate(Object srcVal, Collection<DateTimeFormatter> parsers, ParsePosition parsePosition) {
+ if (srcVal instanceof CharSequence) {
+ String srcStringVal = srcVal.toString();
+ // trim single quotes around date if present
+ // see issue #5279 (Apache HttpClient)
+ int stringValLen = srcStringVal.length();
+ if (stringValLen > 1
+ && srcStringVal.startsWith("'")
+ && srcStringVal.endsWith("'")
+ ) {
+ srcStringVal = srcStringVal.substring(1, stringValLen - 1);
+ }
+
+ for (DateTimeFormatter parser: parsers) {
+ try {
+ return Date.from(parseInstant(parser, srcStringVal, parsePosition));
+ } catch (DateTimeParseException e) {
+ if (log.isDebugEnabled()) {
+ log.debug("value '{}' is not parseable with format '{}'", srcStringVal, parser);
+ }
+ }
+ }
+ log.debug("value '{}' was not parsed by any configured format, thus was not mutated", srcStringVal);
+ return null;
+ }
+ if (srcVal instanceof Date) {
+ return srcVal;
+ }
+ return null;
+ }
+
public static void validateFormatter(DateTimeFormatter formatter) {
// check it's valid via round-trip
try {
diff --git a/solr/core/src/java/org/apache/solr/update/processor/ParseDoubleFieldUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/ParseDoubleFieldUpdateProcessorFactory.java
index ad7eabb773e..af4b12fd484 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/ParseDoubleFieldUpdateProcessorFactory.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/ParseDoubleFieldUpdateProcessorFactory.java
@@ -97,23 +97,8 @@ public class ParseDoubleFieldUpdateProcessorFactory extends ParseNumericFieldUpd
@Override
protected Object mutateValue(Object srcVal) {
- if (srcVal instanceof CharSequence) {
- String stringVal = srcVal.toString();
- ParsePosition pos = new ParsePosition(0);
- Number number = numberFormat.get().parse(stringVal, pos);
- if (pos.getIndex() != stringVal.length()) {
- if (log.isDebugEnabled()) {
- log.debug("value '{}' is not parseable, thus not mutated; unparsed chars: '{}'",
- new Object[]{srcVal, stringVal.substring(pos.getIndex())});
- }
- return SKIP_FIELD_VALUE_LIST_SINGLETON;
- }
- return number.doubleValue();
- }
- if (srcVal instanceof Double) {
- return srcVal;
- }
- return SKIP_FIELD_VALUE_LIST_SINGLETON;
+ Object parsed = parsePossibleDouble(srcVal, numberFormat.get());
+ return parsed != null ? parsed : SKIP_FIELD_VALUE_LIST_SINGLETON;
}
}
@@ -121,4 +106,27 @@ public class ParseDoubleFieldUpdateProcessorFactory extends ParseNumericFieldUpd
protected boolean isSchemaFieldTypeCompatible(FieldType type) {
return type instanceof DoubleValueFieldType;
}
+
+ public static Object parsePossibleDouble(Object srcVal, NumberFormat numberFormat) {
+ if (srcVal instanceof CharSequence) {
+ String stringVal = srcVal.toString();
+ ParsePosition pos = new ParsePosition(0);
+ Number number = numberFormat.parse(stringVal, pos);
+ if (pos.getIndex() != stringVal.length()) {
+ if (log.isDebugEnabled()) {
+ log.debug("value '{}' is not parseable, thus not mutated; unparsed chars: '{}'",
+ srcVal, stringVal.substring(pos.getIndex()));
+ }
+ return null;
+ }
+ return number.doubleValue();
+ }
+ if (srcVal instanceof Double) {
+ return srcVal;
+ }
+ if (srcVal instanceof Float) {
+ return ((Float) srcVal).doubleValue();
+ }
+ return null;
+ }
}
diff --git a/solr/core/src/java/org/apache/solr/update/processor/ParseLongFieldUpdateProcessorFactory.java b/solr/core/src/java/org/apache/solr/update/processor/ParseLongFieldUpdateProcessorFactory.java
index b93a28fc942..ee8ce0a3e3f 100644
--- a/solr/core/src/java/org/apache/solr/update/processor/ParseLongFieldUpdateProcessorFactory.java
+++ b/solr/core/src/java/org/apache/solr/update/processor/ParseLongFieldUpdateProcessorFactory.java
@@ -93,23 +93,8 @@ public class ParseLongFieldUpdateProcessorFactory extends ParseNumericFieldUpdat
@Override
protected Object mutateValue(Object srcVal) {
- if (srcVal instanceof CharSequence) {
- String stringVal = srcVal.toString();
- ParsePosition pos = new ParsePosition(0);
- Number number = numberFormat.get().parse(stringVal, pos);
- if (pos.getIndex() != stringVal.length()) {
- if (log.isDebugEnabled()) {
- log.debug("value '{}' is not parseable, thus not mutated; unparsed chars: '{}'",
- new Object[]{srcVal, stringVal.substring(pos.getIndex())});
- }
- return SKIP_FIELD_VALUE_LIST_SINGLETON;
- }
- return number.longValue();
- }
- if (srcVal instanceof Long) {
- return srcVal;
- }
- return SKIP_FIELD_VALUE_LIST_SINGLETON;
+ Object parsed = parsePossibleLong(srcVal, numberFormat.get());
+ return parsed != null ? parsed : SKIP_FIELD_VALUE_LIST_SINGLETON;
}
}
@@ -117,4 +102,24 @@ public class ParseLongFieldUpdateProcessorFactory extends ParseNumericFieldUpdat
protected boolean isSchemaFieldTypeCompatible(FieldType type) {
return type instanceof LongValueFieldType;
}
+
+ public static Object parsePossibleLong(Object srcVal, NumberFormat numberFormat) {
+ if (srcVal instanceof CharSequence) {
+ String stringVal = srcVal.toString();
+ ParsePosition pos = new ParsePosition(0);
+ Number number = numberFormat.parse(stringVal, pos);
+ if (pos.getIndex() != stringVal.length()) {
+ if (log.isDebugEnabled()) {
+ log.debug("value '{}' is not parseable, thus not mutated; unparsed chars: '{}'",
+ srcVal, stringVal.substring(pos.getIndex()));
+ }
+ return null;
+ }
+ return number.longValue();
+ }
+ if (srcVal instanceof Long) {
+ return srcVal;
+ }
+ return null;
+ }
}
diff --git a/solr/core/src/test-files/schema-designer/add-new-field.json b/solr/core/src/test-files/schema-designer/add-new-field.json
new file mode 100644
index 00000000000..c25575fa20a
--- /dev/null
+++ b/solr/core/src/test-files/schema-designer/add-new-field.json
@@ -0,0 +1,9 @@
+{
+ "add-field": {
+ "name":"keywords",
+ "type":"string",
+ "stored":true,
+ "docValues": true,
+ "indexed": false
+ }
+}
\ No newline at end of file
diff --git a/solr/core/src/test-files/schema-designer/add-new-type.json b/solr/core/src/test-files/schema-designer/add-new-type.json
new file mode 100644
index 00000000000..3cd2f9ecd8c
--- /dev/null
+++ b/solr/core/src/test-files/schema-designer/add-new-type.json
@@ -0,0 +1,77 @@
+{ "add-field-type": {
+ "name": "test_txt",
+ "class": "solr.TextField",
+ "positionIncrementGap": "100",
+ "autoGeneratePhraseQueries": false,
+ "enableGraphQueries": true,
+ "synonymQueryStyle": "AS_SAME_TERM",
+ "indexed": true,
+ "stored": true,
+ "docValues": false,
+ "termVectors": false,
+ "omitNorms": false,
+ "omitTermFreqAndPositions": false,
+ "omitPositions": false,
+ "storeOffsetsWithPositions": false,
+ "multiValued": false,
+ "large": false,
+ "uninvertible": true,
+ "tokenized": true,
+ "indexAnalyzer": {
+ "tokenizer": {
+ "name": "standard"
+ },
+ "filters": [
+ {
+ "words": "lang/stopwords_en.txt",
+ "ignoreCase": "true",
+ "name": "stop"
+ },
+ {
+ "name": "lowercase"
+ },
+ {
+ "name": "englishPossessive"
+ },
+ {
+ "name": "keywordMarker",
+ "protected": "protwords.txt"
+ },
+ {
+ "name": "porterStem"
+ }
+ ]
+ },
+ "queryAnalyzer": {
+ "tokenizer": {
+ "name": "standard"
+ },
+ "filters": [
+ {
+ "name": "synonymGraph",
+ "synonyms": "synonyms.txt",
+ "ignoreCase": "true",
+ "expand": "true"
+ },
+ {
+ "words": "lang/stopwords_en.txt",
+ "ignoreCase": "true",
+ "name": "stop"
+ },
+ {
+ "name": "lowercase"
+ },
+ {
+ "name": "englishPossessive"
+ },
+ {
+ "name": "keywordMarker",
+ "protected": "protwords.txt"
+ },
+ {
+ "name": "porterStem"
+ }
+ ]
+ }
+}
+}
diff --git a/solr/core/src/test-files/schema-designer/update-author-field.json b/solr/core/src/test-files/schema-designer/update-author-field.json
new file mode 100644
index 00000000000..13d06cdcb21
--- /dev/null
+++ b/solr/core/src/test-files/schema-designer/update-author-field.json
@@ -0,0 +1,11 @@
+{
+ "href": "field/author",
+ "name": "author",
+ "type": "string",
+ "indexed": true,
+ "stored": true,
+ "docValues": true,
+ "multiValued": true,
+ "id": "field/author_anchor",
+ "copyDest": "_text_"
+}
\ No newline at end of file
diff --git a/solr/core/src/test-files/schema-designer/update-type.json b/solr/core/src/test-files/schema-designer/update-type.json
new file mode 100644
index 00000000000..00b7eefaa52
--- /dev/null
+++ b/solr/core/src/test-files/schema-designer/update-type.json
@@ -0,0 +1,76 @@
+{
+ "name": "test_txt",
+ "class": "solr.TextField",
+ "positionIncrementGap": "100",
+ "enableGraphQueries": true,
+ "autoGeneratePhraseQueries": false,
+ "synonymQueryStyle": "org.apache.solr.parser.SolrQueryParserBase$SynonymQueryStyle:AS_SAME_TERM",
+ "indexed": true,
+ "stored": true,
+ "docValues": false,
+ "termVectors": false,
+ "omitNorms": false,
+ "omitTermFreqAndPositions": false,
+ "omitPositions": false,
+ "storeOffsetsWithPositions": false,
+ "multiValued": true,
+ "large": false,
+ "uninvertible": true,
+ "tokenized": true,
+ "indexAnalyzer": {
+ "tokenizer": {
+ "name": "standard"
+ },
+ "filters": [
+ {
+ "name": "stop",
+ "ignoreCase": "true",
+ "words": "lang/stopwords_en.txt"
+ },
+ {
+ "name": "lowercase"
+ },
+ {
+ "name": "englishPossessive"
+ },
+ {
+ "name": "keywordMarker",
+ "protected": "protwords.txt"
+ },
+ {
+ "name": "porterStem"
+ }
+ ]
+ },
+ "queryAnalyzer": {
+ "tokenizer": {
+ "name": "standard"
+ },
+ "filters": [
+ {
+ "name": "synonymGraph",
+ "expand": "true",
+ "ignoreCase": "true",
+ "synonyms": "synonyms.txt"
+ },
+ {
+ "name": "stop",
+ "ignoreCase": "true",
+ "words": "lang/stopwords_en.txt"
+ },
+ {
+ "name": "lowercase"
+ },
+ {
+ "name": "englishPossessive"
+ },
+ {
+ "name": "keywordMarker",
+ "protected": "protwords.txt"
+ },
+ {
+ "name": "porterStem"
+ }
+ ]
+ }
+}
\ No newline at end of file
diff --git a/solr/core/src/test/org/apache/solr/handler/TestSampleDocumentsLoader.java b/solr/core/src/test/org/apache/solr/handler/TestSampleDocumentsLoader.java
new file mode 100644
index 00000000000..057b97f0a7b
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/TestSampleDocumentsLoader.java
@@ -0,0 +1,123 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.SolrInputField;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.ContentStreamBase;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.handler.designer.DefaultSampleDocumentsLoader;
+import org.apache.solr.handler.designer.SampleDocuments;
+import org.apache.solr.handler.designer.SampleDocumentsLoader;
+import org.apache.solr.util.ExternalPaths;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.apache.solr.handler.designer.DefaultSampleDocumentsLoader.CSV_MULTI_VALUE_DELIM_PARAM;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+public class TestSampleDocumentsLoader {
+
+ SampleDocumentsLoader loader;
+ File exampleDir;
+
+ @Before
+ public void setup() throws IOException {
+ loader = new DefaultSampleDocumentsLoader();
+ loader.init(new NamedList<>());
+ exampleDir = new File(ExternalPaths.SOURCE_HOME, "example");
+ assertTrue("Required test data directory " + exampleDir.getCanonicalPath() + " not found!", exampleDir.isDirectory());
+ }
+
+ @Test
+ public void testJson() throws Exception {
+ loadTestDocs(null, new File(exampleDir, "films/films.json"), 500, 500);
+ }
+
+ @Test
+ public void testCsv() throws Exception {
+ ModifiableSolrParams params = new ModifiableSolrParams();
+ params.set(CSV_MULTI_VALUE_DELIM_PARAM, "\\|");
+ List<SolrInputDocument> docs = loadTestDocs(params, new File(exampleDir, "films/films.csv"), -1, 1100);
+ boolean foundIt = false;
+ for (SolrInputDocument next : docs) {
+ if (".45".equals(next.getFieldValue("name"))) {
+ SolrInputField genreField = next.getField("genre");
+ assertNotNull(genreField);
+ assertEquals(8, genreField.getValueCount());
+ foundIt = true;
+ break;
+ }
+ }
+ if (!foundIt) {
+ fail("Didn't find the expected film with name '.45' in the parsed docs!");
+ }
+ }
+
+ @Test
+ public void testSolrXml() throws Exception {
+ loadTestDocs(null, new File(exampleDir, "films/films.xml"), 1000, 1000);
+ }
+
+ protected List<SolrInputDocument> loadTestDocs(SolrParams params, File inputDocs, int maxDocsToLoad, int expectedDocs) throws IOException {
+ assertTrue(inputDocs.getCanonicalPath() + " not found", inputDocs.isFile());
+ ContentStream stream = getContentStream(inputDocs);
+ SampleDocuments sampleDocs = loader.parseDocsFromStream(params, stream, maxDocsToLoad);
+ assertNotNull(sampleDocs);
+ assertEquals(sampleDocs.parsed.size(), expectedDocs);
+ return sampleDocs.parsed;
+ }
+
+ public static String guessContentTypeFromFilename(String name) {
+ int dotAt = name.lastIndexOf('.');
+ if (dotAt != -1) {
+ final String ext = name.substring(dotAt + 1);
+ switch (ext) {
+ case "csv":
+ return "text/csv";
+ case "txt":
+ return "text/plain";
+ case "json":
+ return "application/json";
+ case "xml":
+ return "text/xml";
+ }
+ }
+ return "application/octet-stream";
+ }
+
+ protected ContentStream getContentStream(File file) {
+ return getContentStream(file, guessContentTypeFromFilename(file.getName()));
+ }
+
+ protected ContentStream getContentStream(File file, String contentType) {
+ ContentStreamBase.FileStream stream = new ContentStreamBase.FileStream(file);
+ stream.setContentType(contentType);
+ return stream;
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/handler/designer/ManagedSchemaDiffTest.java b/solr/core/src/test/org/apache/solr/handler/designer/ManagedSchemaDiffTest.java
new file mode 100644
index 00000000000..df3fcddeb31
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/designer/ManagedSchemaDiffTest.java
@@ -0,0 +1,116 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.designer;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.schema.BoolField;
+import org.apache.solr.schema.IntPointField;
+import org.apache.solr.schema.ManagedIndexSchema;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.util.ExternalPaths;
+import org.junit.Assert;
+import org.junit.BeforeClass;
+
+import static org.apache.solr.handler.admin.ConfigSetsHandler.DEFAULT_CONFIGSET_NAME;
+import static org.apache.solr.handler.designer.ManagedSchemaDiff.mapFieldsToPropertyValues;
+
+public class ManagedSchemaDiffTest extends SolrCloudTestCase {
+
+ @BeforeClass
+ public static void createCluster() throws Exception {
+ System.setProperty("managed.schema.mutable", "true");
+ configureCluster(1).addConfig(DEFAULT_CONFIGSET_NAME, new File(ExternalPaths.DEFAULT_CONFIGSET).toPath()).configure();
+ }
+
+ public void testFieldDiff() {
+ SchemaDesignerConfigSetHelper helper =
+ new SchemaDesignerConfigSetHelper(cluster.getJettySolrRunner(0).getCoreContainer(), null);
+ ManagedIndexSchema schema = helper.loadLatestSchema(DEFAULT_CONFIGSET_NAME);
+
+ Map<String, SchemaField> schema1FieldMap = new HashMap<>();
+ schema1FieldMap.put("strfield", schema.newField("strfield", "string", Collections.emptyMap()));
+ schema1FieldMap.put("boolfield", new SchemaField("boolfield", new BoolField()));
+
+ Map<String, SchemaField> schema2FieldMap = new HashMap<>();
+ schema2FieldMap.put("strfield", schema.newField("strfield", "strings", Collections.emptyMap()));
+ schema2FieldMap.put("intfield", new SchemaField("intfield", new IntPointField()));
+
+ Map<String, Object> diff = ManagedSchemaDiff.diff(mapFieldsToPropertyValues(schema1FieldMap), mapFieldsToPropertyValues(schema2FieldMap));
+ Assert.assertTrue(diff.containsKey("updated"));
+ Assert.assertTrue(diff.containsKey("added"));
+ Assert.assertTrue(diff.containsKey("removed"));
+
+ Map<String, Object> changedFields = getInnerMap(diff, "updated");
+ Assert.assertEquals(1, changedFields.size());
+ Assert.assertTrue(changedFields.containsKey("strfield"));
+ Assert.assertEquals(
+ Arrays.asList(Map.of("type", "string", "multiValued", false),
+ Map.of("type", "strings", "multiValued", true)),
+ changedFields.get("strfield"));
+
+ Map<String, Object> addedFields = getInnerMap(diff, "added");
+ Assert.assertEquals(1, addedFields.size());
+ Assert.assertTrue(addedFields.containsKey("intfield"));
+ Assert.assertEquals(schema2FieldMap.get("intfield").getNamedPropertyValues(true), addedFields.get("intfield"));
+
+ Map<String, Object> removedFields = getInnerMap(diff, "removed");
+ Assert.assertEquals(1, removedFields.size());
+ Assert.assertTrue(removedFields.containsKey("boolfield"));
+ Assert.assertEquals(schema1FieldMap.get("boolfield").getNamedPropertyValues(true), removedFields.get("boolfield"));
+ }
+
+ public void testSimpleOrderedMapListDiff() {
+ SimpleOrderedMap<Object> obj1 = new SimpleOrderedMap<>();
+ obj1.add("name", "obj1");
+ obj1.add("type", "objtype1");
+
+ SimpleOrderedMap<Object> obj2 = new SimpleOrderedMap<>();
+ obj2.add("name", "obj2");
+ obj2.add("type", "objtype2");
+
+ SimpleOrderedMap<Object> obj3 = new SimpleOrderedMap<>();
+ obj3.add("name", "obj3");
+ obj3.add("type", "objtype3");
+
+ SimpleOrderedMap<Object> obj4 = new SimpleOrderedMap<>();
+ obj4.add("name", "obj4");
+ obj4.add("type", "objtype4");
+
+ List<SimpleOrderedMap<Object>> list1 = Arrays.asList(obj1, obj2);
+ List<SimpleOrderedMap<Object>> list2 = Arrays.asList(obj1, obj3, obj4);
+
+ Map<String, Object> diff = ManagedSchemaDiff.diff(list1, list2);
+ Assert.assertTrue(diff.containsKey("old"));
+ Assert.assertTrue(diff.containsKey("new"));
+ Assert.assertEquals(Collections.singletonList(obj2), diff.get("old"));
+ Assert.assertEquals(Arrays.asList(obj3, obj4), diff.get("new"));
+ }
+
+ @SuppressWarnings("unchecked")
+ private Map<String, Object> getInnerMap(Map<String, Object> map, String key) {
+ return (Map<String, Object>) map.get(key);
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerAPI.java b/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerAPI.java
new file mode 100644
index 00000000000..7e472bd4b24
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerAPI.java
@@ -0,0 +1,873 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.designer;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+
+import com.google.common.collect.ImmutableMap;
+import org.apache.solr.client.solrj.SolrQuery;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.client.solrj.response.QueryResponse;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.SolrDocumentList;
+import org.apache.solr.common.SolrException;
+import org.apache.solr.common.cloud.SolrZkClient;
+import org.apache.solr.common.params.CommonParams;
+import org.apache.solr.common.params.ModifiableSolrParams;
+import org.apache.solr.common.params.SolrParams;
+import org.apache.solr.common.util.ContentStream;
+import org.apache.solr.common.util.ContentStreamBase;
+import org.apache.solr.common.util.NamedList;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.handler.TestSampleDocumentsLoader;
+import org.apache.solr.request.SolrQueryRequest;
+import org.apache.solr.response.SolrQueryResponse;
+import org.apache.solr.schema.ManagedIndexSchema;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.util.ExternalPaths;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.noggit.JSONUtil;
+
+import static org.apache.solr.common.params.CommonParams.JSON_MIME;
+import static org.apache.solr.handler.admin.ConfigSetsHandler.DEFAULT_CONFIGSET_NAME;
+import static org.apache.solr.handler.designer.SchemaDesignerAPI.getMutableId;
+import static org.apache.solr.response.RawResponseWriter.CONTENT;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+public class TestSchemaDesignerAPI extends SolrCloudTestCase implements SchemaDesignerConstants {
+
+ private CoreContainer cc;
+ private SchemaDesignerAPI schemaDesignerAPI;
+
+ @BeforeClass
+ public static void createCluster() throws Exception {
+ System.setProperty("managed.schema.mutable", "true");
+ configureCluster(1).addConfig(DEFAULT_CONFIGSET_NAME, new File(ExternalPaths.DEFAULT_CONFIGSET).toPath()).configure();
+ // SchemaDesignerAPI depends on the blob store
+ CollectionAdminRequest.createCollection(BLOB_STORE_ID, 1, 1).process(cluster.getSolrClient());
+ cluster.waitForActiveCollection(BLOB_STORE_ID, 1, 1);
+ }
+
+ @AfterClass
+ public static void tearDownCluster() throws Exception {
+ if (cluster != null && cluster.getSolrClient() != null) {
+ cluster.deleteAllCollections();
+ cluster.deleteAllConfigSets();
+ }
+ }
+
+ @Before
+ public void setupTest() {
+ assumeWorkingMockito();
+ assertNotNull(cluster);
+ cc = cluster.getJettySolrRunner(0).getCoreContainer();
+ assertNotNull(cc);
+ schemaDesignerAPI = new SchemaDesignerAPI(cc);
+ }
+
+ public void testTSV() throws Exception {
+ String configSet = "testTSV";
+
+ ModifiableSolrParams reqParams = new ModifiableSolrParams();
+
+ // GET /schema-designer/info
+ SolrQueryResponse rsp = new SolrQueryResponse();
+ SolrQueryRequest req = mock(SolrQueryRequest.class);
+
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+ reqParams.set(LANGUAGES_PARAM, "en");
+ reqParams.set(ENABLE_DYNAMIC_FIELDS_PARAM, false);
+ when(req.getParams()).thenReturn(reqParams);
+
+ String tsv = "id\tcol1\tcol2\n1\tfoo\tbar\n2\tbaz\tbah\n";
+
+ // POST some sample TSV docs
+ ContentStream stream = new ContentStreamBase.StringStream(tsv, "text/csv");
+ when(req.getContentStreams()).thenReturn(Collections.singletonList(stream));
+
+ // POST /schema-designer/analyze
+ schemaDesignerAPI.analyze(req, rsp);
+ assertNotNull(rsp.getValues().get(CONFIG_SET_PARAM));
+ assertNotNull(rsp.getValues().get(SCHEMA_VERSION_PARAM));
+ assertEquals(2, rsp.getValues().get("numDocs"));
+
+ reqParams.clear();
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+ rsp = new SolrQueryResponse();
+ schemaDesignerAPI.cleanupTemp(req, rsp);
+
+ String mutableId = getMutableId(configSet);
+ assertFalse(cc.getZkController().getClusterState().hasCollection(mutableId));
+ SolrZkClient zkClient = cc.getZkController().getZkClient();
+ assertFalse(zkClient.exists("/configs/" + mutableId, true));
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testAddTechproductsProgressively() throws Exception {
+ File docsDir = new File(ExternalPaths.SOURCE_HOME, "example/exampledocs");
+ assertTrue(docsDir.getAbsolutePath() + " not found!", docsDir.isDirectory());
+ File[] toAdd = docsDir.listFiles((dir, name) -> name.endsWith(".xml") || name.endsWith(".json") || name.endsWith(".csv") || name.endsWith(".jsonl"));
+ assertNotNull("No test data files found in " + docsDir.getAbsolutePath(), toAdd);
+
+ String configSet = "techproducts";
+
+ ModifiableSolrParams reqParams = new ModifiableSolrParams();
+
+ // GET /schema-designer/info
+ SolrQueryResponse rsp = new SolrQueryResponse();
+ SolrQueryRequest req = mock(SolrQueryRequest.class);
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+ when(req.getParams()).thenReturn(reqParams);
+ schemaDesignerAPI.getInfo(req, rsp);
+ // response should just be the default values
+ Map<String, Object> expSettings = Map.of(
+ ENABLE_DYNAMIC_FIELDS_PARAM, true,
+ ENABLE_FIELD_GUESSING_PARAM, true,
+ ENABLE_NESTED_DOCS_PARAM, false,
+ LANGUAGES_PARAM, Collections.emptyList());
+ assertDesignerSettings(expSettings, rsp.getValues());
+ SolrParams rspData = rsp.getValues().toSolrParams();
+ int schemaVersion = rspData.getInt(SCHEMA_VERSION_PARAM);
+ assertEquals(schemaVersion, -1); // shouldn't exist yet
+
+ // Use the prep endpoint to prepare the new schema
+ reqParams.clear();
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+ rsp = new SolrQueryResponse();
+ req = mock(SolrQueryRequest.class);
+ when(req.getParams()).thenReturn(reqParams);
+ schemaDesignerAPI.prepNewSchema(req, rsp);
+ assertNotNull(rsp.getValues().get(CONFIG_SET_PARAM));
+ assertNotNull(rsp.getValues().get(SCHEMA_VERSION_PARAM));
+ rspData = rsp.getValues().toSolrParams();
+ schemaVersion = rspData.getInt(SCHEMA_VERSION_PARAM);
+
+ for (File next : toAdd) {
+ // Analyze some sample documents to refine the schema
+ reqParams.clear();
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+ reqParams.set(LANGUAGES_PARAM, "en");
+ reqParams.set(ENABLE_DYNAMIC_FIELDS_PARAM, false);
+ reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion));
+ req = mock(SolrQueryRequest.class);
+ when(req.getParams()).thenReturn(reqParams);
+
+ // POST some sample JSON docs
+ ContentStreamBase.FileStream stream = new ContentStreamBase.FileStream(next);
+ stream.setContentType(TestSampleDocumentsLoader.guessContentTypeFromFilename(next.getName()));
+ when(req.getContentStreams()).thenReturn(Collections.singletonList(stream));
+
+ rsp = new SolrQueryResponse();
+
+ // POST /schema-designer/analyze
+ schemaDesignerAPI.analyze(req, rsp);
+
+ assertNotNull(rsp.getValues().get(CONFIG_SET_PARAM));
+ assertNotNull(rsp.getValues().get(SCHEMA_VERSION_PARAM));
+ assertNotNull(rsp.getValues().get("fields"));
+ assertNotNull(rsp.getValues().get("fieldTypes"));
+ assertNotNull(rsp.getValues().get("docIds"));
+
+ // capture the schema version for MVCC
+ rspData = rsp.getValues().toSolrParams();
+ schemaVersion = rspData.getInt(SCHEMA_VERSION_PARAM);
+ }
+
+ // get info (from the temp)
+ reqParams.clear();
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+ req = mock(SolrQueryRequest.class);
+ when(req.getParams()).thenReturn(reqParams);
+ rsp = new SolrQueryResponse();
+
+ // GET /schema-designer/info
+ schemaDesignerAPI.getInfo(req, rsp);
+ expSettings = Map.of(
+ ENABLE_DYNAMIC_FIELDS_PARAM, false,
+ ENABLE_FIELD_GUESSING_PARAM, true,
+ ENABLE_NESTED_DOCS_PARAM, false,
+ LANGUAGES_PARAM, Collections.singletonList("en"),
+ COPY_FROM_PARAM, "_default");
+ assertDesignerSettings(expSettings, rsp.getValues());
+
+ // query to see how the schema decisions impact retrieval / ranking
+ reqParams.clear();
+ reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion));
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+ reqParams.set(CommonParams.Q, "*:*");
+ req = mock(SolrQueryRequest.class);
+ when(req.getParams()).thenReturn(reqParams);
+ rsp = new SolrQueryResponse();
+
+ // GET /schema-designer/query
+ schemaDesignerAPI.query(req, rsp);
+ assertNotNull(rsp.getResponseHeader());
+ SolrDocumentList results = (SolrDocumentList) rsp.getResponse();
+ assertEquals(48, results.getNumFound());
+
+ // publish schema to a config set that can be used by real collections
+ reqParams.clear();
+ reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion));
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+
+ String collection = "techproducts";
+ reqParams.set(NEW_COLLECTION_PARAM, collection);
+ reqParams.set(INDEX_TO_COLLECTION_PARAM, true);
+ reqParams.set(RELOAD_COLLECTIONS_PARAM, true);
+ reqParams.set(CLEANUP_TEMP_PARAM, true);
+ reqParams.set(DISABLE_DESIGNER_PARAM, true);
+
+ rsp = new SolrQueryResponse();
+ schemaDesignerAPI.publish(req, rsp);
+ assertNotNull(cc.getZkController().zkStateReader.getCollection(collection));
+
+ // listCollectionsForConfig
+ reqParams.clear();
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+ rsp = new SolrQueryResponse();
+ schemaDesignerAPI.listCollectionsForConfig(req, rsp);
+ List<String> collections = (List<String>) rsp.getValues().get("collections");
+ assertNotNull(collections);
+ assertTrue(collections.contains(collection));
+
+ // now try to create another temp, which should fail since designer is disabled for this configSet now
+ reqParams.clear();
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+ rsp = new SolrQueryResponse();
+ req = mock(SolrQueryRequest.class);
+ when(req.getParams()).thenReturn(reqParams);
+ try {
+ schemaDesignerAPI.prepNewSchema(req, rsp);
+ fail("Prep should fail for locked schema " + configSet);
+ } catch (SolrException solrExc) {
+ assertEquals(SolrException.ErrorCode.BAD_REQUEST.code, solrExc.code());
+ }
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testSuggestFilmsXml() throws Exception {
+ String configSet = "films";
+
+ ModifiableSolrParams reqParams = new ModifiableSolrParams();
+
+ File filmsDir = new File(ExternalPaths.SOURCE_HOME, "example/films");
+ assertTrue(filmsDir.getAbsolutePath() + " not found!", filmsDir.isDirectory());
+ File filmsXml = new File(filmsDir, "films.xml");
+ assertTrue("example/films/films.xml not found", filmsXml.isFile());
+
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+ reqParams.set(ENABLE_DYNAMIC_FIELDS_PARAM, "true");
+
+ SolrQueryRequest req = mock(SolrQueryRequest.class);
+ when(req.getParams()).thenReturn(reqParams);
+
+ // POST some sample XML docs
+ ContentStreamBase.FileStream stream = new ContentStreamBase.FileStream(filmsXml);
+ stream.setContentType("application/xml");
+ when(req.getContentStreams()).thenReturn(Collections.singletonList(stream));
+
+ SolrQueryResponse rsp = new SolrQueryResponse();
+
+ // POST /schema-designer/analyze
+ schemaDesignerAPI.analyze(req, rsp);
+
+ assertNotNull(rsp.getValues().get(CONFIG_SET_PARAM));
+ assertNotNull(rsp.getValues().get(SCHEMA_VERSION_PARAM));
+ assertNotNull(rsp.getValues().get("fields"));
+ assertNotNull(rsp.getValues().get("fieldTypes"));
+ List<String> docIds = (List<String>) rsp.getValues().get("docIds");
+ assertNotNull(docIds);
+ assertEquals(100, docIds.size()); // designer limits doc ids to top 100
+
+ String idField = rsp.getValues()._getStr(UNIQUE_KEY_FIELD_PARAM, null);
+ assertNotNull(idField);
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testBasicUserWorkflow() throws Exception {
+ String configSet = "testJson";
+
+ ModifiableSolrParams reqParams = new ModifiableSolrParams();
+
+ // Use the prep endpoint to prepare the new schema
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+ SolrQueryResponse rsp = new SolrQueryResponse();
+ SolrQueryRequest req = mock(SolrQueryRequest.class);
+ when(req.getParams()).thenReturn(reqParams);
+ schemaDesignerAPI.prepNewSchema(req, rsp);
+ assertNotNull(rsp.getValues().get(CONFIG_SET_PARAM));
+ assertNotNull(rsp.getValues().get(SCHEMA_VERSION_PARAM));
+
+ Map<String, Object> expSettings = Map.of(
+ ENABLE_DYNAMIC_FIELDS_PARAM, true,
+ ENABLE_FIELD_GUESSING_PARAM, true,
+ ENABLE_NESTED_DOCS_PARAM, false,
+ LANGUAGES_PARAM, Collections.emptyList(),
+ COPY_FROM_PARAM, "_default");
+ assertDesignerSettings(expSettings, rsp.getValues());
+
+ // Analyze some sample documents to refine the schema
+ reqParams.clear();
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+ req = mock(SolrQueryRequest.class);
+ when(req.getParams()).thenReturn(reqParams);
+
+ // POST some sample JSON docs
+ File booksJson = new File(ExternalPaths.SOURCE_HOME, "example/exampledocs/books.json");
+ ContentStreamBase.FileStream stream = new ContentStreamBase.FileStream(booksJson);
+ stream.setContentType(JSON_MIME);
+ when(req.getContentStreams()).thenReturn(Collections.singletonList(stream));
+
+ rsp = new SolrQueryResponse();
+
+ // POST /schema-designer/analyze
+ schemaDesignerAPI.analyze(req, rsp);
+
+ assertNotNull(rsp.getValues().get(CONFIG_SET_PARAM));
+ assertNotNull(rsp.getValues().get(SCHEMA_VERSION_PARAM));
+ assertNotNull(rsp.getValues().get("fields"));
+ assertNotNull(rsp.getValues().get("fieldTypes"));
+ assertNotNull(rsp.getValues().get("docIds"));
+ String idField = rsp.getValues()._getStr(UNIQUE_KEY_FIELD_PARAM, null);
+ assertNotNull(idField);
+ assertDesignerSettings(expSettings, rsp.getValues());
+
+ // capture the schema version for MVCC
+ SolrParams rspData = rsp.getValues().toSolrParams();
+ reqParams.clear();
+ int schemaVersion = rspData.getInt(SCHEMA_VERSION_PARAM);
+
+ // load the contents of a file
+ Collection<String> files = (Collection<String>) rsp.getValues().get("files");
+ assertTrue(files != null && !files.isEmpty());
+
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+ String file = null;
+ for (String f : files) {
+ if ("solrconfig.xml".equals(f)) {
+ file = f;
+ break;
+ }
+ }
+ assertNotNull("solrconfig.xml not found in files!", file);
+ reqParams.set("file", file);
+ req = mock(SolrQueryRequest.class);
+ when(req.getParams()).thenReturn(reqParams);
+ rsp = new SolrQueryResponse();
+ schemaDesignerAPI.getFileContents(req, rsp);
+ String solrconfigXml = (String) rsp.getValues().get(file);
+ assertNotNull(solrconfigXml);
+ reqParams.clear();
+
+ // Update solrconfig.xml
+ rsp = new SolrQueryResponse();
+ reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion));
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+ reqParams.set("file", file);
+
+ req = mock(SolrQueryRequest.class);
+ when(req.getParams()).thenReturn(reqParams);
+ when(req.getContentStreams()).thenReturn(
+ Collections.singletonList(new ContentStreamBase.StringStream(solrconfigXml, "application/xml")));
+
+ schemaDesignerAPI.updateFileContents(req, rsp);
+ rspData = rsp.getValues().toSolrParams();
+ reqParams.clear();
+ schemaVersion = rspData.getInt(SCHEMA_VERSION_PARAM);
+
+ // update solrconfig.xml with some invalid XML mess
+ rsp = new SolrQueryResponse();
+ reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion));
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+ reqParams.set("file", file);
+
+ req = mock(SolrQueryRequest.class);
+ when(req.getParams()).thenReturn(reqParams);
+ when(req.getContentStreams()).thenReturn(
+ Collections.singletonList(new ContentStreamBase.StringStream("<config/>", "application/xml")));
+
+ // this should fail b/c the updated solrconfig.xml is invalid
+ schemaDesignerAPI.updateFileContents(req, rsp);
+ rspData = rsp.getValues().toSolrParams();
+ reqParams.clear();
+ assertNotNull(rspData.get("updateFileError"));
+
+ // remove dynamic fields and change the language to "en" only
+ rsp = new SolrQueryResponse();
+ // POST /schema-designer/analyze
+ reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion));
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+ reqParams.set(LANGUAGES_PARAM, "en");
+ reqParams.set(ENABLE_DYNAMIC_FIELDS_PARAM, false);
+ reqParams.set(ENABLE_FIELD_GUESSING_PARAM, false);
+ req = mock(SolrQueryRequest.class);
+ when(req.getParams()).thenReturn(reqParams);
+ schemaDesignerAPI.analyze(req, rsp);
+
+ expSettings = Map.of(
+ ENABLE_DYNAMIC_FIELDS_PARAM, false,
+ ENABLE_FIELD_GUESSING_PARAM, false,
+ ENABLE_NESTED_DOCS_PARAM, false,
+ LANGUAGES_PARAM, Collections.singletonList("en"),
+ COPY_FROM_PARAM, "_default");
+ assertDesignerSettings(expSettings, rsp.getValues());
+
+ List<String> filesInResp = (List<String>) rsp.getValues().get("files");
+ assertEquals(5, filesInResp.size());
+ assertTrue(filesInResp.contains("lang/stopwords_en.txt"));
+
+ rspData = rsp.getValues().toSolrParams();
+ schemaVersion = rspData.getInt(SCHEMA_VERSION_PARAM);
+
+ reqParams.clear();
+
+ // add the dynamic fields back and change the languages too
+ rsp = new SolrQueryResponse();
+ reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion));
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+ reqParams.add(LANGUAGES_PARAM, "en");
+ reqParams.add(LANGUAGES_PARAM, "fr");
+ reqParams.set(ENABLE_DYNAMIC_FIELDS_PARAM, true);
+ reqParams.set(ENABLE_FIELD_GUESSING_PARAM, false);
+ req = mock(SolrQueryRequest.class);
+ when(req.getParams()).thenReturn(reqParams);
+ schemaDesignerAPI.analyze(req, rsp);
+
+ expSettings = Map.of(
+ ENABLE_DYNAMIC_FIELDS_PARAM, true,
+ ENABLE_FIELD_GUESSING_PARAM, false,
+ ENABLE_NESTED_DOCS_PARAM, false,
+ LANGUAGES_PARAM, Arrays.asList("en", "fr"),
+ COPY_FROM_PARAM, "_default");
+ assertDesignerSettings(expSettings, rsp.getValues());
+
+ filesInResp = (List<String>) rsp.getValues().get("files");
+ assertEquals(7, filesInResp.size());
+ assertTrue(filesInResp.contains("lang/stopwords_fr.txt"));
+
+ rspData = rsp.getValues().toSolrParams();
+ reqParams.clear();
+ schemaVersion = rspData.getInt(SCHEMA_VERSION_PARAM);
+
+ // add back all the default languages
+ rsp = new SolrQueryResponse();
+ reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion));
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+ reqParams.add(LANGUAGES_PARAM, "*");
+ reqParams.set(ENABLE_DYNAMIC_FIELDS_PARAM, false);
+ req = mock(SolrQueryRequest.class);
+ when(req.getParams()).thenReturn(reqParams);
+ schemaDesignerAPI.analyze(req, rsp);
+
+ expSettings = Map.of(
+ ENABLE_DYNAMIC_FIELDS_PARAM, false,
+ ENABLE_FIELD_GUESSING_PARAM, false,
+ ENABLE_NESTED_DOCS_PARAM, false,
+ LANGUAGES_PARAM, Collections.emptyList(),
+ COPY_FROM_PARAM, "_default");
+ assertDesignerSettings(expSettings, rsp.getValues());
+
+ filesInResp = (List<String>) rsp.getValues().get("files");
+ assertEquals(43, filesInResp.size());
+ assertTrue(filesInResp.contains("lang/stopwords_fr.txt"));
+ assertTrue(filesInResp.contains("lang/stopwords_en.txt"));
+ assertTrue(filesInResp.contains("lang/stopwords_it.txt"));
+
+ rspData = rsp.getValues().toSolrParams();
+ reqParams.clear();
+ schemaVersion = rspData.getInt(SCHEMA_VERSION_PARAM);
+
+ // Get the value of a sample document
+ String docId = "978-0641723445";
+ String fieldName = "series_t";
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+ reqParams.set(DOC_ID_PARAM, docId);
+ reqParams.set(FIELD_PARAM, fieldName);
+ reqParams.set(UNIQUE_KEY_FIELD_PARAM, idField);
+ req = mock(SolrQueryRequest.class);
+ when(req.getParams()).thenReturn(reqParams);
+ rsp = new SolrQueryResponse();
+
+ // GET /schema-designer/sample
+ schemaDesignerAPI.getSampleValue(req, rsp);
+ rspData = rsp.getValues().toSolrParams();
+ assertNotNull(rspData.get(idField));
+ assertNotNull(rspData.get(fieldName));
+ assertNotNull(rspData.get("analysis"));
+
+ reqParams.clear();
+
+ // at this point the user would refine the schema by
+ // editing suggestions for fields and adding/removing fields / field types as needed
+
+ // add a new field
+ reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion));
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+
+ req = mock(SolrQueryRequest.class);
+ when(req.getParams()).thenReturn(reqParams);
+ stream = new ContentStreamBase.FileStream(getFile("schema-designer/add-new-field.json"));
+ stream.setContentType(JSON_MIME);
+ when(req.getContentStreams()).thenReturn(Collections.singletonList(stream));
+ rsp = new SolrQueryResponse();
+
+ // POST /schema-designer/add
+ schemaDesignerAPI.addSchemaObject(req, rsp);
+ assertNotNull(rsp.getValues().get("add-field"));
+ rspData = rsp.getValues().toSolrParams();
+ schemaVersion = rspData.getInt(SCHEMA_VERSION_PARAM);
+ assertNotNull(rsp.getValues().get("fields"));
+
+ // update an existing field
+ reqParams.clear();
+ reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion));
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+
+ req = mock(SolrQueryRequest.class);
+ when(req.getParams()).thenReturn(reqParams);
+ // switch a single-valued field to a multi-valued field, which triggers a full rebuild of the "temp" collection
+ stream = new ContentStreamBase.FileStream(getFile("schema-designer/update-author-field.json"));
+ stream.setContentType(JSON_MIME);
+ when(req.getContentStreams()).thenReturn(Collections.singletonList(stream));
+
+ rsp = new SolrQueryResponse();
+
+ // PUT /schema-designer/update
+ schemaDesignerAPI.updateSchemaObject(req, rsp);
+ assertNotNull(rsp.getValues().get("field"));
+ rspData = rsp.getValues().toSolrParams();
+ schemaVersion = rspData.getInt(SCHEMA_VERSION_PARAM);
+
+ // add a new type
+ reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion));
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+
+ req = mock(SolrQueryRequest.class);
+ when(req.getParams()).thenReturn(reqParams);
+ stream = new ContentStreamBase.FileStream(getFile("schema-designer/add-new-type.json"));
+ stream.setContentType(JSON_MIME);
+ when(req.getContentStreams()).thenReturn(Collections.singletonList(stream));
+ rsp = new SolrQueryResponse();
+
+ // POST /schema-designer/add
+ schemaDesignerAPI.addSchemaObject(req, rsp);
+ final String expectedTypeName = "test_txt";
+ assertEquals(expectedTypeName, rsp.getValues().get("add-field-type"));
+ rspData = rsp.getValues().toSolrParams();
+ schemaVersion = rspData.getInt(SCHEMA_VERSION_PARAM);
+ assertNotNull(rsp.getValues().get("fieldTypes"));
+ List<SimpleOrderedMap<Object>> fieldTypes = (List<SimpleOrderedMap<Object>>) rsp.getValues().get("fieldTypes");
+ Optional<SimpleOrderedMap<Object>> expected = fieldTypes.stream().filter(m -> expectedTypeName.equals(m.get("name"))).findFirst();
+ assertTrue("New field type '" + expectedTypeName + "' not found in add type response!", expected.isPresent());
+
+ reqParams.clear();
+ reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion));
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+
+ req = mock(SolrQueryRequest.class);
+ when(req.getParams()).thenReturn(reqParams);
+ stream = new ContentStreamBase.FileStream(getFile("schema-designer/update-type.json"));
+ stream.setContentType(JSON_MIME);
+ when(req.getContentStreams()).thenReturn(Collections.singletonList(stream));
+ rsp = new SolrQueryResponse();
+
+ // POST /schema-designer/update
+ schemaDesignerAPI.updateSchemaObject(req, rsp);
+ rspData = rsp.getValues().toSolrParams();
+ schemaVersion = rspData.getInt(SCHEMA_VERSION_PARAM);
+
+ // query to see how the schema decisions impact retrieval / ranking
+ reqParams.clear();
+ reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion));
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+ reqParams.set(CommonParams.Q, "*:*");
+ req = mock(SolrQueryRequest.class);
+ when(req.getParams()).thenReturn(reqParams);
+ rsp = new SolrQueryResponse();
+
+ // GET /schema-designer/query
+ schemaDesignerAPI.query(req, rsp);
+ assertNotNull(rsp.getResponseHeader());
+ SolrDocumentList results = (SolrDocumentList) rsp.getResponse();
+ assertEquals(4, results.size());
+
+ // Download ZIP
+ reqParams.clear();
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+ req = mock(SolrQueryRequest.class);
+ when(req.getParams()).thenReturn(reqParams);
+ rsp = new SolrQueryResponse();
+ schemaDesignerAPI.downloadConfig(req, rsp);
+ assertNotNull(rsp.getValues().get(CONTENT));
+
+ // publish schema to a config set that can be used by real collections
+ reqParams.clear();
+ reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion));
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+
+ String collection = "test123";
+ reqParams.set(NEW_COLLECTION_PARAM, collection);
+ reqParams.set(INDEX_TO_COLLECTION_PARAM, true);
+ reqParams.set(RELOAD_COLLECTIONS_PARAM, true);
+ reqParams.set(CLEANUP_TEMP_PARAM, true);
+
+ rsp = new SolrQueryResponse();
+ schemaDesignerAPI.publish(req, rsp);
+
+ assertNotNull(cc.getZkController().zkStateReader.getCollection(collection));
+
+ // listCollectionsForConfig
+ reqParams.clear();
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+ rsp = new SolrQueryResponse();
+ schemaDesignerAPI.listCollectionsForConfig(req, rsp);
+ List<String> collections = (List<String>) rsp.getValues().get("collections");
+ assertNotNull(collections);
+ assertTrue(collections.contains(collection));
+
+ // verify temp designer objects were cleaned up during the publish operation ...
+ String mutableId = getMutableId(configSet);
+ assertFalse(cc.getZkController().getClusterState().hasCollection(mutableId));
+ SolrZkClient zkClient = cc.getZkController().getZkClient();
+ assertFalse(zkClient.exists("/configs/" + mutableId, true));
+
+ SolrQuery query = new SolrQuery("*:*");
+ query.setRows(0);
+ QueryResponse qr = cluster.getSolrClient().query(collection, query);
+ // this proves the docs were stored in the blob store too
+ assertEquals(4, qr.getResults().getNumFound());
+ }
+
+ @SuppressWarnings("unchecked")
+ public void testFieldUpdates() throws Exception {
+ String configSet = "fieldUpdates";
+
+ ModifiableSolrParams reqParams = new ModifiableSolrParams();
+
+ // Use the prep endpoint to prepare the new schema
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+ SolrQueryResponse rsp = new SolrQueryResponse();
+ SolrQueryRequest req = mock(SolrQueryRequest.class);
+ when(req.getParams()).thenReturn(reqParams);
+ schemaDesignerAPI.prepNewSchema(req, rsp);
+ assertNotNull(rsp.getValues().get(CONFIG_SET_PARAM));
+ assertNotNull(rsp.getValues().get(SCHEMA_VERSION_PARAM));
+ SolrParams rspData = rsp.getValues().toSolrParams();
+ int schemaVersion = rspData.getInt(SCHEMA_VERSION_PARAM);
+
+ // add our test field that we'll test various updates to
+ reqParams.clear();
+ reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(schemaVersion));
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+ req = mock(SolrQueryRequest.class);
+ when(req.getParams()).thenReturn(reqParams);
+ ContentStreamBase.FileStream stream = new ContentStreamBase.FileStream(getFile("schema-designer/add-new-field.json"));
+ stream.setContentType(JSON_MIME);
+ when(req.getContentStreams()).thenReturn(Collections.singletonList(stream));
+ rsp = new SolrQueryResponse();
+
+ // POST /schema-designer/add
+ schemaDesignerAPI.addSchemaObject(req, rsp);
+ assertNotNull(rsp.getValues().get("add-field"));
+
+ final String fieldName = "keywords";
+
+ Optional<SimpleOrderedMap<Object>> maybeField =
+ ((List<SimpleOrderedMap<Object>>) rsp.getValues().get("fields")).stream().filter(m -> fieldName.equals(m.get("name"))).findFirst();
+ assertTrue(maybeField.isPresent());
+ SimpleOrderedMap<Object> field = maybeField.get();
+ assertEquals(Boolean.FALSE, field.get("indexed"));
+ assertEquals(Boolean.FALSE, field.get("required"));
+ assertEquals(Boolean.TRUE, field.get("stored"));
+ assertEquals(Boolean.TRUE, field.get("docValues"));
+ assertEquals(Boolean.TRUE, field.get("useDocValuesAsStored"));
+ assertEquals(Boolean.FALSE, field.get("multiValued"));
+ assertEquals("string", field.get("type"));
+
+ String mutableId = getMutableId(configSet);
+ SchemaDesignerConfigSetHelper configSetHelper =
+ new SchemaDesignerConfigSetHelper(cc, SchemaDesignerAPI.newSchemaSuggester(cc));
+ ManagedIndexSchema schema = schemaDesignerAPI.loadLatestSchema(mutableId);
+
+ // make it required
+ Map<String, Object> updateField = Map.of("name", fieldName, "type", field.get("type"), "required", true);
+ configSetHelper.updateField(configSet, updateField, schema);
+
+ schema = schemaDesignerAPI.loadLatestSchema(mutableId);
+ SchemaField schemaField = schema.getField(fieldName);
+ assertTrue(schemaField.isRequired());
+
+ updateField = Map.of("name", fieldName, "type", field.get("type"), "required", false, "stored", false);
+ configSetHelper.updateField(configSet, updateField, schema);
+ schema = schemaDesignerAPI.loadLatestSchema(mutableId);
+ schemaField = schema.getField(fieldName);
+ assertFalse(schemaField.isRequired());
+ assertFalse(schemaField.stored());
+
+ updateField = Map.of("name", fieldName, "type", field.get("type"), "required", false, "stored", false, "multiValued", true);
+ configSetHelper.updateField(configSet, updateField, schema);
+ schema = schemaDesignerAPI.loadLatestSchema(mutableId);
+ schemaField = schema.getField(fieldName);
+ assertFalse(schemaField.isRequired());
+ assertFalse(schemaField.stored());
+ assertTrue(schemaField.multiValued());
+
+ updateField = Map.of("name", fieldName, "type", "strings", "copyDest", "_text_");
+ configSetHelper.updateField(configSet, updateField, schema);
+ schema = schemaDesignerAPI.loadLatestSchema(mutableId);
+ schemaField = schema.getField(fieldName);
+ assertTrue(schemaField.multiValued());
+ assertEquals("strings", schemaField.getType().getTypeName());
+ assertFalse(schemaField.isRequired());
+ assertTrue(schemaField.stored());
+ List<String> srcFields = schema.getCopySources("_text_");
+ assertEquals(Collections.singletonList(fieldName), srcFields);
+ }
+
+ @SuppressWarnings({"unchecked"})
+ public void testSchemaDiffEndpoint() throws Exception {
+ String configSet = "testDiff";
+
+ ModifiableSolrParams reqParams = new ModifiableSolrParams();
+
+ // Use the prep endpoint to prepare the new schema
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+ SolrQueryResponse rsp = new SolrQueryResponse();
+ SolrQueryRequest req = mock(SolrQueryRequest.class);
+ when(req.getParams()).thenReturn(reqParams);
+ schemaDesignerAPI.prepNewSchema(req, rsp);
+ assertNotNull(rsp.getValues().get(CONFIG_SET_PARAM));
+ assertNotNull(rsp.getValues().get(SCHEMA_VERSION_PARAM));
+
+ // publish schema to a config set that can be used by real collections
+ reqParams.clear();
+ reqParams.set(SCHEMA_VERSION_PARAM, String.valueOf(rsp.getValues().get(SCHEMA_VERSION_PARAM)));
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+
+ String collection = "diff456";
+ reqParams.set(NEW_COLLECTION_PARAM, collection);
+ reqParams.set(INDEX_TO_COLLECTION_PARAM, true);
+ reqParams.set(RELOAD_COLLECTIONS_PARAM, true);
+ reqParams.set(CLEANUP_TEMP_PARAM, true);
+
+ rsp = new SolrQueryResponse();
+ schemaDesignerAPI.publish(req, rsp);
+
+ assertNotNull(cc.getZkController().zkStateReader.getCollection(collection));
+
+ // Load the schema designer for the existing config set and make some changes to it
+ reqParams.clear();
+ reqParams.set(CONFIG_SET_PARAM, configSet);
+ reqParams.set(ENABLE_DYNAMIC_FIELDS_PARAM, "true");
+ reqParams.set(ENABLE_FIELD_GUESSING_PARAM, "false");
+ rsp = new SolrQueryResponse();
+ schemaDesignerAPI.analyze(req, rsp);
+
+ // Update id field to not use docValues
+ List<SimpleOrderedMap<Object>> fields = (List<SimpleOrderedMap<Object>>) rsp.getValues().get("fields");
+ SimpleOrderedMap<Object> idFieldMap = fields.stream().filter(field -> field.get("name").equals("id")).findFirst().get();
+ idFieldMap.remove("copyDest"); // Don't include copyDest as it is not a property of SchemaField
+ SimpleOrderedMap<Object> idFieldMapUpdated = idFieldMap.clone();
+ idFieldMapUpdated.setVal(idFieldMapUpdated.indexOf("docValues", 0), Boolean.FALSE);
+ idFieldMapUpdated.setVal(idFieldMapUpdated.indexOf("useDocValuesAsStored", 0), Boolean.FALSE);
+ idFieldMapUpdated.setVal(idFieldMapUpdated.indexOf("omitTermFreqAndPositions", 0), Boolean.FALSE);
+
+ SolrParams solrParams = idFieldMapUpdated.toSolrParams();
+ Map<String, Object> mapParams = solrParams.toMap(new HashMap<>());
+ mapParams.put("termVectors", Boolean.FALSE);
+ reqParams.set(SCHEMA_VERSION_PARAM, rsp.getValues().toSolrParams().getInt(SCHEMA_VERSION_PARAM));
+
+ ContentStreamBase.StringStream stringStream = new ContentStreamBase.StringStream(JSONUtil.toJSON(mapParams), JSON_MIME);
+ when(req.getContentStreams()).thenReturn(Collections.singletonList(stringStream));
+
+ rsp = new SolrQueryResponse();
+ schemaDesignerAPI.updateSchemaObject(req, rsp);
+
+ // Add a new field
+ Integer schemaVersion = rsp.getValues().toSolrParams().getInt(SCHEMA_VERSION_PARAM);
+ reqParams.set(SCHEMA_VERSION_PARAM, schemaVersion);
+ ContentStreamBase.FileStream fileStream = new ContentStreamBase.FileStream(getFile("schema-designer/add-new-field.json"));
+ fileStream.setContentType(JSON_MIME);
+ when(req.getContentStreams()).thenReturn(Collections.singletonList(fileStream));
+ rsp = new SolrQueryResponse();
+ // POST /schema-designer/add
+ schemaDesignerAPI.addSchemaObject(req, rsp);
+ assertNotNull(rsp.getValues().get("add-field"));
+
+ // Add a new field type
+ schemaVersion = rsp.getValues().toSolrParams().getInt(SCHEMA_VERSION_PARAM);
+ reqParams.set(SCHEMA_VERSION_PARAM, schemaVersion);
+ fileStream = new ContentStreamBase.FileStream(getFile("schema-designer/add-new-type.json"));
+ fileStream.setContentType(JSON_MIME);
+ when(req.getContentStreams()).thenReturn(Collections.singletonList(fileStream));
+ rsp = new SolrQueryResponse();
+ // POST /schema-designer/add
+ schemaDesignerAPI.addSchemaObject(req, rsp);
+ assertNotNull(rsp.getValues().get("add-field-type"));
+
+ // Let's do a diff now
+ rsp = new SolrQueryResponse();
+ schemaDesignerAPI.getSchemaDiff(req, rsp);
+
+ Map<String, Object> diff = (Map<String, Object>) rsp.getValues().get("diff");
+
+ // field asserts
+ assertNotNull(diff.get("fields"));
+ Map<String, Object> fieldsDiff = (Map<String, Object>) diff.get("fields");
+ assertNotNull(fieldsDiff.get("updated"));
+ Map<String, Object> mapDiff = (Map<String, Object>) fieldsDiff.get("updated");
+ assertEquals(
+ Arrays.asList(
+ ImmutableMap.of("omitTermFreqAndPositions", true, "useDocValuesAsStored", true, "docValues", true),
+ ImmutableMap.of("omitTermFreqAndPositions", false, "useDocValuesAsStored", false, "docValues", false)),
+ mapDiff.get("id"));
+ assertNotNull(fieldsDiff.get("added"));
+ Map<String, Object> fieldsAdded = (Map<String, Object>) fieldsDiff.get("added");
+ assertNotNull(fieldsAdded.get("keywords"));
+
+ // field type asserts
+ assertNotNull(diff.get("fieldTypes"));
+ Map<String, Object> fieldTypesDiff = (Map<String, Object>) diff.get("fieldTypes");
+ assertNotNull(fieldTypesDiff.get("added"));
+ Map<String, Object> fieldTypesAdded = (Map<String, Object>) fieldTypesDiff.get("added");
+ assertNotNull(fieldTypesAdded.get("test_txt"));
+ }
+
+ @SuppressWarnings("rawtypes")
+ protected void assertDesignerSettings(Map<String, Object> expected, NamedList actual) {
+ for (String expKey : expected.keySet()) {
+ Object expValue = expected.get(expKey);
+ assertEquals("Value for designer setting '" + expKey + "' not match expected!", expValue, actual.get(expKey));
+ }
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerConfigSetHelper.java b/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerConfigSetHelper.java
new file mode 100644
index 00000000000..dcbdfd6527a
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerConfigSetHelper.java
@@ -0,0 +1,359 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.designer;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.common.SolrInputDocument;
+import org.apache.solr.common.util.SimpleOrderedMap;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.core.SolrConfig;
+import org.apache.solr.schema.FieldType;
+import org.apache.solr.schema.ManagedIndexSchema;
+import org.apache.solr.schema.SchemaField;
+import org.apache.solr.util.ExternalPaths;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.apache.solr.common.util.Utils.toJavabin;
+import static org.apache.solr.handler.admin.ConfigSetsHandler.DEFAULT_CONFIGSET_NAME;
+import static org.apache.solr.handler.designer.SchemaDesignerAPI.getMutableId;
+import static org.apache.solr.schema.IndexSchema.NEST_PATH_FIELD_NAME;
+import static org.apache.solr.schema.IndexSchema.ROOT_FIELD_NAME;
+
+public class TestSchemaDesignerConfigSetHelper extends SolrCloudTestCase implements SchemaDesignerConstants {
+
+ private CoreContainer cc;
+ private SchemaDesignerConfigSetHelper helper;
+
+ @BeforeClass
+ public static void createCluster() throws Exception {
+ System.setProperty("managed.schema.mutable", "true");
+ configureCluster(1).addConfig(DEFAULT_CONFIGSET_NAME, new File(ExternalPaths.DEFAULT_CONFIGSET).toPath()).configure();
+ // SchemaDesignerConfigSetHelper depends on the blob store
+ CollectionAdminRequest.createCollection(BLOB_STORE_ID, 1, 1).process(cluster.getSolrClient());
+ cluster.waitForActiveCollection(BLOB_STORE_ID, 1, 1);
+ }
+
+ @AfterClass
+ public static void tearDownCluster() throws Exception {
+ if (cluster != null && cluster.getSolrClient() != null) {
+ cluster.deleteAllCollections();
+ cluster.deleteAllConfigSets();
+ }
+ }
+
+ @Before
+ public void setupTest() {
+ assumeWorkingMockito();
+ assertNotNull(cluster);
+ cc = cluster.getJettySolrRunner(0).getCoreContainer();
+ assertNotNull(cc);
+ helper = new SchemaDesignerConfigSetHelper(cc, SchemaDesignerAPI.newSchemaSuggester(cc));
+ }
+
+ @Test
+ public void testSetupMutable() throws Exception {
+ String configSet = "testSetupMutable";
+ String mutableId = getMutableId(configSet);
+ // create our test config by copying from _default
+ cc.getConfigSetService().copyConfig(DEFAULT_CONFIGSET_NAME, mutableId);
+
+ SolrConfig solrConfig = helper.loadSolrConfig(mutableId);
+ assertNotNull(solrConfig);
+ ManagedIndexSchema schema = helper.loadLatestSchema(solrConfig);
+ assertNotNull(schema);
+ int currentVersion = helper.getCurrentSchemaVersion(mutableId);
+ assertEquals(schema.getSchemaZkVersion(), currentVersion);
+
+ assertTrue(schema.persistManagedSchema(false));
+
+ // create the temp collection
+ helper.createCollection(mutableId, mutableId);
+
+ List<String> collsForConfig = helper.listCollectionsForConfig(configSet);
+ assertTrue(collsForConfig.isEmpty());
+
+ helper.reloadTempCollection(mutableId, true);
+
+ // version is incremented b/c we called persist on the schema above
+ int version = helper.getCurrentSchemaVersion(mutableId);
+ assertEquals(1, version);
+
+ helper.checkSchemaVersion(mutableId, version, -1);
+
+ schema = helper.syncLanguageSpecificObjectsAndFiles(configSet, schema, Collections.emptyList(), true, DEFAULT_CONFIGSET_NAME);
+ assertEquals(2, schema.getSchemaZkVersion());
+
+ byte[] zipped = helper.downloadAndZipConfigSet(mutableId);
+ assertTrue(zipped != null && zipped.length > 0);
+ }
+
+ @Test
+ public void testEnableDisableOptions() throws Exception {
+ String configSet = "testEnableDisableOptions";
+ String mutableId = getMutableId(configSet);
+ cc.getConfigSetService().copyConfig(DEFAULT_CONFIGSET_NAME, mutableId);
+
+ ManagedIndexSchema schema = helper.loadLatestSchema(helper.loadSolrConfig(mutableId));
+ assertEquals(schema.getSchemaZkVersion(), helper.getCurrentSchemaVersion(mutableId));
+
+ schema = helper.syncLanguageSpecificObjectsAndFiles(configSet, schema, Collections.singletonList("en"), true, DEFAULT_CONFIGSET_NAME);
+ assertNotNull(schema.getFieldTypeByName("text_en"));
+ assertNotNull(schema.getFieldOrNull("*_txt_en"));
+ assertNull(schema.getFieldTypeByName("text_fr"));
+
+ schema = helper.syncLanguageSpecificObjectsAndFiles(configSet, schema, Collections.singletonList("en"), false, DEFAULT_CONFIGSET_NAME);
+ assertNotNull(schema.getFieldTypeByName("text_en"));
+ assertNull(schema.getFieldOrNull("*_txt_en"));
+ assertNull(schema.getFieldTypeByName("text_fr"));
+
+ schema = helper.syncLanguageSpecificObjectsAndFiles(configSet, schema, Arrays.asList("en", "fr"), false, DEFAULT_CONFIGSET_NAME);
+ assertNotNull(schema.getFieldTypeByName("text_en"));
+ assertNull(schema.getFieldOrNull("*_txt_en"));
+ assertNotNull(schema.getFieldTypeByName("text_fr"));
+
+ schema = helper.syncLanguageSpecificObjectsAndFiles(configSet, schema, Arrays.asList("en", "fr"), true, DEFAULT_CONFIGSET_NAME);
+ assertNotNull(schema.getFieldTypeByName("text_en"));
+ assertNotNull(schema.getFieldOrNull("*_txt_en"));
+ assertTrue(cluster.getZkClient().exists(SchemaDesignerAPI.getConfigSetZkPath(mutableId, "lang/stopwords_en.txt"), true));
+ assertNotNull(schema.getFieldTypeByName("text_fr"));
+ assertNotNull(schema.getFieldOrNull("*_txt_fr"));
+ assertNull(schema.getFieldOrNull("*_txt_ga"));
+
+ // add a field that uses text_en and then try removing "en" from the lang set
+ helper.createCollection(mutableId, mutableId); // need to create field
+ Map<String, Object> addField = Map.of("name", "title", "type", "text_en");
+ String addedFieldName = helper.addSchemaObject(configSet, Collections.singletonMap("add-field", addField));
+ assertEquals("title", addedFieldName);
+
+ schema = helper.loadLatestSchema(helper.loadSolrConfig(mutableId));
+ assertNotNull(schema.getField("title"));
+
+ schema = helper.syncLanguageSpecificObjectsAndFiles(configSet, schema, Collections.singletonList("fr"), true, DEFAULT_CONFIGSET_NAME);
+ assertNotNull(schema.getFieldTypeByName("text_en")); // being used, so not removed
+ assertNotNull(schema.getFieldOrNull("*_txt_en"));
+ assertTrue(cluster.getZkClient().exists(SchemaDesignerAPI.getConfigSetZkPath(mutableId, "lang/stopwords_en.txt"), true));
+ assertNotNull(schema.getFieldTypeByName("text_fr"));
+ assertNotNull(schema.getFieldOrNull("*_txt_fr"));
+ assertNull(schema.getFieldOrNull("*_txt_ga"));
+
+ schema = helper.syncLanguageSpecificObjectsAndFiles(configSet, schema, Collections.emptyList(), true, DEFAULT_CONFIGSET_NAME);
+ assertNotNull(schema.getFieldTypeByName("text_en"));
+ assertNotNull(schema.getFieldOrNull("*_txt_en"));
+ assertNotNull(schema.getFieldTypeByName("text_fr"));
+ assertNotNull(schema.getFieldOrNull("*_txt_fr"));
+ assertNotNull(schema.getFieldTypeByName("text_ga"));
+ assertNotNull(schema.getFieldOrNull("*_txt_ga"));
+
+ schema = helper.syncLanguageSpecificObjectsAndFiles(configSet, schema, Collections.emptyList(), false, DEFAULT_CONFIGSET_NAME);
+ assertNotNull(schema.getFieldTypeByName("text_en"));
+ assertNull(schema.getFieldOrNull("*_txt_en"));
+ assertNotNull(schema.getFieldTypeByName("text_fr"));
+ assertNull(schema.getFieldOrNull("*_txt_fr"));
+ assertNotNull(schema.getFieldTypeByName("text_ga"));
+ assertNull(schema.getFieldOrNull("*_txt_ga"));
+
+ schema = helper.toggleNestedDocsFields(schema, true);
+ assertTrue(schema.hasExplicitField(ROOT_FIELD_NAME));
+ assertTrue(schema.hasExplicitField(NEST_PATH_FIELD_NAME));
+ schema = helper.toggleNestedDocsFields(schema, false);
+ assertFalse(schema.hasExplicitField(ROOT_FIELD_NAME));
+ assertFalse(schema.hasExplicitField(NEST_PATH_FIELD_NAME));
+ schema = helper.toggleNestedDocsFields(schema, true);
+ assertTrue(schema.hasExplicitField(ROOT_FIELD_NAME));
+ assertTrue(schema.hasExplicitField(NEST_PATH_FIELD_NAME));
+ }
+
+ @Test
+ public void testPersistSampleDocs() throws Exception {
+ String configSet = "testPersistSampleDocs";
+
+ SolrInputDocument doc = new SolrInputDocument();
+ doc.setField("id", "1");
+ doc.setField("author", "Ken Follet");
+ doc.setField("title", "The Pillars of the Earth");
+ doc.setField("series", "Kingsbridge Series");
+ doc.setField("pages", 809);
+ doc.setField("published_year", 1989);
+
+ helper.postDataToBlobStore(cluster.getSolrClient(), configSet + "_sample",
+ DefaultSampleDocumentsLoader.streamAsBytes(toJavabin(Collections.singletonList(doc))));
+
+ List<SolrInputDocument> docs = helper.getStoredSampleDocs(configSet);
+ assertTrue(docs != null && docs.size() == 1);
+ assertEquals("1", docs.get(0).getFieldValue("id"));
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testAnalyzeField() throws Exception {
+ String configSet = "testAnalyzeField";
+
+ String mutableId = getMutableId(configSet);
+ cc.getConfigSetService().copyConfig(DEFAULT_CONFIGSET_NAME, mutableId);
+
+ ManagedIndexSchema schema = helper.loadLatestSchema(helper.loadSolrConfig(mutableId));
+ assertEquals(schema.getSchemaZkVersion(), helper.getCurrentSchemaVersion(mutableId));
+ helper.createCollection(mutableId, mutableId);
+
+ Map<String, Object> addField = Map.of("name", "title", "type", "text_en");
+ String addedFieldName = helper.addSchemaObject(configSet, Collections.singletonMap("add-field", addField));
+ assertEquals("title", addedFieldName);
+
+ Map<String, Object> analysis = helper.analyzeField(configSet, "title", "The Pillars of the Earth");
+
+ Map<String, Object> title = (Map<String, Object>) ((Map<String, Object>) analysis.get("field_names")).get("title");
+ assertNotNull(title);
+ List<Object> index = (List<Object>) title.get("index");
+ assertNotNull(index);
+ assertFalse(index.isEmpty());
+ }
+
+ @Test
+ public void testCopyFieldUpdates() throws Exception {
+ String configSet = "testCopyFieldUpdates";
+ String mutableId = getMutableId(configSet);
+ cc.getConfigSetService().copyConfig(DEFAULT_CONFIGSET_NAME, mutableId);
+
+ ManagedIndexSchema schema = helper.loadLatestSchema(helper.loadSolrConfig(mutableId));
+ assertEquals(schema.getSchemaZkVersion(), helper.getCurrentSchemaVersion(mutableId));
+ helper.createCollection(mutableId, mutableId);
+
+ // add / update field
+ Map<String, Object> addField = Map.of("name", "author", "type", "string");
+ String addedFieldName = helper.addSchemaObject(configSet, Collections.singletonMap("add-field", addField));
+ assertEquals("author", addedFieldName);
+
+ helper.addSchemaObject(configSet,
+ Collections.singletonMap("add-field", Map.of("name", "_catch_all_", "type", "text_general")));
+
+ Map<String, Object> updateField = Map.of("name", "author", "type", "string", "copyDest", "_text_");
+ ManagedIndexSchema latest = helper.loadLatestSchema(helper.loadSolrConfig(mutableId));
+ latest.getField("_catch_all_");
+
+ Map<String, Object> resp = helper.updateSchemaObject(configSet, updateField, latest);
+ assertNotNull(resp);
+ assertEquals("field", resp.get("updateType"));
+ assertEquals(false, resp.get("rebuild"));
+
+ updateField = Map.of("name", "author", "type", "string", "copyDest", "_text_,_catch_all_");
+ latest = helper.loadLatestSchema(helper.loadSolrConfig(mutableId));
+ resp = helper.updateSchemaObject(configSet, updateField, latest);
+ assertNotNull(resp);
+ assertEquals("field", resp.get("updateType"));
+ assertEquals(false, resp.get("rebuild"));
+
+ latest = helper.loadLatestSchema(helper.loadSolrConfig(mutableId));
+ assertEquals(Collections.singletonList("author"), latest.getCopySources("_text_"));
+ assertEquals(Collections.singletonList("author"), latest.getCopySources("_catch_all_"));
+
+ updateField = Map.of("name", "author", "type", "string", "copyDest", "");
+ latest = helper.loadLatestSchema(helper.loadSolrConfig(mutableId));
+ resp = helper.updateSchemaObject(configSet, updateField, latest);
+ assertNotNull(resp);
+ assertEquals("field", resp.get("updateType"));
+ assertEquals(false, resp.get("rebuild"));
+ latest = helper.loadLatestSchema(helper.loadSolrConfig(mutableId));
+ assertTrue(latest.getCopySources("_text_").isEmpty());
+ assertTrue(latest.getCopySources("_catch_all_").isEmpty());
+ }
+
+ @Test
+ public void testAddUpdateObjects() throws Exception {
+ String configSet = "testAddUpdateObjects";
+ String mutableId = getMutableId(configSet);
+ cc.getConfigSetService().copyConfig(DEFAULT_CONFIGSET_NAME, mutableId);
+
+ ManagedIndexSchema schema = helper.loadLatestSchema(helper.loadSolrConfig(mutableId));
+ assertEquals(schema.getSchemaZkVersion(), helper.getCurrentSchemaVersion(mutableId));
+ helper.createCollection(mutableId, mutableId);
+
+ // add / update field
+ Map<String, Object> addField = Map.of("name", "author", "type", "string");
+ String addedFieldName = helper.addSchemaObject(configSet, Collections.singletonMap("add-field", addField));
+ assertEquals("author", addedFieldName);
+
+ Map<String, Object> updateField = Map.of("name", "author", "type", "string", "required", true);
+ ManagedIndexSchema latest = helper.loadLatestSchema(helper.loadSolrConfig(mutableId));
+ Map<String, Object> resp = helper.updateSchemaObject(configSet, updateField, latest);
+ assertNotNull(resp);
+ assertEquals("field", resp.get("updateType"));
+ assertEquals(false, resp.get("rebuild"));
+
+ SchemaField addedField = latest.getField("author");
+ assertFalse(addedField.multiValued());
+ assertTrue(addedField.hasDocValues());
+
+ // an update that requires a full-rebuild
+ updateField = Map.of("name", "author", "type", "string", "required", true, "docValues", true, "multiValued", true, "copyDest", "_text_");
+ resp = helper.updateSchemaObject(configSet, updateField, helper.loadLatestSchema(helper.loadSolrConfig(mutableId)));
+ assertNotNull(resp);
+ assertEquals("field", resp.get("updateType"));
+ assertEquals(true, resp.get("rebuild"));
+
+ // did the copy field update get applied?
+ latest = helper.loadLatestSchema(helper.loadSolrConfig(mutableId));
+ assertEquals(Collections.singletonList("author"), latest.getCopySources("_text_"));
+
+ // switch the author field type to strings
+ updateField = Map.of("name", "author", "type", "strings", "docValues", true, "copyDest", "_text_");
+ resp = helper.updateSchemaObject(configSet, updateField, helper.loadLatestSchema(helper.loadSolrConfig(mutableId)));
+ assertNotNull(resp);
+ assertEquals("field", resp.get("updateType"));
+ assertEquals(false, resp.get("rebuild")); // tricky, we didn't actually change the field to multiValue (it already was)
+
+ // add / update field type
+ Map<String, Object> addType = Map.of("name", "testType", "class", "solr.StrField", "docValues", true);
+ String addTypeName = helper.addSchemaObject(configSet, Collections.singletonMap("add-field-type", addType));
+ assertEquals("testType", addTypeName);
+
+ latest = helper.loadLatestSchema(helper.loadSolrConfig(mutableId));
+ FieldType addedType = latest.getFieldTypeByName(addTypeName);
+ assertNotNull(addedType);
+ SimpleOrderedMap<Object> props = addedType.getNamedPropertyValues(false);
+ assertTrue(props.getBooleanArg("docValues"));
+ assertFalse(addedType.isMultiValued());
+
+ Map<String, Object> updateType = Map.of("name", "testType", "class", "solr.StrField", "docValues", true, "multiValued", true);
+ resp = helper.updateSchemaObject(configSet, updateType, helper.loadLatestSchema(helper.loadSolrConfig(mutableId)));
+ assertNotNull(resp);
+ assertEquals("type", resp.get("updateType"));
+ assertEquals(true, resp.get("rebuild"));
+
+ // add / update dynamic field
+ Map<String, Object> addDynField = Map.of("name", "*_test", "type", "string");
+ String addedDynFieldName = helper.addSchemaObject(configSet, Collections.singletonMap("add-dynamic-field", addDynField));
+ assertEquals("*_test", addedDynFieldName);
+
+ // update the dynamic field
+ Map<String, Object> updateDynField = Map.of("name", "*_test", "type", "string", "docValues", false);
+ resp = helper.updateSchemaObject(configSet, updateDynField, helper.loadLatestSchema(helper.loadSolrConfig(mutableId)));
+ assertEquals("*_test", addedDynFieldName);
+ assertNotNull(resp);
+ assertEquals("dynamicField", resp.get("updateType"));
+ }
+}
diff --git a/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerSettingsDAO.java b/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerSettingsDAO.java
new file mode 100644
index 00000000000..f1a71d93a2a
--- /dev/null
+++ b/solr/core/src/test/org/apache/solr/handler/designer/TestSchemaDesignerSettingsDAO.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.solr.handler.designer;
+
+import java.io.File;
+import java.util.Collections;
+import java.util.Map;
+
+import org.apache.solr.client.solrj.SolrResponse;
+import org.apache.solr.client.solrj.request.CollectionAdminRequest;
+import org.apache.solr.cloud.SolrCloudTestCase;
+import org.apache.solr.core.CoreContainer;
+import org.apache.solr.handler.admin.CollectionsHandler;
+import org.apache.solr.util.ExternalPaths;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+import static org.apache.solr.handler.admin.ConfigSetsHandler.DEFAULT_CONFIGSET_NAME;
+
+public class TestSchemaDesignerSettingsDAO extends SolrCloudTestCase implements SchemaDesignerConstants {
+
+ private CoreContainer cc;
+
+ @BeforeClass
+ public static void createCluster() throws Exception {
+ System.setProperty("managed.schema.mutable", "true");
+ configureCluster(1).addConfig(DEFAULT_CONFIGSET_NAME, new File(ExternalPaths.DEFAULT_CONFIGSET).toPath()).configure();
+ }
+
+ @AfterClass
+ public static void tearDownCluster() throws Exception {
+ if (cluster != null && cluster.getSolrClient() != null) {
+ cluster.deleteAllCollections();
+ cluster.deleteAllConfigSets();
+ }
+ }
+
+ @Before
+ public void setupTest() {
+ assertNotNull(cluster);
+ cc = cluster.getJettySolrRunner(0).getCoreContainer();
+ assertNotNull(cc);
+ }
+
+ @Test
+ public void testDAO() throws Exception {
+ String collection = "testDAO";
+ String configSet = DEFAULT_CONFIGSET_NAME;
+
+ SolrResponse rsp =
+ CollectionAdminRequest.createCollection(collection, configSet, 1, 1).process(cluster.getSolrClient());
+ CollectionsHandler.waitForActiveCollection(collection, cc, rsp);
+
+ SchemaDesignerSettingsDAO dao = new SchemaDesignerSettingsDAO(cc);
+ SchemaDesignerSettings settings = dao.getSettings(configSet);
+ assertNotNull(settings);
+
+ Map<String, Object> expSettings = Map.of(
+ DESIGNER_KEY + ENABLE_DYNAMIC_FIELDS_PARAM, true,
+ AUTO_CREATE_FIELDS, true,
+ DESIGNER_KEY + ENABLE_NESTED_DOCS_PARAM, false,
+ DESIGNER_KEY + LANGUAGES_PARAM, Collections.emptyList());
+
+ assertDesignerSettings(expSettings, settings);
+ settings.setDisabled(false);
+ settings.setCopyFrom("foo");
+
+ assertTrue("updated settings should have changed in ZK", dao.persistIfChanged(configSet, settings));
+
+ settings = dao.getSettings(configSet);
+ assertNotNull(settings);
+
+ expSettings = Map.of(
+ DESIGNER_KEY + DISABLED, false,
+ DESIGNER_KEY + COPY_FROM_PARAM, "foo",
+ DESIGNER_KEY + ENABLE_DYNAMIC_FIELDS_PARAM, true,
+ AUTO_CREATE_FIELDS, true,
+ DESIGNER_KEY + ENABLE_NESTED_DOCS_PARAM, false,
+ DESIGNER_KEY + LANGUAGES_PARAM, Collections.emptyList());
+ assertDesignerSettings(expSettings, settings);
+ assertFalse("should not be disabled", dao.isDesignerDisabled(configSet));
+
+ settings.setDisabled(true);
+ settings.setCopyFrom("bar");
+ settings.setDynamicFieldsEnabled(false);
+ settings.setNestedDocsEnabled(true);
+ settings.setFieldGuessingEnabled(false);
+ settings.setLanguages(Collections.singletonList("en"));
+
+ assertTrue("updated settings should have changed in ZK", dao.persistIfChanged(configSet, settings));
+ settings = dao.getSettings(configSet);
+ assertNotNull(settings);
+
+ expSettings = Map.of(
+ DESIGNER_KEY + DISABLED, true,
+ DESIGNER_KEY + COPY_FROM_PARAM, "bar",
+ DESIGNER_KEY + ENABLE_DYNAMIC_FIELDS_PARAM, false,
+ AUTO_CREATE_FIELDS, false,
+ DESIGNER_KEY + ENABLE_NESTED_DOCS_PARAM, true,
+ DESIGNER_KEY + LANGUAGES_PARAM, Collections.singletonList("en"));
+ assertDesignerSettings(expSettings, settings);
+ assertTrue("should be disabled", dao.isDesignerDisabled(configSet));
+ }
+
+ protected void assertDesignerSettings(Map<String, Object> expectedMap, SchemaDesignerSettings actual) {
+ assertEquals(new SchemaDesignerSettings(expectedMap), actual);
+ }
+}
diff --git a/solr/webapp/web/css/angular/menu.css b/solr/webapp/web/css/angular/menu.css
index 1bafdff5cfd..c0d09ec69e0 100644
--- a/solr/webapp/web/css/angular/menu.css
+++ b/solr/webapp/web/css/angular/menu.css
@@ -271,6 +271,8 @@ limitations under the License.
#menu #cloud.global .zkstatus a { background-image: url( ../../img/ico/node-leader.png ); }
#menu #cloud.global .graph a { background-image: url( ../../img/ico/molecule.png ); }
+#menu #schema-designer.global p a { background-image: url( ../../img/ico/book-open-text.png ); }
+
.sub-menu .ping.error a
{
diff --git a/solr/webapp/web/css/angular/schema-designer.css b/solr/webapp/web/css/angular/schema-designer.css
new file mode 100644
index 00000000000..eb281d84f01
--- /dev/null
+++ b/solr/webapp/web/css/angular/schema-designer.css
@@ -0,0 +1,1603 @@
+/*
+
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+
+*/
+#content #designer
+{
+ position: relative;
+}
+
+#content #designer .loader
+{
+ background-position: 0 50%;
+ padding-left: 21px;
+}
+
+#content #designer #designer-top
+{
+ padding: 0px;
+}
+
+#content #designer #tree-content
+{
+ overflow: auto;
+ float: left;
+ height: 520px;
+ width: 275px;
+ padding: 3px;
+}
+#content #designer #tree-node-content
+{
+ height: 560px;
+ margin: 2px;
+ overflow: hidden;
+}
+
+#content #designer .designer-left
+{
+ float: left;
+ width: 380px;
+ min-width: 380px; /* so the text area doesn't bleed over to the right */
+ padding: 7px;
+}
+
+#content #designer .designer-middle
+{
+ float: left;
+ width: 48%;
+ padding: 7px;
+ height: 585px;
+}
+
+#content #designer .designer-right
+{
+ padding: 7px;
+ overflow: hidden;
+}
+
+#content #designer .query-tester-left
+{
+ float: left;
+ width: 30%;
+ padding: 7px;
+}
+
+#content #designer .query-tester-right
+{
+ padding: 7px;
+ overflow: auto;
+}
+
+#content #designer .data
+{
+ padding-bottom: 12px;
+ overflow: hidden;
+}
+
+#content #designer .data:hover
+{
+ overflow-x: auto;
+}
+
+#content #designer .data li
+{
+ padding-top: 3px;
+ padding-bottom: 3px;
+}
+
+#content #designer .data li dt
+{
+ float: left;
+ white-space: nowrap;
+ width: 20%;
+}
+
+#content #designer .data li dd
+{
+ float: right;
+ text-overflow: ellipsis;
+ white-space: nowrap;
+ width: 80%
+}
+
+#content #designer .data li dd.odd
+{
+ background-color: #f0f0f0;
+}
+
+#content #designer .data dt span
+{
+ background-position: 1px 50%;
+ display: block;
+ padding-left: 22px;
+ font-weight: bold;
+ text-align: left;
+}
+
+#content #designer #sample-docs h2 { background-image: url( ../../img/ico/document-text.png ); }
+#content #designer #query-tester h2 { background-image: url( ../../img/ico/magnifier.png ); }
+#content #designer #query-results h2 { background-image: url( ../../img/ico/documents-stack.png ); }
+#content #designer #schema-editor h2 { background-image: url( ../../img/ico/book-open-text.png ); }
+#content #designer #analysis h2 { background-image: url( ../../img/ico/property.png ); }
+
+#content #designer #form
+{
+ margin-bottom: 10px;
+}
+
+#content #designer #form label
+{
+ cursor: pointer;
+ display: block;
+ margin-top: 5px;
+}
+
+#content #designer #field-form-left
+{
+ float: left;
+ width: 45%;
+ padding: 7px;
+}
+
+#content #designer #field-form-right
+{
+ margin-top: 8px;
+ padding: 7px;
+ overflow: hidden;
+}
+
+#content #designer .field-form
+{
+ display: block;
+ padding: 5px;
+}
+
+#content #designer .field-form-right
+{
+ display: block;
+ padding: 5px;
+}
+
+#content #designer .field-form-checkbox {
+ margin-top: 10px;
+ margin-left: 130px;
+ display: block;
+}
+
+#content #designer .field-form-right label
+{
+ float: left;
+ padding-top: 3px;
+ padding-bottom: 3px;
+ text-align: right;
+ width: 120px;
+ margin-right: 6px;
+}
+
+#content #designer #languages
+{
+ width: 125px;
+}
+
+#content #designer .checkbox
+{
+ text-align: left;
+ padding-left: 3px;
+}
+
+#content #designer .actions
+{
+ margin-bottom: 5px;
+}
+
+#content #designer .actions form .error
+{
+ background-image: url( ../../img/ico/cross-button.png );
+ background-position: 22% 1px;
+ color: #c00;
+ font-weight: bold;
+}
+
+#content #designer .actions form p
+{
+ padding-bottom: 8px;
+}
+
+#content #designer .actions form label
+{
+ float: left;
+ padding-top: 3px;
+ padding-bottom: 3px;
+ text-align: right;
+ width: 25%;
+ margin-right: 6px;
+}
+
+#content #designer .actions #download-button span
+{
+ background-image: url( ../../img/ico/download-cloud.png );
+}
+
+#content #designer #schema-selector .left
+{
+ float: left;
+ width: 405px;
+}
+
+#content #designer #schema-selector .middle
+{
+ float: left;
+ width: 48%;
+}
+
+#content #designer #schema-selector .right
+{
+ text-align: right;
+ overflow: hidden;
+ margin-right: 10px;
+}
+
+#content #designer #schema-selector #apply
+{
+ float: right;
+ text-align: right;
+ width: 300px;
+}
+
+#content #designer #schema-selector a
+{
+ padding: 0;
+ padding-left: 8px;
+}
+
+#content #designer #schema-selector select
+{
+ width: 150px;
+}
+
+#content #designer .actions form input,
+#content #designer .actions form select,
+#content #designer .actions form .buttons,
+#content #designer .actions form .note span
+{
+ float: right;
+ width: 71%;
+}
+
+#content #designer .actions form .note span
+{
+ padding-left: 3px;
+ padding-right: 3px;
+}
+
+#content #designer .actions form .buttons
+{
+ padding-top: 10px;
+}
+
+#content #designer .actions form button.submit
+{
+ margin-right: 20px;
+}
+
+#content #designer .actions form button.submit span
+{
+ background-image: url( ../../img/ico/tick.png );
+}
+
+#content #designer .actions form button.reset span
+{
+ background-image: url( ../../img/ico/cross.png );
+}
+
+#content #designer .actions #add span
+{
+ background-image: url( ../../img/ico/plus-button.png );
+}
+
+#content #designer .actions #unload
+{
+ margin-right: 20px;
+}
+
+#content #designer .actions #unload span
+{
+ background-image: url( ../../img/ico/cross.png );
+}
+
+#content #designer .actions #reload span
+{
+ background-image: url( ../../img/ico/arrow-circle.png );
+}
+
+#content #designer .actions #rename span
+{
+ background-image: url( ../../img/ico/ui-text-field-select.png );
+}
+
+#content #designer .actions #swap span
+{
+ background-image: url( ../../img/ico/arrow-switch.png );
+}
+
+#content #designer .actions div.action
+{
+ z-index: 100;
+ background-color: #fff;
+ border: 1px solid #f0f0f0;
+ box-shadow: 5px 5px 10px #c0c0c0;
+ -moz-box-shadow: 5px 5px 10px #c0c0c0;
+ -webkit-box-shadow: 5px 5px 10px #c0c0c0;
+ position: absolute;
+ left: 0px;
+ top: 36px;
+ padding: 10px;
+ width: 320px;
+}
+
+/* Sample Documents */
+
+#content #designer #analyze-buttons{
+ float: right;
+ margin-right: 12px;
+}
+
+#content #designer #query-buttons{
+ margin-top: 20px;
+ float: right;
+ margin-right: 8px;
+}
+
+#content #designer #queryButton span
+{
+ background-image: url( ../../img/ico/magnifier.png );
+}
+
+#content #designer #analyze span
+{
+ background-image: url( ../../img/ico/gear.png );
+}
+
+#content #designer #document-container
+{
+ padding: 5px;
+}
+
+#content #designer #file-upload
+{
+ margin-bottom: 20px;
+}
+
+/* Section header bars */
+
+#content #designer .bar-desc
+{
+ color: #4D4D4D;
+ font-weight: normal;
+ margin-left: 10px;
+ white-space: pre;
+}
+
+#content #designer .bar-holder
+{
+ box-shadow: 5px 5px 10px #c0c0c0;
+ -moz-box-shadow: 5px 5px 10px #c0c0c0;
+ -webkit-box-shadow: 5px 5px 10px #c0c0c0;
+ height: 35px;
+}
+
+#content #designer .bar-holder .bar
+{
+ height: 100%;
+ position: relative;
+}
+
+#content #designer .bar-holder div .val
+{
+ border-right: 1px solid #f00;
+ display: block;
+ padding-right: 5px;
+ position: absolute;
+ right: 0;
+ top: 35px;
+ white-space: nowrap;
+}
+
+#content #designer .bar-holder .bar-max.bar
+{
+ background-color: #f0f0f0;
+}
+
+#content #designer .bar-holder .bar-max.val
+{
+ border-color: #f0f0f0;
+ color: #8D8D8D;
+}
+
+#content #designer .bar-holder .bar-total.bar
+{
+ background-color: #c0c0c0;
+}
+
+#content #designer .bar-holder .bar-total.val
+{
+ border-color: #c0c0c0;
+ color: #4D4D4D;
+}
+
+#content #designer .bar-holder .bar-used.bar
+{
+ background-color: #969696;
+}
+
+#content #designer .bar-holder .bar-used.val
+{
+ border-color: #969696;
+ color: #969696;
+}
+
+#content #designer .bar-holder.bar-lvl-2 .bar-max.val { padding-top: 25px; }
+#content #designer .bar-holder.bar-lvl-2 .bar-total.val { padding-top: 5px; }
+#content #designer .bar-holder.bar-lvl-2 { margin-bottom: 45px; }
+
+#content #designer .bar-holder.bar-lvl-3 .bar-max.val { padding-top: 45px; }
+#content #designer .bar-holder.bar-lvl-3 .bar-total.val { padding-top: 25px; }
+#content #designer .bar-holder.bar-lvl-3 .bar-used.val { padding-top: 5px; }
+#content #designer .bar-holder.bar-lvl-3 { margin-bottom: 65px; }
+
+#content #designer .analyzer,
+#content #designer .analyzer li,
+#content #designer .analyzer ul,
+#content #designer .analyzer ul li
+{
+}
+
+#content #designer .analyzer p,
+#content #designer .analyzer dl
+{
+ float: left;
+}
+
+#content #designer .analyzer p
+{
+ margin-right: 5px;
+ text-align: left;
+ width: 100px;
+ white-space: pre;
+}
+
+#content #designer .analyzer p a
+{
+ cursor: auto;
+}
+
+#content #designer .analyzer p a.analysis
+{
+ cursor: pointer;
+ display: block;
+}
+
+#content #designer .analyzer p a.analysis span
+{
+ background-image: url( ../../img/ico/question-white.png );
+ background-position: 0 50%;
+ padding-left: 21px;
+}
+
+#content #designer .analyzer p a.analysis:hover span
+{
+ background-image: url( ../../img/ico/question.png );
+ color: #008;
+}
+
+#content #designer .analyzer a
+{
+ cursor: pointer;
+}
+
+#content #designer .file-link
+{
+ cursor: pointer;
+ color: #003eff;
+ text-decoration-line: underline;
+ text-decoration-color: #003eff;
+}
+
+#content #designer .analyzer .toggle
+{
+ background-image: url( ../../img/ico/chevron-small-expand.png );
+ background-position: 100% 50%;
+ cursor: pointer;
+ display: block;
+ padding-right: 21px;
+}
+
+#content #designer .analyzer .open .toggle
+{
+ background-image: url( ../../img/ico/chevron-small.png );
+}
+
+#content #designer .analyzer li
+{
+ border-top: 1px solid #f0f0f0;
+ margin-top: 10px;
+ padding-top: 10px;
+ margin-left: 10px;
+}
+
+#content #designer .analyzer ul
+{
+ clear: left;
+ margin-left: 25px;
+ padding-top: 5px;
+}
+
+#content #designer .analyzer .open ul
+{
+ display: block;
+}
+
+#content #designer .analyzer ul li
+{
+ border-top: 1px solid #f8f8f8;
+ margin-top: 5px;
+ padding-top: 5px;
+}
+
+#content #designer .analyzer ul p
+{
+ color: #4D4D4D;
+ margin-right: 5px;
+ text-align: left;
+ width: 80px;
+}
+
+#content #designer .analyzer ul dd
+{
+ margin-left: 10px;
+}
+
+#content #designer .analyzer ul dd
+{
+ background-image: url( ../../img/ico/document-list.png );
+ background-position: 0 50%;
+ color: #4D4D4D;
+ padding-left: 21px;
+}
+
+#content #designer .analyzer ul dd.ico-0
+{
+ background-image: url( ../../img/ico/slash.png );
+}
+
+#content #designer .analyzer ul dd.ico-1
+{
+ background-image: url( ../../img/ico/tick.png );
+}
+
+#content #designer #field-top
+{
+ width: 100%;
+}
+
+#content #designer #field-top-left
+{
+ float: left;
+ width: 240px;
+ padding: 7px;
+}
+
+#content #designer #field-top-right
+{
+ display: contents;
+}
+
+#content #designer #tree-node-content form button.submit
+{
+ margin-left: 140px;
+}
+
+#content #designer #tree-node-content form button.submit span
+{
+ background-image: url( ../../img/ico/tick.png );
+}
+
+#content #designer #field-bottom
+{
+ float: left;
+ width: 100%;
+}
+
+#content #designer #schema-actions {
+ float: left;
+ width: 99%;
+}
+
+#content #designer .schema-actions #addField { margin-right: 15px; }
+#content #designer .schema-actions #addFieldType { margin-right: 15px; }
+
+#content #designer .schema-actions #addField span { background-image: url( ../../img/ico/document-list.png ); }
+#content #designer .schema-actions #addCopyField span { background-image: url( ../../img/ico/document-import.png ); }
+#content #designer .schema-actions #addFieldType span { background-image: url( ../../img/ico/document-text.png ); }
+
+#content #designer #updateSelected { float: right; margin-top:5px; }
+#content #designer #updateSelected span { background-image: url( ../../img/ico/tick.png ); }
+#content #designer #deleteSelected { margin-right: 0px; }
+#content #designer #deleteSelected span { background-image: url( ../../img/ico/cross-button.png ); }
+
+#content #designer #updateStatusMessage { color: darkgreen; float: right; margin-top: 5px; background-image: url( ../../img/ico/tick.png ); }
+#content #designer #updateStatusMessage span { margin-left: 18px; }
+#content #designer .working { font-style: italic; color: black !important; float: right; margin-top: 5px; background-image: url( ../../img/loader.gif ) !important; }
+
+#content #designer .schema-actions div.action
+{
+ z-index: 100;
+ background-color: #fff;
+ border: 1px solid #f0f0f0;
+ box-shadow: 5px 5px 10px #c0c0c0;
+ -moz-box-shadow: 5px 5px 10px #c0c0c0;
+ -webkit-box-shadow: 5px 5px 10px #c0c0c0;
+ position: absolute;
+ left: 448px;
+ top: 31px;
+ padding: 10px;
+ width: 350px;
+}
+
+#content #designer .schema-actions div.copyfield
+{
+ z-index: 100;
+ background-color: #fff;
+ border: 1px solid #f0f0f0;
+ box-shadow: 5px 5px 10px #c0c0c0;
+ -moz-box-shadow: 5px 5px 10px #c0c0c0;
+ -webkit-box-shadow: 5px 5px 10px #c0c0c0;
+ position: absolute;
+ left: 700px;
+ top: 31px;
+ padding: 10px;
+ width: 350px;
+}
+
+
+#content #designer .schema-actions p
+{
+ padding-bottom: 8px;
+}
+
+#content #designer .schema-actions label
+{
+ margin-right: 6px;
+ float: left;
+ padding-top: 3px;
+ padding-bottom: 3px;
+ text-align: right;
+ width: 25%;
+}
+
+#content #designer .schema-actions input,
+#content #designer .schema-actions select,
+#content #designer .schema-actions .buttons,
+#content #designer .schema-actions .note span
+{
+ float: right;
+ width: 71%;
+}
+
+#content #designer #add-errors
+{
+ float: left;
+ margin-top:10px;
+ margin-bottom: 10px;
+ min-height: 30px;
+}
+
+#content #designer #add-errors .error
+{
+ background-image: url( ../../img/ico/cross-button.png );
+ background-position: 10px 1px;
+ color: #c00;
+ font-weight: bold;
+}
+
+#content #designer #add-errors .error span
+{
+ margin-left: 30px;
+}
+
+#content #designer .schema-actions .chosen-container
+{
+ width: 71% !important;
+}
+
+#content #designer .chosen-container {
+ width: 100%;
+}
+#content #designer .chosen-drop input,
+#content #designer .chosen-results {
+ width: 100% !important;
+}
+
+#content #designer .schema-actions label.checkbox {
+ margin-left: 27%;
+ text-align: left;
+ width: 73%;
+ padding: 0px;
+ margin-top: 0px;
+}
+#content #designer .schema-actions .checkbox input {
+ float: none;
+ width: auto;
+}
+
+#content #designer .add_showhide {
+ background-image: url( ../../img/ico/chevron-small-expand.png );
+ background-position: 100% 50%;
+ cursor: pointer;
+ padding-right: 21px;
+}
+
+#content #designer .add_showhide.open {
+ background-image: url( ../../img/ico/chevron-small.png );
+}
+
+#content #designer #schema-actions button span
+{
+ background-image: url( ../../img/ico/cross.png );
+}
+#content #designer #schema-actions button.submit span
+{
+ background-image: url( ../../img/ico/tick.png );
+}
+
+#content #designer #field-analysis
+{
+ margin-bottom: 0;
+ overflow: auto;
+ max-height: 135px;
+}
+
+#content #designer #analysis #field-analysis-holder
+{
+ width: 100%;
+ padding: 5px;
+ max-height: 480px;
+ overflow: auto;
+}
+
+#content #designer #analysis #field-analysis-holder #field-analysis-form
+{
+ width: 100%;
+ padding: 5px;
+}
+
+#content #designer #analysis .settings
+{
+ clear: both;
+ padding: 5px;
+}
+
+#content #designer #analysis select
+{
+ width: 200px;
+}
+
+#content #designer #analysis .settings select.loader
+{
+ background-position: 3px 50%;
+ padding-left: 21px;
+}
+
+#content #designer #analysis .settings select optgroup
+{
+ font-style: normal;
+ padding: 5px;
+}
+
+#content #designer #analysis .settings select option
+{
+ padding-left: 10px;
+}
+
+#content #designer #analysis .settings label
+{
+ display: block;
+ width: 150px;
+}
+
+#content #designer #analysis .settings textarea
+{
+ display: block;
+}
+
+#content #designer #analysis .settings
+{
+ float: left;
+ margin-right: 0.5%;
+}
+
+#content #designer #analysis .analysis-error
+{
+ background-color: #f00;
+ background-image: url( ../../img/ico/construction.png );
+ background-position: 10px 50%;
+ color: #fff;
+ font-weight: bold;
+ margin-bottom: 20px;
+ padding: 10px;
+ padding-left: 35px;
+}
+
+#content #designer #analysis .analysis-error .head a
+{
+ color: #fff;
+ cursor: auto;
+}
+
+#content #designer #analysis #analysis-result
+{
+ width: 100%;
+ float: left;
+ overflow: auto;
+}
+
+#content #designer #analysis #analysis-result .index
+{
+ background-color: #fff;
+ padding-top: 20px;
+}
+
+#content #designer #analysis #analysis-result table
+{
+ border-collapse: collapse;
+}
+
+#content #designer #analysis #analysis-result td
+{
+ vertical-align: top;
+ white-space: nowrap;
+}
+
+#content #designer #analysis #analysis-result td.part.analyzer div,
+#content #designer #analysis #analysis-result td.part.spacer .holder,
+#content #designer #analysis #analysis-result td td td
+{
+ padding-top: 1px;
+ padding-bottom: 1px;
+}
+
+#content #designer #analysis #analysis-result.verbose_output td.legend
+{
+ display: table-cell;
+}
+
+#content #designer #analysis #analysis-result.verbose_output td.data tr.verbose_output
+{
+ display: table-row;
+}
+
+#content #designer #analysis #analysis-result .match
+{
+ background-color: #F0D9C3;
+}
+
+#content #designer #analysis #analysis-result td.part
+{
+ padding-bottom: 10px;
+}
+
+#content #designer #analysis #analysis-result td.part.analyzer div
+{
+ border-right: 1px solid #f0f0f0;
+ padding-right: 10px;
+}
+
+#content #designer #analysis #analysis-result td.part.analyzer abbr
+{
+ color: #4D4D4D;
+}
+
+#content #designer #analysis #analysis-result td.part.legend .holder,
+#content #designer #analysis #analysis-result td.part.data .holder
+{
+ padding-left: 10px;
+ padding-right: 10px;
+ border-right: 1px solid #c0c0c0;
+}
+
+#content #designer #analysis #analysis-result td.part.legend td
+{
+ color: #4D4D4D;
+}
+
+#content #designer #analysis #analysis-result td.part.legend .holder
+{
+ border-right-color: #f0f0f0;
+}
+
+#content #designer #analysis #analysis-result td.part.data:last-child .holder
+{
+ padding-right: 0;
+ border-right: 0;
+}
+
+#content #designer #analysis #analysis-result td.details
+{
+ padding-left: 10px;
+ padding-right: 10px;
+ border-left: 1px solid #f0f0f0;
+ border-right: 1px solid #f0f0f0;
+}
+
+#content #designer #analysis #analysis-result td.details:first-child
+{
+ padding-left: 0;
+ border-left: 0;
+}
+
+#content #designer #analysis #analysis-result td.details:last-child
+{
+ padding-right: 0;
+ border-right: 0;
+}
+
+#content #designer #analysis #analysis-result td.details tr.empty td
+{
+ color: #f0f0f0;
+}
+
+#content #designer #analysis #analysis-result td.details tr.raw_bytes td
+{
+ letter-spacing: -1px;
+}
+
+#content #designer #analysis #analysis-result .part table table td
+{
+ border-top: 1px solid #f0f0f0;
+}
+
+#content #designer #analysis #analysis-result .part table table tr:first-child td
+{
+ border-top: 0;
+}
+
+#content #designer #analysis #field-analysis h2 { background-image: url( ../../img/ico/receipt.png ); }
+#content #designer #analysis .analysis-result h2 { background-image: url( ../../img/ico/receipt-invoice.png ); }
+
+#content #designer #schema-root-details
+{
+ min-height: 360px;
+ padding: 7px;
+}
+
+#content #designer #schema-root-details #unique_key
+{
+ width: 150px;
+}
+
+#content #designer #schema-root-details .note span
+{
+ float: right;
+ padding-left: 18px;
+}
+
+#content #designer #schema-root-details .error
+{
+ background-image: url( ../../img/ico/cross-button.png );
+ background-position: 0px 2px;
+ color: #c00;
+ font-weight: bold;
+}
+
+/* diff dialog */
+#content #designer div.diff
+{
+ z-index: 100;
+ position: absolute;
+ background-color: #fff;
+ border: 1px solid #f0f0f0;
+ box-shadow: 5px 5px 10px #c0c0c0;
+ -moz-box-shadow: 5px 5px 10px #c0c0c0;
+ -webkit-box-shadow: 5px 5px 10px #c0c0c0;
+ top: 31px;
+ left: 650px;
+ padding: 10px;
+ width: 600px;
+ overflow: auto;
+ min-height: 370px;
+ max-height: 600px;
+}
+
+#content #designer table.diff {
+ font-family: arial, sans-serif;
+ border-collapse: collapse;
+ width: 100%;
+ display: inline-block;
+ overflow: auto;
+}
+
+#content #designer table.diff th {
+ font-family: arial, sans-serif;
+ font-weight: bold;
+ min-width: 100px;
+}
+
+#content #designer p.diff-text {
+ margin-top: 15px;
+ margin-bottom: 5px;
+ font-size: medium;
+ font-weight: bold;
+}
+
+#content #designer p.diff-field-name {
+ margin-top: 5px;
+ margin-bottom: 15px;
+ font-size: small;
+ font-weight: bold;
+}
+
+#content #designer li.element
+{
+ border: 1px solid #f0f0f0;
+ box-shadow: 5px 5px 10px #c0c0c0;
+ -moz-box-shadow: 5px 5px 10px #c0c0c0;
+ -webkit-box-shadow: 5px 5px 10px #c0c0c0;
+ margin-top: 5px;
+ margin-bottom: 15px;
+}
+
+#content #designer .actions .diff-buttons {
+ margin-right: 15px;
+ float: right;
+ width: 48%;
+}
+
+#content #designer p.old-value {
+ color: indianred;
+ text-decoration: line-through;
+}
+
+#content #designer p.rem-value {
+ color: indianred;
+}
+
+#content #designer p.new-value {
+ color: green;
+}
+
+/* publish dialog */
+#content #designer .actions div.publish
+{
+ z-index: 100;
+ position: absolute;
+ background-color: #fff;
+ border: 1px solid #f0f0f0;
+ box-shadow: 5px 5px 10px #c0c0c0;
+ -moz-box-shadow: 5px 5px 10px #c0c0c0;
+ -webkit-box-shadow: 5px 5px 10px #c0c0c0;
+ top: 31px;
+ left: 1150px;
+ padding: 10px;
+ width: 380px;
+ height: 430px;
+}
+
+#content #designer .actions .publish-button {
+ margin-right: 15px;
+}
+
+#content #designer .actions .publish-button span
+{
+ background-image: url( ../../img/ico/disk-black.png );
+}
+
+#content #designer .actions .diff-button {
+ margin-right: 15px;
+}
+
+#content #designer .actions .diff-button span {
+ background-image: url( ../../img/ico/document-convert.png );
+}
+
+#content #designer .actions button.reset span
+{
+ background-image: url( ../../img/ico/cross.png );
+}
+
+#content #designer #publish-dialog-note {
+ margin-bottom: 15px;
+}
+
+#content #designer #publish-new-coll {
+ margin-top: 15px;
+ margin-bottom: 10px;
+}
+
+#content #designer .publish-buttons
+{
+ margin-top: 20px;
+ float: right;
+ width: 48%;
+}
+
+#content #designer #publish-affected {
+ overflow: auto;
+ height: 60px;
+}
+
+#content #designer #reload-form {
+ height: 20px;
+}
+
+#content #designer #file-node-text {
+ display:block;
+}
+
+#content #designer #file-node {
+ padding: 7px;
+}
+
+#content #designer #save-file span {
+ background-image: url( ../../img/ico/disk-black.png );
+}
+
+#content #designer #results-tree-left {
+ float: left;
+ width: 330px;
+ overflow: auto;
+ height: 350px;
+}
+
+#content #designer #results-tree-right {
+ max-width: 800px;
+ float: left;
+ overflow: auto;
+ height: 350px;
+}
+
+#content #designer #save-file-buttons {
+ margin-top: 15px;
+ margin-right: 5px;
+ display: block;
+ float: right;
+}
+
+#content #designer #copy_dest {
+ width: 300px;
+}
+
+#content #designer .hit-field {
+ padding: 3px;
+ width: 120px;
+}
+
+#content #designer .hit-value {
+ padding: 3px;
+ word-wrap: break-word;
+ max-width: 700px;
+}
+
+#content #designer #leaf-node .error
+{
+ background-image: url( ../../img/ico/cross-button.png );
+ background-position: 0px 2px;
+ color: #c00;
+ font-weight: bold;
+}
+
+#content #designer #leaf-node .error span
+{
+ padding-left: 18px;
+}
+
+#content #designer #sample-docs .note
+{
+ background-image: url( ../../img/ico/information.png );
+ background-position: 0px 2px;
+ color: #008;
+ font-weight: bold;
+}
+
+#content #designer #confirm-dialog
+{
+ width: 350px;
+ height: 250px;
+}
+
+#content #designer #confirm-dialog .warn span
+{
+ padding-left: 18px;
+}
+
+#content #designer #confirm-dialog .warn
+{
+ background-image: url( ../../img/ico/exclamation-button.png );
+ background-position: 0px 2px;
+ font-weight: bold;
+}
+
+#content #designer #confirm-dialog #confirm-edit span {
+ background-image: url( ../../img/ico/tick.png );
+}
+
+#content #designer #confirm-dialog #confirm-colls {
+ margin-top: 15px;
+ margin-bottom: 15px;
+}
+
+
+#content #designer #sample-docs .note span
+{
+ padding-left: 18px;
+}
+
+#content #designer .information
+{
+ display: block;
+ height: 40px;
+}
+
+#content #designer #field-bottom fieldset.common
+{
+ margin-top: 10px;
+}
+
+#content #designer #field-bottom fieldset legend,
+#content #designer #field-bottom .optional.expanded legend
+{
+ display: block;
+ margin-left: 10px;
+ padding: 0px 5px;
+}
+
+#content #designer #field-bottom fieldset legend label
+{
+ margin-top: 0;
+}
+
+#content #designer #query-form select
+{
+ width: 150px;
+}
+
+#content #designer #selected-hit {
+ padding: 5px;
+}
+
+#content #designer tr.odd
+{
+ background-color: #f8f8f8;
+}
+
+#content #designer .help
+{
+ font-weight: normal !important;
+ position: relative;
+}
+
+#content #designer .help div.help
+{
+ z-index: 200;
+ background-color: #FCF0AD;
+ border: 1px solid #f0f0f0;
+ box-shadow: 5px 5px 10px #c0c0c0;
+ -moz-box-shadow: 5px 5px 10px #c0c0c0;
+ -webkit-box-shadow: 5px 5px 10px #c0c0c0;
+ position: absolute;
+ left: 0px;
+ top: 0px;
+ padding: 6px;
+ width: 340px;
+}
+
+#content #designer .help div.help-wide
+{
+ z-index: 200;
+ background-color: #FCF0AD;
+ border: 1px solid #f0f0f0;
+ box-shadow: 5px 5px 10px #c0c0c0;
+ -moz-box-shadow: 5px 5px 10px #c0c0c0;
+ -webkit-box-shadow: 5px 5px 10px #c0c0c0;
+ position: absolute;
+ left: -90px;
+ top: 0px;
+ padding: 6px;
+ width: 420px;
+}
+
+#content #designer .help div.help-wider
+{
+ z-index: 200;
+ background-color: #FCF0AD;
+ border: 1px solid #f0f0f0;
+ box-shadow: 5px 5px 10px #c0c0c0;
+ -moz-box-shadow: 5px 5px 10px #c0c0c0;
+ -webkit-box-shadow: 5px 5px 10px #c0c0c0;
+ position: absolute;
+ left: 0px;
+ top: 0px;
+ padding: 6px;
+ width: 430px;
+}
+
+#content #designer .help div.help-top
+{
+ z-index: 200;
+ background-color: #FCF0AD;
+ border: 1px solid #f0f0f0;
+ box-shadow: 5px 5px 10px #c0c0c0;
+ -moz-box-shadow: 5px 5px 10px #c0c0c0;
+ -webkit-box-shadow: 5px 5px 10px #c0c0c0;
+ position: absolute;
+ left: 0px;
+ top: -200px;
+ padding: 6px;
+ width: 420px;
+}
+
+#content #designer .help-anchor
+{
+ margin-top: 7px;
+ margin-left: 18px;
+ margin-bottom: 10px;
+}
+
+#content #designer .help-anchor a
+{
+ color: #003eff;
+ text-decoration-line: underline;
+ text-decoration-color: #003eff;
+}
+
+#content #designer .help-ico {
+ margin-left: 3px;
+ margin-top: 3px;
+}
+
+#content #designer #analysis .verbose_output
+{
+ padding: 3px;
+ float: left;
+ margin-left: 260px;
+}
+
+#content #designer #analysis .verbose_output a
+{
+ background-image: url( ../../img/ico/ui-check-box-uncheck.png );
+ background-position: 0 50%;
+ color: #4D4D4D;
+ display: block;
+ padding-left: 21px;
+}
+
+#content #designer #analysis .verbose_output.active a
+{
+ background-image: url( ../../img/ico/ui-check-box.png );
+}
+
+#content #designer #file-node .error
+{
+ background-image: url( ../../img/ico/cross-button.png );
+ color: #c00;
+ font-weight: bold;
+}
+
+#content #designer #file-node .error span
+{
+ margin-left: 20px;
+}
+
+#content #designer .field-top-form {
+ margin-bottom: 7px;
+ width: 300px;
+}
+
+#content #designer .field-top-form label {
+ text-align: right;
+ margin-right: 5px;
+ width: 150px;
+}
+
+#content #designer .field-top-form select
+{
+ display: block;
+ width: 150px;
+}
+
+#content #designer .warn-dialog
+{
+ z-index: 200;
+ background-color: #f0f0f0;
+ border: 1px solid #a0a0a0;
+ box-shadow: 5px 5px 10px #c0c0c0;
+ -moz-box-shadow: 5px 5px 10px #c0c0c0;
+ -webkit-box-shadow: 5px 5px 10px #c0c0c0;
+ position: absolute;
+ left: 350px;
+ top: 95px;
+ padding: 20px;
+ width: 450px;
+}
+
+#content #designer #warn-dialog #warn-dialog-buttons {
+ float: right;
+}
+
+#content #designer #warn-dialog .warn-button {
+ margin-right: 15px;
+}
+
+#content #designer #warn-dialog .warn-button span
+{
+ background-image: url( ../../img/ico/tick.png );
+}
+
+#content #designer .error-dialog
+{
+ z-index: 200;
+ background-color: #f0f0f0;
+ border: 1px solid #c00;
+ box-shadow: 5px 5px 10px #c0c0c0;
+ -moz-box-shadow: 5px 5px 10px #c0c0c0;
+ -webkit-box-shadow: 5px 5px 10px #c0c0c0;
+ position: absolute;
+ left: 350px;
+ top: 95px;
+ padding: 20px;
+ width: 450px;
+}
+
+#content #designer #error-dialog #error-dialog-buttons {
+ float: right;
+}
+
+#content #designer #error-dialog .error-button {
+ margin-right: 15px;
+}
+
+#content #designer #error-dialog .error-button span
+{
+ background-image: url( ../../img/ico/tick.png );
+}
+
+#content #designer #error-dialog .reload-error-button {
+ margin-right: 15px;
+}
+
+#content #designer #error-dialog .reload-error-button span
+{
+ background-image: url( ../../img/ico/arrow-circle.png );
+}
+
+#content #designer #error-dialog-note {
+ color: #c00;
+ font-weight: bold;
+ margin-bottom: 15px;
+}
+
+#content #designer #error-dialog-details {
+ min-height: 80px;
+ margin-bottom: 15px;
+}
+
+#content #designer #edit-analysis-json-link {
+ margin-top: 10px;
+ float: right;
+}
+
+#content #designer #edit-analysis-json-link a {
+ cursor: pointer;
+ color: #003eff;
+ text-decoration-line: underline;
+ text-decoration-color: #003eff;
+}
+
+#content #designer #tree-filters {
+ font-size: 11px;
+ max-width: 400px;
+ overflow: hidden;
+ margin-top: 2px;
+ margin-bottom: 6px;
+}
+
+#content #designer #tree-filters-cat {
+ max-width: 70px;
+ font-size: 11px;
+}
+
+#content #designer #tree-filters-options {
+ max-width: 90px;
+ font-size: 11px;
+}
+
+#content #designer #copy_schema {
+ width: 200px;
+}
+
+#content #designer #currentSchemaLabel {
+ font-weight: bold;
+ margin-top: 2px;
+}
+
+#content #designer #treeFilterFeatureEnabledSpan {
+ display: inline;
+ font-size: 11px;
+}
+
+#content #designer #treeFilterFeatureEnabledSpan input {
+ padding: 0px;
+ margin-top: 4px;
+}
+
+#content #designer .container-node {
+ max-height: 540px;
+ overflow: auto;
+}
+
+
+#content #designer .prop {
+ border-bottom: 1px solid #dddddd;
+ padding: 3px;
+ text-align: left;
+ min-width: 80px;
+ word-wrap: break-word;
+ max-width: 140px;
+}
+
+#content #designer .prop-flag {
+ border-bottom: 1px solid #dddddd;
+ padding: 3px;
+ text-align: center;
+}
+
+#content #designer .prop-hdr {
+ border-bottom: 1px solid #dddddd;
+ padding: 3px;
+ font-weight: bold;
+ text-align: left;
+ min-width: 80px;
+ word-wrap: break-word;
+ max-width: 140px;
+}
+
+#content #designer .prop-hdr-flag {
+ border-bottom: 1px solid #dddddd;
+ padding: 3px;
+ font-weight: bold;
+ text-align: center;
+ min-width: 65px;
+}
+
+#content #designer #editDocumentsToolbar { float: right; margin-top:0px; margin-left:20px; }
+#content #designer #editDocuments span { background-image: url( ../../img/ico/pencil.png ); }
+
+#content #designer .help div.help-small
+{
+ z-index: 200;
+ background-color: #FCF0AD;
+ border: 1px solid #f0f0f0;
+ box-shadow: 5px 5px 10px #c0c0c0;
+ -moz-box-shadow: 5px 5px 10px #c0c0c0;
+ -webkit-box-shadow: 5px 5px 10px #c0c0c0;
+ position: absolute;
+ left: -120px;
+ top: 0px;
+ padding: 6px;
+ width: 250px;
+}
+
+
diff --git a/solr/webapp/web/index.html b/solr/webapp/web/index.html
index 13fb2984629..906acc03cee 100644
--- a/solr/webapp/web/index.html
+++ b/solr/webapp/web/index.html
@@ -26,6 +26,7 @@ limitations under the License.
<link rel="stylesheet" type="text/css" href="css/angular/angular-csp.css?_=${version}">
<link rel="stylesheet" type="text/css" href="css/angular/common.css?_=${version}">
<link rel="stylesheet" type="text/css" href="css/angular/analysis.css?_=${version}">
+ <link rel="stylesheet" type="text/css" href="css/angular/schema-designer.css?_=${version}">
<link rel="stylesheet" type="text/css" href="css/angular/cloud.css?_=${version}">
<link rel="stylesheet" type="text/css" href="css/angular/cores.css?_=${version}">
<link rel="stylesheet" type="text/css" href="css/angular/collections.css?_=${version}">
@@ -88,6 +89,7 @@ limitations under the License.
<script src="js/angular/controllers/plugins.js"></script>
<script src="js/angular/controllers/replication.js"></script>
<script src="js/angular/controllers/schema.js"></script>
+ <script src="js/angular/controllers/schema-designer.js"></script>
<script src="js/angular/controllers/segments.js"></script>
<script src="js/angular/controllers/unknown.js"></script>
<script src="js/angular/controllers/sqlquery.js"></script>
@@ -167,6 +169,8 @@ limitations under the License.
</ul>
</li>
+ <li ng-show="isCloudEnabled" id="schema-designer" class="global" ng-class="{active:page=='schema-designer'}"><p><a href="#/~schema-designer">Schema Designer</a></p></li>
+
<li ng-show="isCloudEnabled" id="collections" class="global" ng-class="{active:page=='collections'}"><p><a href="#/~collections">Collections</a></p></li>
<li ng-hide="isCloudEnabled" id="cores" class="global" ng-class="{active:page=='cores'}"><p><a href="#/~cores">Core Admin</a></p></li>
diff --git a/solr/webapp/web/js/angular/app.js b/solr/webapp/web/js/angular/app.js
index 0f363289dc3..df67edba586 100644
--- a/solr/webapp/web/js/angular/app.js
+++ b/solr/webapp/web/js/angular/app.js
@@ -173,6 +173,10 @@ solrAdminApp.config([
templateUrl: 'partials/segments.html',
controller: 'SegmentsController'
}).
+ when('/~schema-designer', {
+ templateUrl: 'partials/schema-designer.html',
+ controller: 'SchemaDesignerController'
+ }).
otherwise({
templateUrl: 'partials/unknown.html',
controller: 'UnknownController'
@@ -368,7 +372,9 @@ solrAdminApp.config([
if (sessionStorage.getItem("auth.header")) {
config.headers['Authorization'] = sessionStorage.getItem("auth.header");
}
- config.timeout = 10000;
+ if (!config.timeout) {
+ config.timeout = 10000;
+ }
return config || $q.when(config);
};
@@ -430,7 +436,11 @@ solrAdminApp.config([
$location.path('/login');
}
} else {
- $rootScope.exceptions[rejection.config.url] = rejection.data.error;
+ // schema designer prefers to handle errors itselft
+ var isHandledBySchemaDesigner = rejection.config.url && rejection.config.url.startsWith("/api/schema-designer/");
+ if (!isHandledBySchemaDesigner) {
+ $rootScope.exceptions[rejection.config.url] = rejection.data.error;
+ }
}
return $q.reject(rejection);
};
@@ -480,6 +490,9 @@ solrAdminApp.controller('MainController', function($scope, $route, $rootScope, $
delete $scope.currentCore;
for (key in data.status) {
var core = data.status[key];
+ if (core.name.startsWith("._designer_")) {
+ continue;
+ }
$scope.cores.push(core);
if ((!$scope.isSolrCloud || pageType == Constants.IS_CORE_PAGE) && core.name == currentCoreName) {
$scope.currentCore = core;
@@ -511,6 +524,9 @@ solrAdminApp.controller('MainController', function($scope, $route, $rootScope, $
}
$scope.collections = [];
for (key in cdata.collections) {
+ if (cdata.collections[key].startsWith("._designer_")) {
+ continue; // ignore temp designer collections
+ }
var collection = {name: cdata.collections[key], type: 'collection'};
$scope.collections.push(collection);
if (pageType == Constants.IS_COLLECTION_PAGE && collection.name == currentCollectionName) {
diff --git a/solr/webapp/web/js/angular/controllers/cloud.js b/solr/webapp/web/js/angular/controllers/cloud.js
index bf8fc336ee2..2d62989cccb 100644
--- a/solr/webapp/web/js/angular/controllers/cloud.js
+++ b/solr/webapp/web/js/angular/controllers/cloud.js
@@ -562,14 +562,12 @@ var treeSubController = function($scope, Zookeeper) {
var path = decodeURIComponent(link.replace(/.*[\\?&]path=([^&#]*).*/, "$1"));
Zookeeper.detail({path: path}, function(data) {
$scope.znode = data.znode;
- var path = data.znode.path.split( '.' );
- if(path.length >1) {
- $scope.lang = path.pop();
+ if (data.znode.path.endsWith("/managed-schema") || data.znode.path.endsWith(".xml.bak")) {
+ $scope.lang = "xml";
} else {
var lastPathElement = data.znode.path.split( '/' ).pop();
- if (lastPathElement == "managed-schema") {
- $scope.lang = "xml";
- }
+ var lastDotAt = lastPathElement ? lastPathElement.lastIndexOf('.') : -1;
+ $scope.lang = lastDotAt != -1 ? lastPathElement.substring(lastDotAt+1) : "txt";
}
$scope.showData = true;
});
diff --git a/solr/webapp/web/js/angular/controllers/schema-designer.js b/solr/webapp/web/js/angular/controllers/schema-designer.js
new file mode 100644
index 00000000000..762fe8b4fb6
--- /dev/null
+++ b/solr/webapp/web/js/angular/controllers/schema-designer.js
@@ -0,0 +1,1978 @@
+/*
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+*/
+
+solrAdminApp.controller('SchemaDesignerController', function ($scope, $timeout, $cookies, $window, Constants, SchemaDesigner, Luke) {
+ $scope.resetMenu("schema-designer", Constants.IS_ROOT_PAGE);
+
+ $scope.onWarning = function (warnMsg, warnDetails) {
+ $scope.updateWorking = false;
+ delete $scope.updateStatusMessage;
+ $scope.apiWarning = warnMsg;
+ $scope.apiWarningDetails = warnDetails;
+ };
+
+ $scope.onError = function (errorMsg, errorCode, errorDetails) {
+ $scope.updateWorking = false;
+ delete $scope.updateStatusMessage;
+ $scope.designerAPIError = errorMsg;
+ if (errorDetails) {
+ var errorDetailsStr = "";
+ if (errorDetails["error"]) {
+ errorDetailsStr = errorDetails["error"];
+ } else {
+ for (var id in errorDetails) {
+ var msg = errorDetails[id];
+ var at = msg.indexOf("ERROR: ");
+ if (at !== -1) {
+ msg = msg.substring(at+7);
+ }
+ if (!msg.includes(id)) {
+ msg = id+": "+msg;
+ }
+ errorDetailsStr += msg+"\n\n";
+ }
+ }
+ $scope.designerAPIErrorDetails = errorDetailsStr;
+ } else {
+ delete $scope.designerAPIErrorDetails;
+ }
+
+ if (errorCode === 409) {
+ $scope.schemaVersion = -1; // reset to get the latest
+ $scope.isVersionMismatch = true;
+ $scope.errorButton = "Reload Schema";
+ } else if (errorCode < 500) {
+ $scope.isVersionMismatch = false;
+ $scope.errorButton = "OK";
+ } // else 500 errors get the top-level error message
+ };
+
+ $scope.errorHandler = function (e) {
+ var error = e.data && e.data.error ? e.data.error : null;
+ if (error) {
+ $scope.onError(error.msg, error.code, e.data.errorDetails);
+ } else {
+ // when a timeout occurs, the error details are sparse so just give the user a hint that something was off
+ var path = e.config && e.config.url ? e.config.url : "/api/schema-designer";
+ var reloadMsg = "";
+ if (path.includes("/analyze")) {
+ reloadMsg = " Re-try analyzing your sample docs by clicking on 'Analyze Documents' again."
+ }
+ $scope.onError("Request to "+path+" failed!", 408,
+ {"error":"Most likely the request timed out; check server log for more details."+reloadMsg});
+ }
+ };
+
+ $scope.closeWarnDialog = function () {
+ delete $scope.apiWarning;
+ delete $scope.apiWarningDetails;
+ };
+
+ $scope.closeErrorDialog = function () {
+ delete $scope.designerAPIError;
+ delete $scope.designerAPIErrorDetails;
+ if ($scope.isVersionMismatch) {
+ $scope.isVersionMismatch = false;
+ var nodeId = "/";
+ if ($scope.selectedNode) {
+ nodeId = $scope.selectedNode.href;
+ }
+ $scope.doAnalyze(nodeId);
+ }
+ };
+
+ $scope.refresh = function () {
+ delete $scope.helpId;
+
+ $scope.updateStatusMessage = "";
+ $scope.analysisVerbose = false;
+ $scope.updateWorking = false;
+ $scope.currentSchema = "";
+
+ delete $scope.hasDocsOnServer;
+ delete $scope.queryResultsTree;
+
+ $scope.languages = ["*"];
+ $scope.copyFrom = "_default";
+ delete $scope.sampleMessage;
+ delete $scope.sampleDocuments;
+
+ $scope.schemaVersion = -1;
+ $scope.schemaTree = {};
+ $scope.showSchemaActions = false;
+ $scope.sampleDocIds = [];
+ $scope.isSchemaRoot = false;
+
+ delete $scope.enableNestedDocs;
+ delete $scope.enableDynamicFields;
+ delete $scope.enableFieldGuessing;
+
+ // schema editor
+ $scope.showFieldDetails = false;
+ $scope.selectedNode = null;
+ $scope.selectedUpdated = false;
+ delete $scope.updateStatusMessage;
+
+ // text field analysis
+ $scope.showAnalysis = false;
+ $scope.sampleDocId = null;
+ $scope.indexText = "";
+ $scope.result = {};
+
+ // publish vars
+ delete $scope.newCollection;
+ $scope.reloadOnPublish = "true";
+
+ // query form
+ $scope.query = {q: '*:*', sortBy: 'score', sortDir: 'desc'};
+
+ SchemaDesigner.get({path: "configs"}, function (data) {
+ $scope.schemas = [];
+ $scope.schemasWithDefault = [];
+ $scope.schemasWithDefault.push("_default");
+
+ for (var s in data.configSets) {
+ $scope.schemasWithDefault.push(s);
+ if (data.configSets[s]) {
+ $scope.schemas.push(s);
+ }
+ }
+ $scope.schemas.sort();
+ $scope.schemasWithDefault.sort();
+
+ // if no schemas available to select, open the pop-up immediately
+ if ($scope.schemas.length == 0) {
+ $scope.firstSchemaMessage = true;
+ $scope.showNewSchemaDialog();
+ }
+ });
+ };
+
+ $scope.selectNodeInTree = function(nodeId) {
+ nodeId = stripAnchorSuffix(nodeId);
+ if (!nodeId) return;
+
+ var jst = $('#schemaJsTree').jstree(true);
+ if (jst) {
+ var selectedId = null;
+ var selected_node = jst.get_selected();
+ if (selected_node && selected_node.length > 0) {
+ selectedId = selected_node[0];
+ }
+ if (selectedId) {
+ try {
+ jst.deselect_node(selectedId);
+ } catch (err) {
+ // just ignore
+ //console.log("error deselecting "+selectedId);
+ }
+ }
+
+ try {
+ jst.select_node(nodeId, true);
+ } catch (err) {
+ // just ignore, some low-level tree issue
+ //console.log("error selecting "+nodeId);
+ }
+ }
+ };
+
+ $scope.loadFile = function (event) {
+ var t = event.target || event.srcElement || event.currentTarget;
+ if (t && t.text) {
+ $scope.onSelectFileNode("files/" + t.text, true);
+ }
+ };
+
+ $scope.confirmEditSchema = function () {
+ $scope.showConfirmEditSchema = false;
+ if ($scope.hasDocsOnServer || $scope.published) {
+ $scope.doAnalyze();
+ } else {
+ $scope.sampleMessage = "Please upload or paste some sample documents to analyze for building the '" + $scope.currentSchema + "' schema.";
+ }
+ };
+
+ $scope.cancelEditSchema = function () {
+ $scope.currentSchema = "";
+ $scope.showConfirmEditSchema = false;
+ };
+
+ $scope.loadSchema = function () {
+
+ if (!$scope.currentSchema) {
+ return;
+ }
+
+ $scope.resetSchema();
+ var params = {path: "info", configSet: $scope.currentSchema};
+ SchemaDesigner.get(params, function (data) {
+ $scope.currentSchema = data.configSet;
+ $("#select-schema").trigger("chosen:updated");
+
+ $scope.confirmSchema = data.configSet;
+ $scope.collectionsForConfig = data.collections;
+ $scope.hasDocsOnServer = data.numDocs > 0;
+ $scope.published = data.published;
+ $scope.initDesignerSettingsFromResponse(data);
+ if ($scope.collectionsForConfig && $scope.collectionsForConfig.length > 0) {
+ $scope.showConfirmEditSchema = true;
+ } else {
+ if ($scope.hasDocsOnServer || $scope.published) {
+ $scope.doAnalyze();
+ } else {
+ $scope.sampleMessage = "Please upload or paste some sample documents to build the '" + $scope.currentSchema + "' schema.";
+ }
+ }
+ });
+ };
+
+ $scope.showNewSchemaDialog = function () {
+ $scope.hideAll();
+ $scope.showNewSchema = true;
+ $scope.newSchema = "";
+ };
+
+ $scope.addSchema = function () {
+ $scope.firstSchemaMessage = false;
+
+ if (!$scope.newSchema) {
+ $scope.addMessage = "Please provide a schema name!";
+ return;
+ }
+
+ $scope.newSchema = $scope.newSchema.trim();
+ if ($scope.newSchema.length > 50) {
+ $scope.addMessage = "Schema name be 50 characters or less";
+ return;
+ }
+
+ if ($scope.newSchema.indexOf(" ") !== -1 || $scope.newSchema.indexOf("/") !== -1) {
+ $scope.addMessage = "Schema name should not contain spaces or /";
+ return;
+ }
+
+ if ($scope.schemasWithDefault.includes($scope.newSchema)) {
+ $scope.addMessage = "Schema '" + $scope.newSchema + "' already exists!";
+ return;
+ }
+
+ if (!$scope.copyFrom) {
+ $scope.copyFrom = "_default";
+ }
+
+ $scope.resetSchema();
+ $scope.schemas.push($scope.newSchema);
+ $scope.schemasWithDefault.push($scope.newSchema);
+ $scope.showNewSchema = false;
+ $scope.currentSchema = $scope.newSchema;
+ $scope.sampleMessage = "Please upload or paste some sample documents to analyze for building the '" + $scope.currentSchema + "' schema.";
+
+ SchemaDesigner.post({path: "prep", configSet: $scope.newSchema, copyFrom: $scope.copyFrom}, null, function (data) {
+ $scope.initDesignerSettingsFromResponse(data);
+ }, $scope.errorHandler);
+ };
+
+ $scope.cancelAddSchema = function () {
+ delete $scope.addMessage;
+ delete $scope.sampleMessage;
+
+ $scope.showNewSchema = false
+ };
+
+ $scope.hideAll = function () {
+ delete $scope.helpId;
+ $scope.showPublish = false;
+ $scope.showDiff = false;
+ $scope.showNewSchema = false;
+ $scope.showAddField = false;
+ $scope.showAddDynamicField = false;
+ $scope.showAddCopyField = false;
+ $scope.showAnalysis = false;
+ // add more dialogs here
+ };
+
+ $scope.showHelp = function (id) {
+ if ($scope.helpId && ($scope.helpId === id || id === '')) {
+ delete $scope.helpId;
+ } else {
+ $scope.helpId = id;
+ }
+ };
+
+ $scope.hideData = function () {
+ $scope.showData = false;
+ };
+
+ $scope.rootChanged = function () {
+ $scope.selectedUpdated = true;
+ $scope.selectedType = "Schema";
+ };
+
+ $scope.updateUniqueKey = function () {
+ delete $scope.schemaRootMessage;
+ var jst = $('#schemaJsTree').jstree();
+ if (jst) {
+ var node = jst.get_node("field/" + $scope.updateUniqueKeyField);
+ if (node && node.a_attr) {
+ var attrs = node.a_attr;
+ if (attrs.multiValued || attrs.tokenized || !attrs.stored || !attrs.indexed) {
+ $scope.schemaRootMessage = "Field '" + $scope.updateUniqueKeyField +
+ "' cannot be used as the uniqueKey field! Must be single-valued, stored, indexed, and not tokenized.";
+ $scope.updateUniqueKeyField = $scope.uniqueKeyField;
+ return;
+ }
+ }
+ }
+ $scope.uniqueKeyField = $scope.updateUniqueKeyField;
+ $scope.selectedUpdated = true;
+ $scope.selectedType = "Schema";
+ };
+
+ $scope.resetSchema = function () {
+ $scope.hideAll();
+ $scope.analysisVerbose = false;
+ $scope.showSchemaActions = false;
+ $scope.showAnalysis = false;
+ $scope.showFieldDetails = false;
+ $scope.hasDocsOnServer = false;
+ $scope.query = {q: '*:*', sortBy: 'score', sortDir: 'desc'};
+ $scope.schemaVersion = -1;
+
+ $scope.updateWorking = false;
+ $scope.isVersionMismatch = false;
+ delete $scope.updateStatusMessage;
+ delete $scope.designerAPIError;
+ delete $scope.designerAPIErrorDetails;
+ delete $scope.selectedFacets;
+ delete $scope.sampleDocuments;
+ delete $scope.selectedNode;
+ delete $scope.queryResultsTree;
+ };
+
+ $scope.onSchemaUpdated = function (schema, data, nodeId) {
+ $scope.hasDocsOnServer = data.numDocs && data.numDocs > 0;
+ $scope.uniqueKeyField = data.uniqueKeyField;
+ $scope.updateUniqueKeyField = $scope.uniqueKeyField;
+ $scope.initDesignerSettingsFromResponse(data);
+
+ var fieldTypes = fieldTypesToTree(data.fieldTypes);
+ var files = filesToTree(data.files);
+
+ var rootChildren = [];
+ $scope.fieldsSrc = fieldsToTree(data.fields);
+ $scope.fieldsNode = {
+ "id": "fields",
+ "text": "Fields",
+ "state": {"opened": true},
+ "a_attr": {"href": "fields"},
+ "children": $scope.fieldsSrc
+ };
+ rootChildren.push($scope.fieldsNode);
+
+ if ($scope.enableDynamicFields === "true") {
+ $scope.dynamicFieldsSrc = fieldsToTree(data.dynamicFields);
+ $scope.dynamicFieldsNode = {
+ "id": "dynamicFields",
+ "text": "Dynamic Fields",
+ "a_attr": {"href": "dynamicFields"},
+ "children": $scope.dynamicFieldsSrc
+ };
+ rootChildren.push($scope.dynamicFieldsNode);
+ } else {
+ delete $scope.dynamicFieldsNode;
+ delete $scope.dynamicFieldsSrc;
+ }
+
+ rootChildren.push({"id":"fieldTypes", "text": "Field Types", "a_attr": {"href": "fieldTypes"}, "children": fieldTypes});
+ rootChildren.push(files);
+
+ var tree = [{"id":"/", "text": schema, "a_attr": {"href": "/"}, "state":{"opened": true}, "children": rootChildren}];
+
+ $scope.fields = data.fields;
+ $scope.fieldNames = data.fields.map(f => f.name).sort();
+ $scope.possibleIdFields = data.fields.filter(f => f.indexed && f.stored && !f.tokenized).map(f => f.name).sort();
+ $scope.sortableFields = data.fields.filter(f => (f.indexed || f.docValues) && !f.multiValued && !f.tokenized).map(f => f.name).sort();
+ $scope.sortableFields.push("score");
+
+ $scope.facetFields = data.fields.filter(f => (f.indexed || f.docValues) && !f.tokenized && f.name !== '_version_').map(f => f.name).sort();
+ $scope.hlFields = data.fields.filter(f => f.stored && f.tokenized).map(f => f.name).sort();
+
+ $scope.schemaVersion = data.schemaVersion;
+ $scope.currentSchema = data.configSet;
+ $scope.fieldTypes = fieldTypes;
+ $scope.core = data.core;
+ $scope.schemaTree = tree;
+ $scope.refreshTree();
+
+ $scope.collectionsForConfig = data.collectionsForConfig;
+
+ if (data.docIds) {
+ $scope.sampleDocIds = data.docIds;
+ }
+
+ // re-apply the filters on the updated schema
+ $scope.onTreeFilterOptionChanged();
+
+ // Load the Luke schema
+ Luke.schema({core: data.core}, function (schema) {
+ Luke.raw({core: data.core}, function (index) {
+ $scope.luke = mergeIndexAndSchemaData(index, schema.schema);
+ $scope.types = Object.keys(schema.schema.types);
+ $scope.showSchemaActions = true;
+ if (!nodeId) {
+ nodeId = "/";
+ }
+ $scope.onSelectSchemaTreeNode(nodeId);
+
+ $scope.updateWorking = false;
+
+ if (data.updateError != null) {
+ $scope.onError(data.updateError, data.updateErrorCode, data.errorDetails);
+ } else {
+ if ($scope.selectedUpdated) {
+ $scope.selectedUpdated = false;
+ $scope.updateStatusMessage = "Changes applied successfully.";
+ var waitMs = 3000;
+ if (data.rebuild) {
+ $scope.updateStatusMessage += " Did full re-index of sample docs due to incompatible update.";
+ waitMs = 5000; // longer message, more time to read
+ }
+
+ if (data.analysisError) {
+ var updateType = data["updateType"];
+ var updatedObject = data[updateType];
+ var updatedName = updatedObject && updatedObject.name ? updatedObject.name : "";
+ var warnMsg = "Changes to "+updateType+" "+updatedName+" applied, but required the temp collection to be deleted " +
+ "and re-created due to an incompatible Lucene change, see details below.";
+ $scope.onWarning(warnMsg, data.analysisError);
+ } else {
+ $timeout(function () { delete $scope.updateStatusMessage; }, waitMs);
+ }
+ } else {
+ var source = data.sampleSource;
+ if (source) {
+ if (source === "paste") {
+ source = "pasted sample"
+ } else if (source === "blob") {
+ source = "previous upload stored on the server"
+ }
+ if (data.numDocs > 0) {
+ $scope.updateStatusMessage = "Analyzed "+data.numDocs+" docs from "+source;
+ } else {
+ $scope.updateStatusMessage = "Schema '"+$scope.currentSchema+"' loaded.";
+ }
+ }
+ $timeout(function () {
+ delete $scope.updateStatusMessage;
+ }, 5000);
+ }
+ }
+
+ // re-fire the current query to reflect the updated schema
+ $scope.doQuery();
+ $scope.selectNodeInTree(nodeId);
+ });
+ });
+ };
+
+ $scope.toggleAddField = function (type) {
+ if ($scope.showAddField) {
+ $scope.hideAll();
+ } else {
+ $scope.hideAll();
+ $scope.showAddField = true;
+ $scope.adding = type;
+
+ $scope.newField = {
+ stored: "true",
+ indexed: "true",
+ uninvertible: "true",
+ docValues: "true"
+ }
+
+ if (type === "field") {
+ $scope.newField.type = "string";
+ }
+
+ delete $scope.addErrors;
+ }
+ };
+
+ function applyConstraintsOnField(f) {
+
+ if (!f.docValues) {
+ delete f.useDocValuesAsStored;
+ }
+
+ if (!f.docValues && !f.uninvertible) {
+ delete f.sortMissingLast; // remove this setting if no docValues / uninvertible
+ }
+
+ if (f.indexed) {
+ if (f.omitTermFreqAndPositions && !f.omitPositions) {
+ delete f.omitPositions; // :shrug ~ see SchemaField ln 295
+ }
+ if (!f.termVectors) {
+ delete f.termPositions;
+ delete f.termOffsets;
+ delete f.termPayloads;
+ }
+ } else {
+ // if not indexed, a bunch of fields are false
+ f.tokenized = false;
+ f.uninvertible = false;
+
+ // drop these from the request
+ delete f.termVectors;
+ delete f.termPositions;
+ delete f.termOffsets;
+ delete f.termPayloads;
+ delete f.omitNorms;
+ delete f.omitPositions;
+ delete f.omitTermFreqAndPositions;
+ delete f.storeOffsetsWithPositions;
+ }
+
+ return f;
+ }
+
+ $scope.addField = function () {
+
+ // validate the form input
+ $scope.addErrors = [];
+ if (!$scope.newField.name) {
+ $scope.addErrors.push($scope.adding + " name is required!");
+ }
+
+ if ($scope.newField.name.indexOf(" ") != -1) {
+ $scope.addErrors.push($scope.adding + " name should not have whitespace");
+ }
+
+ var command = "add-field-type";
+ if ("field" === $scope.adding) {
+
+ if ($scope.fieldNames.includes($scope.newField.name)) {
+ $scope.addErrors.push("Field '" + $scope.newField.name + "' already exists!");
+ return;
+ }
+
+ // TODO: is this the correct logic for detecting dynamic? Probably good enough for the designer
+ var isDynamic = $scope.newField.name.startsWith("*") || $scope.newField.name.endsWith("*");
+ if (isDynamic) {
+ if ($scope.luke && $scope.luke.dynamic_fields[$scope.newField.name]) {
+ $scope.addErrors.push("dynamic field '" + $scope.newField.name + "' already exists!");
+ }
+ } else {
+ if ($scope.luke && $scope.luke.fields[$scope.newField.name]) {
+ $scope.addErrors.push("field '" + $scope.newField.name + "' already exists!");
+ }
+ }
+
+ if (!$scope.newField.type) {
+ $scope.addErrors.push("field type is required!");
+ }
+
+ command = isDynamic ? "add-dynamic-field" : "add-field";
+ } else if ("type" === $scope.adding) {
+ if ($scope.types.includes($scope.newField.name)) {
+ $scope.addErrors.push("Type '" + $scope.newField.name + "' already exists!");
+ }
+
+ if (!$scope.newField.class) {
+ $scope.addErrors.push("class is required when creating a new field type!");
+ }
+ }
+
+ var addData = {};
+ addData[command] = applyConstraintsOnField($scope.newField);
+ if ($scope.textAnalysisJson) {
+ var text = $scope.textAnalysisJson.trim();
+ if (text.length > 0) {
+ text = text.replace(/\s+/g, ' ');
+ if (!text.startsWith("{")) {
+ text = "{ " + text + " }";
+ }
+ try {
+ var textJson = JSON.parse(text);
+ if (textJson.analyzer) {
+ addData[command].analyzer = textJson.analyzer;
+ } else {
+ if (!textJson.indexAnalyzer || !textJson.queryAnalyzer) {
+ $scope.addErrors.push("Text analysis JSON should define either an 'analyzer' or an 'indexAnalyzer' and 'queryAnalyzer'");
+ return;
+ }
+ addData[command].indexAnalyzer = textJson.indexAnalyzer;
+ addData[command].queryAnalyzer = textJson.queryAnalyzer;
+ }
+ } catch (e) {
+ $scope.addErrors.push("Failed to parse analysis as JSON due to: " + e.message +
+ "; expected JSON object with either an 'analyzer' or 'indexAnalyzer' and 'queryAnalyzer'");
+ return;
+ }
+ }
+ }
+
+ if ($scope.addErrors.length > 0) {
+ return;
+ }
+ delete $scope.addErrors; // no errors!
+
+ SchemaDesigner.post({
+ path: "add",
+ configSet: $scope.currentSchema,
+ schemaVersion: $scope.schemaVersion
+ }, addData, function (data) {
+ if (data.errors) {
+ $scope.addErrors = data.errors[0].errorMessages;
+ if (typeof $scope.addErrors === "string") {
+ $scope.addErrors = [$scope.addErrors];
+ }
+ } else {
+ delete $scope.textAnalysisJson;
+ $scope.added = true;
+ $timeout(function () {
+ $scope.showAddField = false;
+ $scope.added = false;
+ var nodeId = "/";
+ if ("field" === $scope.adding) {
+ nodeId = "field/" + data[command];
+ } else if ("type" === $scope.adding) {
+ nodeId = "type/" + data[command];
+ }
+ $scope.onSchemaUpdated(data.configSet, data, nodeId);
+ }, 500);
+ }
+ }, $scope.errorHandler);
+ }
+
+ function toSortedNameAndTypeList(fields, typeAttr) {
+ var list = [];
+ var keys = Object.keys(fields);
+ for (var f in keys) {
+ var field = fields[keys[f]];
+ var type = field[typeAttr];
+ if (type) {
+ list.push(field.name + ": "+type);
+ } else {
+ list.push(field.name);
+ }
+ }
+ return list.sort();
+ }
+
+ function toSortedFieldList(fields) {
+ var list = [];
+ var keys = Object.keys(fields);
+ for (var f in keys) {
+ list.push(fields[keys[f]]);
+ }
+ return list.sort((a, b) => (a.name > b.name) ? 1 : -1);
+ }
+
+ $scope.toggleDiff = function (event) {
+ if ($scope.showDiff) {
+ // toggle, close dialog
+ $scope.showDiff = false;
+ return;
+ }
+
+ if (event) {
+ var t = event.target || event.currentTarget;
+ var leftPos = t.getBoundingClientRect().left - 600;
+ if (leftPos < 0) leftPos = 0;
+ $('#show-diff-dialog').css({left: leftPos});
+ }
+
+ SchemaDesigner.get({ path: "diff", configSet: $scope.currentSchema }, function (data) {
+ var diff = data.diff;
+
+ var dynamicFields = diff.dynamicFields;
+ var enableDynamicFields = data.enableDynamicFields !== null ? data.enableDynamicFields : true;
+ if (!enableDynamicFields) {
+ dynamicFields = null;
+ }
+
+ $scope.diffSource = data["diff-source"];
+ $scope.schemaDiff = {
+ "fieldsDiff": diff.fields,
+ "addedFields": [],
+ "removedFields": [],
+ "fieldTypesDiff": diff.fieldTypes,
+ "removedTypes": [],
+ "dynamicFieldsDiff": dynamicFields,
+ "copyFieldsDiff": diff.copyFields
+ }
+ if (diff.fields && diff.fields.added) {
+ $scope.schemaDiff.addedFields = toSortedFieldList(diff.fields.added);
+ }
+ if (diff.fields && diff.fields.removed) {
+ $scope.schemaDiff.removedFields = toSortedNameAndTypeList(diff.fields.removed, "type");
+ }
+ if (diff.fieldTypes && diff.fieldTypes.removed) {
+ $scope.schemaDiff.removedTypes = toSortedNameAndTypeList(diff.fieldTypes.removed, "class");
+ }
+
+ $scope.schemaDiffExists = !(diff.fields == null && diff.fieldTypes == null && dynamicFields == null && diff.copyFields == null);
+ $scope.showDiff = true;
+ });
+ }
+
+ $scope.togglePublish = function (event) {
+ if (event) {
+ var t = event.target || event.currentTarget;
+ var leftPos = t.getBoundingClientRect().left - 515;
+ if (leftPos < 0) leftPos = 0;
+ $('#publish-dialog').css({left: leftPos});
+ }
+
+ $scope.showDiff = false;
+ $scope.showPublish = !$scope.showPublish;
+ delete $scope.publishErrors;
+
+ $scope.disableDesigner = "false";
+
+ if ($scope.showPublish && !$scope.newCollection) {
+ $scope.newCollection = {numShards: 1, replicationFactor: 1, indexToCollection: "true"};
+ }
+ };
+
+ $scope.toggleAddCopyField = function () {
+ if ($scope.showAddCopyField) {
+ $scope.hideAll();
+ $scope.showFieldDetails = true;
+ } else {
+ $scope.hideAll();
+ $scope.showAddCopyField = true;
+ $scope.showFieldDetails = false;
+
+ $scope.copyField = {};
+ delete $scope.addCopyFieldErrors;
+ }
+ }
+ $scope.addCopyField = function () {
+ delete $scope.addCopyFieldErrors;
+ var data = {"add-copy-field": $scope.copyField};
+ SchemaDesigner.post({
+ path: "add",
+ configSet: $scope.currentSchema,
+ schemaVersion: $scope.schemaVersion
+ }, data, function (data) {
+ if (data.errors) {
+ $scope.addCopyFieldErrors = data.errors[0].errorMessages;
+ if (typeof $scope.addCopyFieldErrors === "string") {
+ $scope.addCopyFieldErrors = [$scope.addCopyFieldErrors];
+ }
+ } else {
+ $scope.showAddCopyField = false;
+ // TODO:
+ //$timeout($scope.refresh, 1500);
+ }
+ }, $scope.errorHandler);
+ }
+
+ $scope.toggleAnalyzer = function (analyzer) {
+ analyzer.show = !analyzer.show;
+ }
+
+ $scope.initTypeAnalysisInfo = function (typeName) {
+ $scope.analysis = getAnalysisInfo($scope.luke, {type: true}, typeName);
+ if ($scope.analysis && $scope.analysis.data) {
+ $scope.className = $scope.analysis.data.className
+ }
+ $scope.editAnalysis = "Edit JSON";
+ $scope.showAnalysisJson = false;
+ delete $scope.analysisJsonText;
+ };
+
+ $scope.toggleVerbose = function () {
+ $scope.analysisVerbose = !$scope.analysisVerbose;
+ };
+
+ $scope.updateSampleDocId = function () {
+ $scope.indexText = "";
+ $scope.result = {};
+
+ if (!$scope.selectedNode) {
+ return;
+ }
+
+ var field = $scope.selectedNode.name;
+ var params = {path: "sample"};
+ params.configSet = $scope.currentSchema;
+ params.uniqueKeyField = $scope.uniqueKeyField;
+ params.field = field;
+
+ if ($scope.sampleDocId) {
+ params.docId = $scope.sampleDocId;
+ } // else the server will pick the first doc with a non-empty text value for the desired field
+
+ SchemaDesigner.get(params, function (data) {
+ $scope.sampleDocId = data[$scope.uniqueKeyField];
+ $scope.indexText = data[field];
+ if (data.analysis && data.analysis["field_names"]) {
+ $scope.result = processFieldAnalysisData(data.analysis["field_names"][field]);
+ }
+ });
+ };
+
+ $scope.changeLanguages = function () {
+ $scope.selectedUpdated = true;
+ $scope.selectedType = "Schema";
+ };
+
+ function getType(typeName) {
+ if ($scope.fieldTypes) {
+ for (i in $scope.fieldTypes) {
+ if ($scope.fieldTypes[i].text === typeName) {
+ return $scope.fieldTypes[i];
+ }
+ }
+ }
+ return null;
+ }
+
+ $scope.refreshTree = function() {
+ var jst = $('#schemaJsTree').jstree(true);
+ if (jst) {
+ jst.refresh();
+ }
+ };
+
+ $scope.onSchemaTreeLoaded = function (id) {
+ //console.log(">> on tree loaded");
+ };
+
+ $scope.updateFile = function () {
+ var nodeId = "files/" + $scope.selectedFile;
+ var params = {path: "file", file: $scope.selectedFile, configSet: $scope.currentSchema};
+
+ $scope.updateWorking = true;
+ $scope.updateStatusMessage = "Updating file ...";
+
+ SchemaDesigner.post(params, $scope.fileNodeText, function (data) {
+ if (data.updateFileError) {
+ if (data[$scope.selectedFile]) {
+ $scope.fileNodeText = data[$scope.selectedFile];
+ }
+ $scope.updateFileError = data.updateFileError;
+ } else {
+ delete $scope.updateFileError;
+ $scope.updateStatusMessage = "File '"+$scope.selectedFile+"' updated.";
+ $scope.onSchemaUpdated(data.configSet, data, nodeId);
+ }
+ }, $scope.errorHandler);
+ };
+
+ $scope.onSelectFileNode = function (id, doSelectOnTree) {
+ $scope.selectedFile = id.startsWith("files/") ? id.substring("files/".length) : id;
+
+ var params = {path: "file", file: $scope.selectedFile, configSet: $scope.currentSchema};
+ SchemaDesigner.get(params, function (data) {
+ $scope.fileNodeText = data[$scope.selectedFile];
+ $scope.isLeafNode = false;
+ if (doSelectOnTree) {
+ delete $scope.selectedNode;
+ $scope.isLeafNode = false;
+ $scope.showFieldDetails = true;
+ delete $scope.sampleDocId;
+ $scope.showAnalysis = false;
+ $scope.selectNodeInTree(id);
+ }
+ });
+ };
+
+ function fieldNodes(src, type) {
+ var nodes = [];
+ for (var c in src) {
+ var childNode = src[c];
+ if (childNode && childNode.a_attr) {
+ var a = childNode.a_attr;
+ var stored = a.stored || (a.docValues && a.useDocValuesAsStored);
+ var obj = {"name":a.name, "indexed":a.indexed, "docValues": a.docValues,
+ "multiValued":a.multiValued, "stored":stored, "tokenized": a.tokenized};
+ if (type === "field" || type === "dynamicField") {
+ obj.type = a.type;
+ } else if (type === "type") {
+ obj.class = a.class;
+ }
+ nodes.push(obj);
+ }
+ }
+ return nodes;
+ }
+
+ function stripAnchorSuffix(id) {
+ if (id && id.endsWith("_anchor")) {
+ id = id.substring(0, id.length - "_anchor".length);
+ }
+ return id;
+ }
+
+ $scope.onSelectSchemaTreeNode = function (id) {
+ id = stripAnchorSuffix(id);
+ $scope.showFieldDetails = false;
+ $scope.isSchemaRoot = false;
+ $scope.isLeafNode = false;
+ delete $scope.containerNodeLabel;
+ delete $scope.containerNode;
+ delete $scope.containerNodes;
+ delete $scope.selectedFile;
+
+ if (id === "/") {
+ $scope.selectedType = "Schema";
+ $scope.selectedNode = null;
+ $scope.isSchemaRoot = true;
+ $scope.isLeafNode = false;
+ $scope.isContainerNode = false;
+ $scope.showFieldDetails = true;
+ delete $scope.sampleDocId;
+ $scope.showAnalysis = false;
+
+ if (!$scope.treeFilter) {
+ $scope.treeFilter = "type";
+ $scope.treeFilterOption = "*";
+ $scope.initTreeFilters();
+ }
+ return;
+ }
+
+ var jst = $('#schemaJsTree').jstree(true);
+ if (!jst) {
+ return;
+ }
+
+ var node = jst.get_node(id);
+ if (!node) {
+ return;
+ }
+
+ if (id === "files") {
+ $scope.selectedNode = null;
+ $scope.isLeafNode = false;
+ return;
+ }
+
+ if (id.indexOf("/") === -1) {
+ $scope.selectedNode = null;
+ $scope.isLeafNode = false;
+ $scope.containerNode = id;
+
+ if (id === "fields") {
+ $scope.containerNodes = fieldNodes($scope.fieldsNode ? $scope.fieldsNode.children : $scope.fieldsSrc, "field");
+ } else if (id === "dynamicFields") {
+ $scope.containerNodes = fieldNodes($scope.dynamicFieldsNode ? $scope.dynamicFieldsNode.children : $scope.dynamicFieldsSrc, "dynamicField");
+ } else if (id === "fieldTypes") {
+ $scope.containerNodes = fieldNodes($scope.fieldTypes, "type");
+ }
+
+ $scope.containerNodeLabel = node.text;
+ $scope.showFieldDetails = true;
+ delete $scope.sampleDocId;
+ $scope.showAnalysis = false;
+ return;
+ }
+
+ if (id.startsWith("files/")) {
+ $scope.selectedNode = null;
+ $scope.isLeafNode = false;
+ delete $scope.sampleDocId;
+ $scope.showAnalysis = false;
+ if (node.children.length === 0) {
+ // file
+ $scope.showFieldDetails = true;
+ $scope.onSelectFileNode(id, false);
+ } else {
+ // folder
+ $scope.showFieldDetails = false;
+ delete $scope.selectedFile;
+ }
+ return;
+ }
+
+ delete $scope.selectedFile;
+ $scope.selectedNode = node["a_attr"]; // all the info we need is in the a_attr object
+ if (!$scope.selectedNode) {
+ // a node in the tree that isn't a field was selected, just ignore
+ return;
+ }
+
+ $scope.selectedNode.fieldType = getType($scope.selectedNode.type);
+ $scope.isLeafNode = true;
+
+ var nodeType = id.substring(0, id.indexOf("/"));
+ var name = null;
+ if (nodeType === "field") {
+ $scope.selectedType = "Field";
+ name = $scope.selectedNode.type;
+ } else if (nodeType === "dynamic") {
+ $scope.selectedType = "Dynamic Field";
+ } else if (nodeType === "type") {
+ $scope.selectedType = "Type";
+ name = $scope.selectedNode.name;
+ }
+
+ if (name) {
+ $scope.initTypeAnalysisInfo(name, "type");
+ }
+
+ // apply some sanity to the checkboxes
+ $scope.selectedNode = applyConstraintsOnField($scope.selectedNode);
+ $scope.showFieldDetails = true;
+
+ if (nodeType === "field" && $scope.selectedNode.tokenized && $scope.selectedNode.stored) {
+ $scope.showAnalysis = true;
+ $scope.updateSampleDocId();
+ } else {
+ $scope.showAnalysis = false;
+ $scope.indexText = "";
+ $scope.result = {};
+ }
+ };
+
+ function addFileNode(dirs, parent, f) {
+ var path = f.split("/");
+ if (path.length === 1) {
+ if (!parent.children) {
+ parent.children = [];
+ dirs.push(parent); // now parent has children, so track in dirs ...
+ }
+ var nodeId = parent.id + "/" + f;
+ parent.children.push({"text": f, "id": nodeId, "a_attr": {"href": nodeId}});
+ } else {
+ // get the parent for this path
+ var parentId = "files/" + path.slice(0, path.length - 1).join("/");
+ var dir = null;
+ for (var d in dirs) {
+ if (dirs[d].id === parentId) {
+ dir = dirs[d];
+ break;
+ }
+ }
+ if (!dir) {
+ dir = {"text": path[0], "id": parentId, "a_attr": {"href": parentId}, "children": []};
+ dirs.push(dir);
+ parent.children.push(dir);
+ }
+
+ // walk down the next level in this path
+ addFileNode(dirs, dir, path.slice(1).join("/"));
+ }
+ }
+
+ // transform a flat list structure into the nested tree structure
+ function filesToTree(files) {
+ var filesNode = {"text": "Files", "a_attr": {"href": "files"}, "id": "files", "children": []};
+ if (files) {
+ var dirs = []; // lookup for known dirs by path since nodes don't keep a ref to their parent node
+ for (var i in files) {
+ // hide the configoverlay.json from the UI
+ if (files[i] === "configoverlay.json") {
+ continue;
+ }
+
+ addFileNode(dirs, filesNode, files[i]);
+ }
+ delete dirs;
+ }
+ return filesNode;
+ }
+
+ function fieldsToTree(fields) {
+ var children = [];
+ if (fields) {
+ for (var i in fields) {
+ var id = "field/" + fields[i].name;
+ fields[i].href = id;
+ var text = fields[i].name;
+ if (fields[i].name === $scope.uniqueKeyField) {
+ text += "*"; // unique key field
+ }
+ children.push({"text": text, "a_attr": fields[i], "id": id});
+ }
+ }
+ return children;
+ }
+
+ function fieldTypesToTree(types) {
+ var children = [];
+ for (var i in types) {
+ var ft = types[i]
+ var id = "type/" + ft.name;
+ ft.href = id;
+ children.push({"text": ft.name, "a_attr": ft, "id": id});
+ }
+ return children;
+ }
+
+ $scope.onSampleDocumentsChanged = function () {
+ $scope.hasDocsOnServer = false; // so the updates get sent on next analyze action
+ };
+
+ $scope.initDesignerSettingsFromResponse = function (data) {
+ $scope.enableDynamicFields = data.enableDynamicFields !== null ? "" + data.enableDynamicFields : "true";
+ $scope.enableFieldGuessing = data.enableFieldGuessing !== null ? "" + data.enableFieldGuessing : "true";
+ $scope.enableNestedDocs = data.enableNestedDocs !== null ? "" + data.enableNestedDocs : "false";
+ $scope.languages = data.languages !== null && data.languages.length > 0 ? data.languages : ["*"];
+ $scope.copyFrom = data.copyFrom !== null ? data.copyFrom : "_default";
+ };
+
+ $scope.doAnalyze = function (nodeId) {
+ delete $scope.sampleMessage;
+
+ var schema = $scope.currentSchema;
+ if (schema) {
+ delete $scope.copyFrom;
+ } else {
+ schema = $scope.newSchema;
+ if (!$scope.copyFrom) {
+ $scope.copyFrom = "_default";
+ }
+ }
+
+ if (!schema) {
+ return;
+ }
+
+ var params = {path: "analyze", configSet: schema};
+ if ($scope.schemaVersion && $scope.schemaVersion !== -1) {
+ params.schemaVersion = $scope.schemaVersion;
+ }
+
+ if ($scope.enableDynamicFields) {
+ params.enableDynamicFields = $scope.enableDynamicFields;
+ }
+ if ($scope.enableFieldGuessing) {
+ params.enableFieldGuessing = $scope.enableFieldGuessing;
+ }
+ if ($scope.enableNestedDocs) {
+ params.enableNestedDocs = $scope.enableNestedDocs;
+ }
+
+ if ($scope.languages && $scope.languages.length > 0) {
+ params.languages = $scope.languages;
+ }
+
+ if ($scope.copyFrom) {
+ params.copyFrom = $scope.copyFrom;
+ }
+
+ $scope.updateWorking = true;
+ if ($scope.selectedUpdated) {
+ $scope.updateStatusMessage = "Applying " + $scope.selectedType + " updates ..."
+ } else {
+ $scope.updateStatusMessage = "Analyzing your sample data, schema will load momentarily ..."
+ }
+
+ if (!nodeId && $scope.selectedNode) {
+ nodeId = $scope.selectedNode.id;
+ }
+
+ // a bit tricky ...
+ // so users can upload a file or paste in docs
+ // if they upload a file containing a small (<15K) of text data, then we'll show it in the textarea
+ // they can change the text and re-analyze too
+ // if no changes or nothing uploaded, the server-side uses the latest sample data stored in the blob store
+ if ($scope.fileUpload) {
+ var file = $scope.fileUpload;
+ var fd = new FormData();
+ fd.append('file', file);
+ SchemaDesigner.upload(params, fd, function (data) {
+ $("#upload-file").val("");
+ delete $scope.fileUpload;
+ $scope.onSchemaUpdated(schema, data, nodeId);
+ }, $scope.errorHandler);
+ } else {
+ // don't need to keep re-posting the same sample if already stored in the blob store
+ var postData = null;
+ if (!$scope.hasDocsOnServer) {
+ postData = $scope.sampleDocuments;
+ if (!postData && !$scope.published) {
+ return;
+ }
+ }
+
+ var respHandler = function (data) {
+ $scope.onSchemaUpdated(schema, data, nodeId);
+ };
+
+ // TODO: need a better approach to detecting the content type from text content
+ var contentType = "text/plain";
+ if (postData != null) {
+ var txt = postData.trim();
+ if ((txt.startsWith("[") && txt.includes("]")) || (txt.startsWith("{") && txt.includes("}"))) {
+ contentType = "application/json"
+ } else if (txt.startsWith("<") || txt.includes("<add>") || txt.includes("<!--")) {
+ contentType = "text/xml";
+ } else {
+ contentType = "application/csv";
+ }
+ }
+
+ if (contentType === "text/xml") {
+ SchemaDesigner.postXml(params, postData, respHandler, $scope.errorHandler);
+ } else if (contentType === "application/csv") {
+ SchemaDesigner.postCsv(params, postData, respHandler, $scope.errorHandler);
+ } else {
+ SchemaDesigner.post(params, postData, respHandler, $scope.errorHandler);
+ }
+ }
+ };
+
+ $scope.onFieldTypeChanged = function () {
+
+ var copyFromType = ["stored", "indexed", "multiValued", "docValues", "useDocValuesAsStored", "tokenized", "uninvertible", "termVectors", "termPositions", "termOffsets",
+ "omitNorms", "omitTermFreqAndPositions", "omitPositions", "storeOffsetsWithPositions"];
+
+ var type = $scope.selectedNode.type
+
+ // when the user updates the selected field's type, we go apply the
+ // new field type's properties to the selected node
+ for (var i in $scope.fieldTypes) {
+ if ($scope.fieldTypes[i].text == type) {
+ var ft = $scope.fieldTypes[i];
+ $scope.selectedNode.fieldType = ft;
+ for (var i in copyFromType) {
+ var x = copyFromType[i];
+ if (ft.a_attr[x] !== null) {
+ $scope.selectedNode[x] = ft.a_attr[x];
+ } else {
+ delete $scope.selectedNode[x];
+ }
+ }
+ $scope.selectedUpdated = true;
+ break;
+ }
+ }
+
+ $scope.selectedNode = applyConstraintsOnField($scope.selectedNode);
+ if ($scope.selectedUpdated) {
+ // for luke analysis, we need the type info here, not the specific field into b/c it just changed.
+ $scope.initTypeAnalysisInfo(type, "type");
+ }
+ };
+
+ $scope.isDisabled = function (dep) {
+ if (!$scope.selectedNode) {
+ return false;
+ }
+
+ if (dep === "termVectors") {
+ // termVectors dependency depends on indexed
+ return !($scope.selectedNode.indexed && $scope.selectedNode.termVectors);
+ }
+
+ if (dep === "not-text" || dep === "docValues") {
+ if ($scope.selectedNode.fieldType && $scope.selectedNode.fieldType.a_attr.class === "solr.TextField") {
+ // no doc values for TextField
+ return true;
+ }
+ }
+
+ return $scope.selectedNode[dep] === false;
+ };
+
+ // this updates checkboxes based on the current settings
+ $scope.markSelectedUpdated = function (event) {
+ delete $scope.updateStatusMessage;
+ $scope.selectedUpdated = true; // enable the update button for this field
+ };
+
+ $scope.deleteSelected = function () {
+ // console.log(">> deleteSelected");
+ };
+
+ $scope.updateSelected = function () {
+ if (!$scope.selectedNode) {
+
+ if ($scope.selectedUpdated) {
+ // some root level property changed ... re-analyze
+ $scope.doAnalyze("/");
+ }
+
+ return;
+ }
+
+ delete $scope.updateSelectedError;
+
+ // make a copy for the PUT
+ var href = $scope.selectedNode.href;
+ var id = $scope.selectedNode.id;
+
+ var putData = JSON.parse(JSON.stringify($scope.selectedNode));
+ if ($scope.selectedType === "Field" && putData.copyDest) {
+ var fields = putData.copyDest.split(",");
+ for (var f in fields) {
+ var name = fields[f].trim();
+ if (!$scope.fieldNames.includes(name)) {
+ $scope.updateSelectedError = "Copy to field '" + name + "' doesn't exist!";
+ return;
+ }
+ if (name === $scope.selectedNode.name) {
+ $scope.updateSelectedError = "Cannot copy a field to itself!";
+ return;
+ }
+ }
+ } else {
+ delete putData.copyDest;
+ }
+
+ delete putData.fieldType;
+ delete putData.href;
+ delete putData.id;
+
+ putData = applyConstraintsOnField(putData);
+
+ if ($scope.analysisJsonText && !$scope.selectedNode.type) {
+ var text = $scope.analysisJsonText.trim();
+ if (text.length > 0) {
+ text = text.replace(/\s+/g, ' ');
+ if (!text.startsWith("{")) {
+ text = "{ " + text + " }";
+ }
+ try {
+ var textJson = JSON.parse(text);
+ if (textJson.analyzer) {
+ putData.analyzer = textJson.analyzer;
+ } else {
+ if (textJson.indexAnalyzer && textJson.queryAnalyzer) {
+ putData.indexAnalyzer = textJson.indexAnalyzer;
+ putData.queryAnalyzer = textJson.queryAnalyzer;
+ }
+ }
+ } catch (e) {
+ $scope.updateSelectedError = "Failed to parse analysis as JSON due to: " + e.message +
+ "; expected JSON object with either an 'analyzer' or 'indexAnalyzer' and 'queryAnalyzer'";
+ return;
+ }
+ }
+ }
+
+ $scope.updateWorking = true;
+ $scope.updateStatusMessage = "Updating " + $scope.selectedType + " ...";
+
+ SchemaDesigner.put({
+ path: "update",
+ configSet: $scope.currentSchema,
+ schemaVersion: $scope.schemaVersion
+ }, putData, function (data) {
+
+ var nodeType = data.updateType;
+ $scope.schemaVersion = data.schemaVersion;
+ $scope.currentSchema = data.configSet;
+ $scope.core = data.core;
+
+ $scope.selectedNode = data[nodeType];
+ $scope.selectedNode.href = href;
+ $scope.selectedNode.id = id;
+
+ var name = nodeType === "field" ? $scope.selectedNode.type : $scope.selectedNode.name;
+ $scope.initTypeAnalysisInfo(name, "type");
+ $scope.showFieldDetails = true;
+
+ if (nodeType === "field" && $scope.selectedNode.tokenized) {
+ $scope.showAnalysis = true;
+ $scope.updateSampleDocId();
+ }
+
+ $scope.onSchemaUpdated($scope.currentSchema, data, href);
+ }, $scope.errorHandler);
+ };
+
+ // TODO: These are copied from analysis.js, so move to a shared location for both vs. duplicating
+ var getShortComponentName = function (longname) {
+ var short = -1 !== longname.indexOf('$')
+ ? longname.split('$')[1]
+ : longname.split('.').pop();
+ return short.match(/[A-Z]/g).join('');
+ };
+
+ var getCaptionsForComponent = function (data) {
+ var captions = [];
+ for (var key in data[0]) {
+ key = key.replace(/.*#/, '');
+ if (key != "match" && key != "positionHistory") {
+ captions.push(key.replace(/.*#/, ''));
+ }
+ }
+ return captions;
+ };
+
+ var getTokensForComponent = function (data) {
+ var tokens = [];
+ var previousPosition = 0;
+ var index = 0;
+ for (var i in data) {
+ var tokenhash = data[i];
+ var positionDifference = tokenhash.position - previousPosition;
+ for (var j = positionDifference; j > 1; j--) {
+ tokens.push({position: tokenhash.position - j + 1, blank: true, index: index++});
+ }
+
+ var token = {position: tokenhash.position, keys: [], index: index++};
+
+ for (key in tokenhash) {
+ if (key == "match" || key == "positionHistory") {
+ //skip, to not display these keys in the UI
+ } else {
+ var tokenInfo = new Object();
+ tokenInfo.name = key;
+ tokenInfo.value = tokenhash[key];
+ if ('text' === key || 'raw_bytes' === key) {
+ if (tokenhash.match) {
+ tokenInfo.extraclass = 'match'; //to highlight matching text strings
+ }
+ }
+ token.keys.push(tokenInfo);
+ }
+ }
+ tokens.push(token);
+ previousPosition = tokenhash.position;
+ }
+ return tokens;
+ };
+
+ var extractComponents = function (data, result, name) {
+ if (data) {
+ result[name] = [];
+ for (var i = 0; i < data.length; i += 2) {
+ var component = {
+ name: data[i],
+ short: getShortComponentName(data[i]),
+ captions: getCaptionsForComponent(data[i + 1]),
+ tokens: getTokensForComponent(data[i + 1])
+ };
+ result[name].push(component);
+ }
+ }
+ };
+
+ var processFieldAnalysisData = function (analysis) {
+ var response = {};
+ extractComponents(analysis.index, response, "index");
+ return response;
+ };
+
+ $scope.doPublish = function () {
+ var params = {
+ path: "publish",
+ configSet: $scope.currentSchema,
+ schemaVersion: $scope.schemaVersion,
+ reloadCollections: $scope.reloadOnPublish,
+ cleanupTemp: true,
+ disableDesigner: $scope.disableDesigner
+ };
+ if ($scope.newCollection && $scope.newCollection.name) {
+ params.newCollection = $scope.newCollection.name;
+ params.numShards = $scope.newCollection.numShards;
+ params.replicationFactor = $scope.newCollection.replicationFactor;
+ params.indexToCollection = $scope.newCollection.indexToCollection;
+ }
+ SchemaDesigner.put(params, null, function (data) {
+ $scope.schemaVersion = data.schemaVersion;
+ $scope.currentSchema = data.configSet;
+
+ delete $scope.selectedNode;
+ $scope.currentSchema = "";
+ delete $scope.newSchema;
+ $scope.showPublish = false;
+ $scope.refresh();
+
+ if (data.newCollection) {
+ $window.location.href = "#/" + data.newCollection + "/collection-overview";
+ }
+ }, $scope.errorHandler);
+ };
+
+ $scope.downloadConfig = function () {
+ location.href = "/api/schema-designer/download/"+$scope.currentSchema+"_configset.zip?wt=raw&configSet=" + $scope.currentSchema;
+ };
+
+ function docsToTree(docs) {
+ var children = [];
+ for (var i in docs) {
+ var id = docs[i][$scope.uniqueKeyField];
+ if (!id) {
+ id = "" + i; // id not in results so use the position in results as the value
+ }
+ var nodeId = "doc/" + id;
+ docs[i].href = nodeId;
+ children.push({"text": id, "a_attr": docs[i], "id": nodeId});
+ }
+ return children;
+ }
+
+ function debugToTree(debugObj) {
+ var children = [];
+ for (var x in debugObj) {
+ if (typeof debugObj[x] === 'object') {
+ var obj = debugObj[x];
+ var nodeId = "debug/" + x;
+ var tdata = [];
+ for (var a in obj) {
+ if (typeof obj[a] !== 'object') {
+ tdata.push({name: a, value: obj[a]});
+ }
+ }
+ children.push({"text": x, "a_attr": {"href": nodeId}, "tdata": tdata, "id": nodeId, "children": []});
+ }
+ }
+ return children;
+ }
+
+ function facetsToTree(ff) {
+ var children = [];
+ for (var f in ff) {
+ var nodeId = "facet/" + f;
+ if (ff[f] && ff[f].length > 0) {
+ var facet = ff[f];
+ var tdata = [];
+ for (let i = 0; i < facet.length; i += 2) {
+ tdata.push({name: facet[i], value: facet[i + 1]});
+ }
+ children.push({"text": f, "a_attr": {"href": nodeId}, "tdata": tdata, "id": nodeId, "children": []});
+ }
+ }
+ return children;
+ }
+
+ function hlToTree(hl) {
+ var children = [];
+ for (var f in hl) {
+ var nodeId = "hl/" + f;
+ var tdata = [];
+ var obj = hl[f];
+ for (var a in obj) {
+ var v = obj[a];
+ if (v && v.length > 0) {
+ tdata.push({name: a, value: v[0]});
+ }
+ }
+ if (tdata.length > 0) {
+ children.push({"text": f, "a_attr": {"href": nodeId}, "tdata": tdata, "id": nodeId, "children": []});
+ }
+ }
+ return children;
+ }
+
+ $scope.selectField = function (event) {
+ var t = event.target || event.srcElement || event.currentTarget;
+ if (t && t.text) {
+ var nodeId = "field/" + t.text;
+ $scope.onSelectSchemaTreeNode(nodeId);
+ $scope.selectNodeInTree(nodeId);
+ }
+ };
+
+ $scope.selectFieldType = function (event) {
+ var t = event.target || event.srcElement || event.currentTarget;
+ if (t && t.text) {
+ var nodeId = "type/" + t.text;
+ $scope.onSelectSchemaTreeNode(nodeId);
+ $scope.selectNodeInTree(nodeId);
+ }
+ };
+
+ function omitVersionField(key,value) {
+ return (key === "_version_") ? undefined : value;
+ }
+
+ $scope.editDocuments = function() {
+ delete $scope.helpId;
+ if ($scope.queryDocs) {
+ $scope.hasDocsOnServer = false; // so the updated docs apply
+ $("#upload-file").val("");
+ delete $scope.fileUpload;
+ $scope.sampleDocuments = $scope.queryDocs;
+ $scope.onSampleDocumentsChanged();
+ }
+ };
+
+ $scope.renderResultsTree = function (data) {
+ var h = data.responseHeader;
+ var sort = h.params.sort;
+ if (!sort) {
+ sort = "score desc";
+ }
+
+ if (data.response && data.response.docs && data.response.docs.length > 0) {
+ $scope.queryDocs = JSON.stringify(data.response.docs, omitVersionField, 2);
+ } else {
+ delete $scope.queryDocs;
+ }
+
+ $scope.resultsMeta = [
+ {name: "Query", value: h.params.q},
+ {name: "QTime", value: h.QTime},
+ {name: "Hits", value: data.response.numFound},
+ {name: "sort", value: sort},
+ ];
+
+ var excParams = ["q", "handler", "debug", "configSet", "wt", "version", "_", "sort"];
+ for (var p in h.params) {
+ if (!excParams.includes(p)) {
+ $scope.resultsMeta.push({name: p, value: h.params[p]});
+ }
+ }
+
+ $scope.debugMeta = [];
+ for (var d in data.debug) {
+ if (typeof data.debug[d] !== 'object') {
+ var nvp = {name: d, value: data.debug[d]};
+ $scope.debugMeta.push(nvp);
+ $scope.resultsMeta.push(nvp);
+ }
+ }
+
+ var rootChildren = [{
+ "id":"docs",
+ "text": "Documents",
+ "state": {"opened": true},
+ "a_attr": {"href": "docs"},
+ "children": docsToTree(data.response.docs)
+ }];
+
+ if (data.facet_counts && data.facet_counts.facet_fields) {
+ rootChildren.push({
+ "id":"facets",
+ "text": "Facets",
+ "state": {"opened": true},
+ "a_attr": {"href": "facets"},
+ "children": facetsToTree(data.facet_counts.facet_fields)
+ });
+ }
+
+ if (data.highlighting) {
+ var hlNodes = hlToTree(data.highlighting);
+ if (hlNodes.length > 0) {
+ rootChildren.push({
+ "id":"hl",
+ "text": "Highlighting",
+ "state": {"opened": true},
+ "a_attr": {"href": "hl"},
+ "children": hlNodes
+ });
+ }
+ }
+
+ if (data.debug) {
+ rootChildren.push({"id":"debug", "text": "Debug", "a_attr": {"href": "debug"}, "children": debugToTree(data.debug)});
+ }
+
+ var tree = [{"id":"/","text": "Results", "a_attr": {"href": "/"}, "children": rootChildren}];
+ $scope.queryResultsTree = tree;
+ };
+
+ $scope.onSelectQueryResultsNode = function (id) {
+ $scope.selectedResultsNode = id;
+
+ if (id === "/" || id === "docs") {
+ $scope.resultsData = $scope.resultsMeta;
+ return;
+ }
+
+ if (id === "debug") {
+ $scope.resultsData = $scope.debugMeta;
+ return;
+ }
+
+ var jst = $('#queryResultsJsTree').jstree();
+ var node = jst.get_node(id);
+ if (!node || !node.a_attr) {
+ $scope.resultsData = $scope.resultsMeta;
+ return;
+ }
+
+ if (node.original && node.original.tdata) {
+ $scope.resultsData = node.original.tdata;
+ } else {
+ $scope.resultsData = [];
+ for (var a in node.a_attr) {
+ if (a === "href") continue;
+ var row = {name: a, value: node.a_attr[a]};
+ if (id.startsWith("doc/")) {
+ row.type = "f"; // so we can link to fields in the schema tree from results!
+ }
+
+ $scope.resultsData.push(row);
+ }
+ }
+
+ if (id.startsWith("doc/")) {
+ $scope.sampleDocId = id.substring(4);
+ $scope.updateSampleDocId();
+ }
+ };
+
+ $scope.doQuery = function () {
+
+ delete $scope.queryDocs;
+
+ var params = {path: "query", configSet: $scope.currentSchema, debug: "true", "wt": "json"};
+
+ if ($scope.selectedFacets && $scope.selectedFacets.length > 0) {
+ params["facet"] = true;
+ params["facet.field"] = $scope.selectedFacets;
+ params["facet.limit"] = 20;
+ params["facet.mincount"] = 1;
+ } else {
+ params["facet"] = false;
+ delete params["facet.field"];
+ }
+
+ var set = function (key, value) {
+ if (params[key]) {
+ params[key].push(value);
+ } else {
+ params[key] = [value];
+ }
+ }
+
+ params["sort"] = $scope.query.sortBy + " " + $scope.query.sortDir;
+ params["q"] = $scope.query.q.trim();
+ if (!params["q"]) {
+ params["q"] = "*:*";
+ }
+
+ if ($scope.rawParams) {
+ var rawParams = $scope.rawParams.split(/[&\n]/);
+ for (var i in rawParams) {
+ var param = rawParams[i];
+ var equalPos = param.indexOf("=");
+ if (equalPos > -1) {
+ set(param.substring(0, equalPos), param.substring(equalPos + 1));
+ } else {
+ set(param, ""); // Use empty value for params without "="
+ }
+ }
+ }
+
+ if (params["q"] !== '*:*' && $scope.query.highlight) {
+ params["hl"] = true;
+ params["hl.fl"] = $scope.query.highlight;
+ if (!params["hl.method"]) {
+ // lookup the field props
+ var method = "unified";
+ var field = $scope.fields.find(f => f.name === $scope.query.highlight);
+ if (field) {
+ if (field.termVectors && field.termOffsets && field.termPositions) {
+ method = "fastVector";
+ }
+ }
+ params["hl.method"] = method;
+ }
+ } else {
+ delete params["hl"];
+ delete params["hl.fl"];
+ }
+
+ var qt = params["qt"] ? params["qt"] : "/select";
+ if (qt[0] === '/') {
+ params.handler = qt.substring(1);
+ } else { // Support legacy style handleSelect=true configs
+ params.handler = "select";
+ params["qt"] = qt;
+ }
+
+ SchemaDesigner.get(params, function (data) {
+ $("#sort").trigger("chosen:updated");
+ $("#ff").trigger("chosen:updated");
+ $("#hl").trigger("chosen:updated");
+
+ $scope.renderResultsTree(data);
+
+ var nodeId = "/";
+ if ($scope.sampleDocId) {
+ if (data.response.docs) {
+ var hit = data.response.docs.find(d => d[$scope.uniqueKeyField] === $scope.sampleDocId);
+ if (hit) {
+ nodeId = "doc/"+$scope.sampleDocId;
+ }
+ }
+ }
+ $scope.onSelectQueryResultsNode(nodeId);
+ });
+ };
+
+ $scope.toggleShowAnalysisJson = function () {
+ if ($scope.showAnalysisJson) {
+ $scope.showAnalysisJson = false;
+ $scope.editAnalysis = "Edit JSON";
+ } else {
+ $scope.showAnalysisJson = true;
+ $scope.editAnalysis = "Hide JSON";
+
+ var node = $scope.selectedNode;
+ var analysisJson = {};
+ if (node.analyzer) {
+ analysisJson.analyzer = node.analyzer;
+ } else {
+ if (node.indexAnalyzer) {
+ analysisJson.indexAnalyzer = node.indexAnalyzer;
+ }
+ if (node.queryAnalyzer) {
+ analysisJson.queryAnalyzer = node.queryAnalyzer;
+ }
+ }
+ $scope.analysisJsonText = JSON.stringify(analysisJson, null, 2);
+ }
+ };
+
+ function filterFieldsByType(fieldsSrc, typeFilter) {
+ var children = [];
+ for (var f in fieldsSrc) {
+ var field = fieldsSrc[f];
+ if (field.a_attr && field.a_attr.type === typeFilter) {
+ children.push(field);
+ }
+ }
+ return children;
+ }
+
+ function filterFieldsByFeature(fieldsSrc, opt, enabled) {
+ var children = [];
+ var isEnabled = enabled === "true";
+ for (var f in fieldsSrc) {
+ var field = fieldsSrc[f];
+ if (!field.a_attr) {
+ continue;
+ }
+ var attr = field.a_attr;
+ if (opt === "indexed") {
+ if (attr.indexed === isEnabled) {
+ children.push(field);
+ }
+ } else if (opt === "text") {
+ if (attr.tokenized === isEnabled) {
+ children.push(field);
+ }
+ } else if (opt === "facet") {
+ if (((attr.indexed || attr.docValues) && !attr.tokenized && attr.name !== '_version_') === isEnabled) {
+ children.push(field);
+ }
+ } else if (opt === "highlight") {
+ if ((attr.stored && attr.tokenized) === isEnabled) {
+ children.push(field);
+ }
+ } else if (opt === "sortable") {
+ if (((attr.indexed || attr.docValues) && !attr.multiValued) === isEnabled) {
+ children.push(field);
+ }
+ } else if (opt === "docValues") {
+ if (attr.docValues === isEnabled) {
+ children.push(field);
+ }
+ } else if (opt === "stored") {
+ if ((attr.stored || (attr.docValues && attr.useDocValuesAsStored)) === isEnabled) {
+ children.push(field);
+ }
+ }
+ }
+ return children;
+ }
+
+ $scope.findSelectedNodeId = function() {
+ var nodeId = "/";
+ if ($scope.fieldsNode.children.length > 0) {
+ if ($scope.selectedNode) {
+ var found = $scope.fieldsNode.children.find(n => n.a_attr.name === $scope.selectedNode.name);
+ if (found) {
+ nodeId = $scope.selectedNode.href;
+ } else {
+ delete $scope.selectedNode;
+ nodeId = $scope.fieldsNode.children[0].a_attr.href;
+ }
+ } else {
+ nodeId = $scope.fieldsNode.children[0].a_attr.href;
+ }
+ }
+ return nodeId;
+ };
+
+ $scope.onTreeFilterOptionChanged = function() {
+ if (!$scope.fieldsNode) {
+ return;
+ }
+
+ if (!$scope.treeFilter || !$scope.treeFilterOption || $scope.treeFilterOption === "*") {
+ // restore tree to unfiltered state
+ $scope.fieldsNode.children = $scope.fieldsSrc;
+ if ($scope.dynamicFieldsNode) {
+ $scope.dynamicFieldsNode.children = $scope.dynamicFieldsSrc;
+ }
+ $scope.refreshTree();
+ return;
+ }
+
+ if ($scope.treeFilter === "type") {
+ $scope.fieldsNode.children = filterFieldsByType($scope.fieldsSrc, $scope.treeFilterOption);
+ if ($scope.dynamicFieldsNode) {
+ $scope.dynamicFieldsNode.children = filterFieldsByType($scope.dynamicFieldsSrc, $scope.treeFilterOption);
+ }
+ } else if ($scope.treeFilter === "feature") {
+ $scope.fieldsNode.children = filterFieldsByFeature($scope.fieldsSrc, $scope.treeFilterOption, $scope.treeFilterFeatureEnabled);
+ if ($scope.dynamicFieldsNode) {
+ $scope.dynamicFieldsNode.children = filterFieldsByFeature($scope.dynamicFieldsSrc, $scope.treeFilterOption, $scope.treeFilterFeatureEnabled);
+ }
+ } else {
+ // otherwise, restore the tree to original state
+ $scope.fieldsNode.children = $scope.fieldsSrc;
+ if ($scope.dynamicFieldsNode) {
+ $scope.dynamicFieldsNode.children = $scope.dynamicFieldsSrc;
+ }
+ }
+ $scope.refreshTree();
+
+ var nodeId = $scope.findSelectedNodeId();
+ $scope.onSelectSchemaTreeNode(nodeId);
+ $scope.selectNodeInTree(nodeId);
+ };
+
+ $scope.initTreeFilters = function() {
+ $scope.treeFilterFeatureEnabled = "true";
+ $scope.treeFilterOptions = [];
+ $scope.treeFilterOption = "";
+ if ($scope.treeFilter === "type") {
+ var usedFieldTypes = [];
+ if ($scope.fields) {
+ var usedTypes = $scope.fields.map(f => f.type);
+ for (var t in usedTypes) {
+ if (!usedFieldTypes.includes(usedTypes[t])) {
+ usedFieldTypes.push(usedTypes[t]);
+ }
+ }
+ }
+ $scope.treeFilterOptions = usedFieldTypes.sort();
+ $scope.treeFilterOptions.unshift("*");
+ $scope.treeFilterOption = "*";
+ } else if ($scope.treeFilter === "feature") {
+ $scope.treeFilterOptions = ["indexed","text","facet","highlight","sortable","docValues","stored"].sort();
+ $scope.treeFilterOptions.unshift("*");
+ $scope.treeFilterOption = "*";
+ }
+ };
+
+ $scope.applyTreeFilterOption = function() {
+ if (!$scope.fields) {
+ return;
+ }
+ $scope.initTreeFilters();
+ $scope.onTreeFilterOptionChanged();
+ };
+
+ $scope.refresh();
+})
diff --git a/solr/webapp/web/js/angular/services.js b/solr/webapp/web/js/angular/services.js
index d8e39efd320..4d2c3c3138f 100644
--- a/solr/webapp/web/js/angular/services.js
+++ b/solr/webapp/web/js/angular/services.js
@@ -259,6 +259,17 @@ solrAdminServices.factory('System',
get: {method: "GET"}
})
}])
+.factory('SchemaDesigner',
+ ['$resource', function($resource) {
+ return $resource('/api/schema-designer/:path', {wt: 'json', path: '@path', _:Date.now()}, {
+ get: {method: "GET"},
+ post: {method: "POST", timeout: 90000},
+ put: {method: "PUT"},
+ postXml: {headers: {'Content-type': 'text/xml'}, method: "POST", timeout: 90000},
+ postCsv: {headers: {'Content-type': 'application/csv'}, method: "POST", timeout: 90000},
+ upload: {method: "POST", transformRequest: angular.identity, headers: {'Content-Type': undefined}, timeout: 90000}
+ })
+}])
.factory('AuthenticationService',
['base64', function (base64) {
var service = {};
diff --git a/solr/webapp/web/partials/schema-designer.html b/solr/webapp/web/partials/schema-designer.html
new file mode 100644
index 00000000000..88f99cc161d
--- /dev/null
+++ b/solr/webapp/web/partials/schema-designer.html
@@ -0,0 +1,1030 @@
+<!--
+Licensed to the Apache Software Foundation (ASF) under one or more
+contributor license agreements. See the NOTICE file distributed with
+this work for additional information regarding copyright ownership.
+The ASF licenses this file to You under the Apache License, Version 2.0
+(the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+<div id="designer">
+ <div class="clearfix">
+ <div id="designer-top">
+ <div id="actions" class="actions">
+
+ <div id="schema-selector">
+ <div class="left">
+ <select id="select-schema" placeholder-text-single="'Schema Selector'"
+ chosen
+ ng-model="currentSchema"
+ ng-change="loadSchema()"
+ ng-options="schema for schema in schemas"></select>
+ <button id="add" class="action" ng-click="showNewSchemaDialog()"><span>New Schema</span></button>
+ </div>
+ <div class="middle">
+ <div id="schema-actions" class="schema-actions clearfix">
+ <button id="addField" class="action" ng-click="toggleAddField('field')" ng-show="showSchemaActions"><span>Add Field</span></button>
+ <button id="addFieldType" class="action" ng-click="toggleAddField('type')" ng-show="showSchemaActions"><span>Add Field Type</span></button>
+ <div id="updateStatusMessage" ng-class="{working: updateWorking}" ng-show="updateStatusMessage"><span>{{updateStatusMessage}}</span></div>
+
+ <div class="action add" data-rel="add" ng-show="showAddField" escape-pressed="hideAll()">
+
+ <p class="clearfix"><label for="add_name">name:</label>
+ <input type="text" id="add_name" ng-model="newField.name" focus-when="showAddField" placeholder="enter a name"></p>
+
+ <p class="clearfix" ng-show="adding=='field'"><label for="add_type">field type:</label>
+ <select chosen type="text" id="add_type" ng-model="newField.type" ng-options="type for type in types"></select>
+ </p>
+ <p class="clearfix" ng-show="adding=='type'"><label for="add_class">class:</label>
+ <input type="text" id="add_class" ng-model="newField.class" placeholder="class name"></p>
+
+ <p class="clearfix" ng-show="adding=='field'"><label for="add_default">default:</label>
+ <input type="text" id="add_default" ng-model="newField.default" placeholder="enter a default value if needed"></p>
+
+ <p class="clearfix">
+ <label class="checkbox" for="add_stored">
+ <input type="checkbox" ng-model="newField.stored" id="add_stored" title="Full field should be stored in index." ng-true-value="'true'" ng-false-value="'false'">
+ stored
+ </label>
+ </p>
+
+ <p class="clearfix">
+ <label class="checkbox" for="add_indexed">
+ <input type="checkbox" ng-model="newField.indexed" id="add_indexed" title="Field should be indexed." ng-true-value="'true'" ng-false-value="'false'">
+ indexed
+ </label>
+ </p>
+
+ <p class="clearfix">
+ <label class="checkbox" for="add_uninvertible">
+ <input type="checkbox" ng-model="newField.uninvertible" id="add_uninvertible" title="Field should be uninvertible, it is generally recomended to use docValues instead."
+ ng-true-value="'true'" ng-false-value="'false'">
+ uninvertible
+ </label>
+ </p>
+
+ <p class="clearfix">
+ <label class="checkbox" for="add_docValues">
+ <input type="checkbox" ng-model="newField.docValues" id="add_docValues" title="DocValues should be stored for the field." ng-true-value="'true'" ng-false-value="'false'">
+ docValues
+ </label>
+ </p>
+
+ <p class="clearfix">
+ <label class="checkbox" for="add_multiValued">
+ <input type="checkbox" ng-model="newField.multiValued" id="add_multiValued" title="Multiple values are allowed for this field." ng-true-value="'true'" ng-false-value="'false'">
+ multiValued
+ </label>
+ </p>
+
+ <p class="clearfix" ng-show="adding=='field'">
+ <label class="checkbox" for="add_required">
+ <input type="checkbox" ng-model="newField.required" id="add_required" title="Field must be provided for all documents." ng-true-value="'true'" ng-false-value="'false'">
+ required
+ </label>
+ </p>
+
+ <p class="clearfix">
+ <a ng-click="showOmit=!showOmit">
+ <span class="add_showhide" ng-hide="showOmit">Show omit options</span>
+ <span class="add_showhide open" ng-show="showOmit">Hide omit options</span>
+ </a>
+ </p>
+
+ <div ng-show="showOmit">
+
+ <p class="clearfix">
+ <label class="checkbox" for="add_omitNorms">
+ <input type="checkbox" ng-model="newField.omitNorms" id="add_omitNorms" title="Full field should be omitNorms in index." ng-true-value="'true'" ng-false-value="'false'">
+ omitNorms
+ </label>
+ </p>
+
+ <p class="clearfix">
+ <label class="checkbox" for="add_omitTermFreqAndPositions">
+ <input type="checkbox" ng-model="newField.omitTermFreqAndPositions" id="add_omitTermFreqAndPositions" title="Full field should be omitTermFreqAndPositions in index."
+ ng-true-value="'true'" ng-false-value="'false'">
+ omitTermFreqAndPositions
+ </label>
+ </p>
+
+ <p class="clearfix">
+ <label class="checkbox" for="add_omitPositions">
+ <input type="checkbox" ng-model="newField.omitPositions" id="add_omitPositions" title="Full field should be omitPositions in index." ng-true-value="'true'" ng-false-value="'false'">
+ omitPositions
+ </label>
+ </p>
+ </div>
+
+ <p class="clearfix">
+ <a ng-click="showTermVectors=!showTermVectors">
+ <span class="add_showhide" ng-hide="showTermVectors">Show term vector options</span>
+ <span class="add_showhide open" ng-show="showTermVectors">Hide term vector options</span>
+ </a>
+ </p>
+ <div ng-show="showTermVectors">
+
+ <p class="clearfix">
+ <label class="checkbox" for="add_termVectors">
+ <input type="checkbox" ng-model="newField.termVectors" id="add_termVectors" title="Full field should be termVectors in index." ng-true-value="'true'" ng-false-value="'false'">
+ termVectors
+ </label>
+ </p>
+
+ <p class="clearfix">
+ <label class="checkbox" for="add_termPositions">
+ <input type="checkbox" ng-model="newField.termPositions" id="add_termPositions" title="Full field should be termPositions in index." ng-true-value="'true'" ng-false-value="'false'">
+ termPositions
+ </label>
+ </p>
+
+ <p class="clearfix">
+ <label class="checkbox" for="add_termOffsets">
+ <input type="checkbox" ng-model="newField.termOffsets" id="add_termOffsets" title="Full field should be termOffsets in index." ng-true-value="'true'" ng-false-value="'false'">
+ termOffsets
+ </label>
+ </p>
+
+ <p class="clearfix">
+ <label class="checkbox" for="add_termPayloads">
+ <input type="checkbox" ng-model="newField.termPayloads" id="add_termPayloads" title="Full field should be termPayloads in index." ng-true-value="'true'" ng-false-value="'false'">
+ termPayloads
+ </label>
+ </p>
+
+ </div>
+
+ <p class="clearfix">
+ <a ng-click="showSort=!showSort">
+ <span class="add_showhide" ng-hide="showSort">Show sort options</span>
+ <span class="add_showhide open" ng-show="showSort">Show sort options</span>
+ </a>
+ </p>
+ <div ng-show="showSort">
+ <p class="clearfix">
+ <label class="checkbox" for="add_sortMissingFirst">
+ <input type="checkbox" ng-model="newField.sortMissingFirst" id="add_sortMissingFirst" title="Full field should be sortMissingFirst in index." ng-true-value="'true'"
+ ng-false-value="'false'">
+ sortMissingFirst
+ </label>
+ </p>
+
+ <p class="clearfix">
+ <label class="checkbox" for="add_sortMissingLast">
+ <input type="checkbox" ng-model="newField.sortMissingLast" id="add_sortMissingLast" title="Full field should be sortMissingLast in index." ng-true-value="'true'"
+ ng-false-value="'false'">
+ sortMissingLast
+ </label>
+ </p>
+
+ </div>
+
+ <p class="clearfix" ng-show="adding=='type'">
+ <a ng-click="showAnalysisJson=!showAnalysisJson">
+ <span class="add_showhide" ng-hide="showAnalysisJson">Show text analysis JSON</span>
+ <span class="add_showhide open" ng-show="showAnalysisJson">Hide text analysis JSON</span>
+ </a>
+ </p>
+ <div ng-show="showAnalysisJson">
+ <textarea ng-model="textAnalysisJson" name="add_analysis_json" id="add_analysis_json" title="Text Analysis JSON" rows="8" cols="40" placeholder=""></textarea>
+ </div>
+
+ <p class="clearfix buttons">
+ <button type="submit" class="submit" ng-class="{success: added}" ng-click="addField()"><span>Add</span></button>
+ <button type="reset" class="reset" ng-click="toggleAddField()"><span>Cancel</span></button>
+ </p>
+
+ <div id="add-errors">
+ <div ng-repeat="error in addErrors" ng-show="addErrors" class="error"><span>{{error}}</span></div>
+ </div>
+ </div>
+ </div>
+ </div>
+ <div class="right">
+ <div ng-show="showSchemaActions">
+ <button class="action diff-button" ng-click="toggleDiff($event)"><span>Show Changes</span></button>
+ <button class="action publish-button" ng-click="togglePublish($event)"><span>Publish</span></button>
+ <button id="download-button" class="action" ng-click="downloadConfig()"><span>Download Config</span></button>
+ </div>
+ </div>
+ </div>
+
+ <div id="error-dialog" class="error-dialog" ng-show="designerAPIError">
+ <div id="error-dialog-note"><p class="clearfix"><img src="img/ico/prohibition.png"/>&nbsp;{{designerAPIError}}</p></div>
+ <div id="error-dialog-details"><div ng-show="designerAPIErrorDetails"><textarea rows="10" cols="55">{{designerAPIErrorDetails}}</textarea></div></div>
+ <div id="error-dialog-buttons" class="clearfix">
+ <button type="reset" class="reload-error-button" ng-click="closeErrorDialog()" ng-show="isVersionMismatch"><span>Reload Schema</span></button>
+ <button type="reset" class="error-button" ng-click="closeErrorDialog()" ng-show="!isVersionMismatch"><span>OK</span></button>
+ </div>
+ </div>
+
+ <div id="warn-dialog" class="warn-dialog" ng-show="apiWarning">
+ <div id="warn-dialog-note"><p class="clearfix"><img src="img/ico/exclamation-button.png"/>&nbsp;{{apiWarning}}</p></div>
+ <div id="warn-dialog-details"><div ng-show="apiWarningDetails"><textarea rows="10" cols="55">{{apiWarningDetails}}</textarea></div></div>
+ <div id="warn-dialog-buttons" class="clearfix">
+ <button type="reset" class="warn-button" ng-click="closeWarnDialog()"><span>OK</span></button>
+ </div>
+ </div>
+
+ <div id="show-diff-dialog" class="diff" ng-show="showDiff">
+ <div ng-hide="schemaDiffExists" ng-show="!schemaDiffExists">
+ <p class="clearfix diff-text">No differences with {{diffSource}} found.</p>
+ </div>
+ {{fieldsDiff=schemaDiff.fieldsDiff;""}}
+ <p class="clearfix diff-text" ng-show="schemaDiffExists">Found differences with {{diffSource}}:</p>
+ <div id="fields-diff" ng-hide="fieldsDiff == null" ng-show="Object.keys(fieldsDiff).length > 0">
+ <div id="fields-updated" ng-hide="fieldsDiff.updated == null" ng-show="Object.keys(fieldsDiff.updated).length > 0">
+ <p class="clearfix diff-text">Updated Fields</p>
+ <table class="diff">
+ <tr>
+ <th>Name</th>
+ <th>Old</th>
+ <th>New</th>
+ </tr>
+ <tr ng-repeat="(field, fieldDiff) in fieldsDiff.updated">
+ <td>{{ field }}</td>
+ <td><p class="old-value">{{fieldDiff[0] | json}}</p></td>
+ <td><p class="new-value">{{fieldDiff[1] | json}}</p></td>
+ </tr>
+ </table>
+ </div>
+ <div id="fields-added" ng-show="schemaDiff.addedFields.length > 0">
+ <p class="clearfix diff-text">New Fields</p>
+ <table border="0" cellspacing="0" cellpadding="0">
+ <tbody>
+ <tr ng-class="{odd:$odd}">
+ <th class="prop-hdr">Name</th>
+ <th class="prop-hdr">Type</th>
+ <th class="prop-hdr-flag">Indexed</th>
+ <th class="prop-hdr-flag">Stored</th>
+ <th class="prop-hdr-flag">MultiValued</th>
+ <th class="prop-hdr-flag">DocValues</th>
+ <th class="prop-hdr-flag">Tokenized</th>
+ </tr>
+ <tr ng-repeat="f in schemaDiff.addedFields" ng-class="{odd:$odd}">
+ <td class="prop">{{f.name}}</td>
+ <td class="prop">{{f.type}}</td>
+ <td class="prop-flag"><span ng-show="f.indexed"><img src="img/ico/tick.png"/></span></td>
+ <td class="prop-flag"><span ng-show="f.stored"><img src="img/ico/tick.png"/></span></td>
+ <td class="prop-flag"><span ng-show="f.multiValued"><img src="img/ico/tick.png"/></span></td>
+ <td class="prop-flag"><span ng-show="f.docValues"><img src="img/ico/tick.png"/></span></td>
+ <td class="prop-flag"><span ng-show="f.tokenized"><img src="img/ico/tick.png"/></span></td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ <div id="fields-removed" ng-show="schemaDiff.removedFields.length > 0">
+ <p class="clearfix diff-text">Removed Fields</p>
+ <ul>
+ <li class="clearfix element" ng-repeat="f in schemaDiff.removedFields">
+ <p class="rem-value"> {{f}} </p>
+ </li>
+ </ul>
+ </div>
+ </div>
+ {{fieldTypesDiff=schemaDiff.fieldTypesDiff;""}}
+ <div id="fieldtypes-diff" ng-hide="fieldTypesDiff == null" ng-show="Object.keys(fieldTypesDiff).length > 0">
+ <div id="fieldtypes-updated" ng-hide="fieldTypesDiff.updated == null" ng-show="Object.keys(fieldTypesDiff.updated).length > 0">
+ <p class="clearfix diff-text">Updated Field Types</p>
+ <table class="diff" ng-show="Object.keys(fieldTypesDiff.updated).length > 0">
+ <tr>
+ <th>Name</th>
+ <th>Old</th>
+ <th>New</th>
+ </tr>
+ <tr ng-repeat="(fieldType, fieldTypeDiff) in fieldTypesDiff.updated">
+ <td>{{ fieldType }}</td>
+ <td><p class="old-value">{{fieldTypeDiff[0] | json}}</p></td>
+ <td><p class="new-value">{{fieldTypeDiff[1] | json}}</p></td>
+ </tr>
+ </table>
+ </div>
+ <div id="fieldtypes-added" ng-hide="fieldTypesDiff.added == null" ng-show="Object.keys(fieldTypesDiff.added).length > 0">
+ <p class="clearfix diff-text">New Field Types</p>
+ <ul>
+ <li class="clearfix element" ng-repeat="fieldtype in fieldTypesDiff.added">
+ <p class="new-value"> {{fieldtype | json}} </p>
+ </li>
+ </ul>
+ </div>
+ <div id="fieldtypes-removed" ng-show="schemaDiff.removedTypes.length > 0">
+ <p class="clearfix diff-text">Removed Field Types</p>
+ <ul>
+ <li class="clearfix element" ng-repeat="fieldtype in schemaDiff.removedTypes">
+ <p class="rem-value"> {{fieldtype}} </p>
+ </li>
+ </ul>
+ </div>
+ </div>
+ {{dynamicFieldsDiff=schemaDiff.dynamicFieldsDiff;""}}
+ <div id="dynamicfields-diff" ng-hide="dynamicFieldsDiff == null" ng-show="Object.keys(dynamicFieldsDiff).length > 0">
+ <div id="dynamicfields-updated" ng-hide="dynamicFieldsDiff.updated == null" ng-show="Object.keys(dynamicFieldsDiff.updated).length > 0">
+ <p class="clearfix diff-text">Updated Dynamic Fields</p>
+ <table class="diff">
+ <tr>
+ <th>Name</th>
+ <th>Old</th>
+ <th>New</th>
+ </tr>
+ <tr ng-repeat="(dfield, dfieldDiff) in dynamicFieldsDiff.updated">
+ <td>{{ dfield }}</td>
+ <td><p class="old-value">{{dfieldDiff[0] | json}}</p></td>
+ <td><p class="new-value">{{dfieldDiff[1] | json}}</p></td>
+ </tr>
+ </table>
+ </div>
+ <div id="dynamicfields-added" ng-hide="dynamicFieldsDiff.added == null" ng-show="Object.keys(dynamicFieldsDiff.added).length > 0">
+ <p class="clearfix diff-text">New Dynamic Fields</p>
+ <ul>
+ <li class="clearfix" ng-repeat="dfield in dynamicFieldsDiff.added">
+ <p class="new-value"> {{dfield | json}} </p>
+ </li>
+ </ul>
+ </div>
+ <div id="dynamicfields-removed" ng-hide="dynamicFieldsDiff.removed == null" ng-show="Object.keys(dynamicFieldsDiff.removed).length > 0">
+ <p class="clearfix diff-text">Removed Dynamic Fields</p>
+ <ul>
+ <li class="clearfix" ng-repeat="dfield in dynamicFieldsDiff.removed">
+ <p class="rem-value"> {{dfield.name}} </p>
+ </li>
+ </ul>
+ </div>
+ </div>
+ {{copyFieldsDiff=schemaDiff.copyFieldsDiff;""}}
+ <div id="copyfields-diff" ng-hide="copyFieldsDiff == null" ng-show="Object.keys(copyFieldsDiff).length > 0">
+ <p class="clearfix diff-text">Copy Fields</p>
+ <table class="diff" ng-show="copyFieldsDiff.old.length > 0 || copyFieldsDiff.new.length > 0">
+ <tr>
+ <th>Source</th>
+ <th>Destination</th>
+ </tr>
+ <tr ng-show="copyFieldsDiff.old.length > 0" ng-repeat="cfield in copyFieldsDiff.old">
+ <td><p class="old-value">{{cfield.source}}</p></td>
+ <td><p class="old-value">{{cfield.dest}}</p></td>
+ </tr>
+ <tr ng-show="copyFieldsDiff.new.length > 0" ng-repeat="cfield in copyFieldsDiff.new">
+ <td><p class="new-value">{{cfield.source}}</p></td>
+ <td><p class="new-value">{{cfield.dest}}</p></td>
+ </tr>
+ </table>
+ </div>
+
+ <div class="clearfix diff-buttons">
+ <button type="reset" class="reset" ng-click="toggleDiff()"><span>Close</span></button>
+ </div>
+ </div>
+
+ <div id="publish-dialog" class="publish" ng-show="showPublish" escape-pressed="hideAll()">
+ <div id="publish-dialog-note"><p class="clearfix">Publish schema and associated configs to Zookeeper as a Solr ConfigSet.</p></div>
+ <div id="publish-affected">
+ <p class="clearfix">Affected collections:</p>
+ <ul>
+ <li class="clearfix" ng-repeat="coll in collectionsForConfig">{{coll}}</li>
+ </ul>
+ <div ng-show="collectionsForConfig.length == 0" class="clearfix">No existing collections</div>
+ </div>
+ <div id="reload-form">
+ <div class="field-form" ng-show="collectionsForConfig.length > 0">
+ <input type="checkbox" ng-model="reloadOnPublish" id="reload_on_publish" title="reload affected collections" ng-true-value="'true'" ng-false-value="'false'"/>
+ <label for="reload_on_publish" class="checkbox" title="Reload affected collections">Reload affected collections?</label>
+ </div>
+ </div>
+
+ <div id="publish-new-coll">
+ <p>Add new collection with published config?</p>
+ <form>
+ <p class="clearfix"><label for="add_coll_name">name:</label>
+ <input type="text" name="name" id="add_coll_name" ng-model="newCollection.name" placeholder="new collection"></p>
+
+ <p class="clearfix"><label for="add_coll_numShards">numShards:</label>
+ <input type="text" name="numShards" id="add_coll_numShards" ng-model="newCollection.numShards"></p>
+
+ <p class="clearfix"><label for="add_coll_replicationFactor">replicationFactor:</label>
+ <input type="text" name="replicationFactor" id="add_coll_replicationFactor" ng-model="newCollection.replicationFactor"></p>
+ </form>
+ <div class="field-form">
+ <input type="checkbox" ng-model="newCollection.indexToCollection" id="add_coll_index_sample" title="index sample docs" ng-true-value="'true'" ng-false-value="'false'"/>
+ <label for="add_coll_index_sample" class="checkbox" title="Index sample docs">Index sample docs in new collection?</label>
+ </div>
+ </div>
+ <div class="field-form">
+ <input type="checkbox" ng-model="disableDesigner" id="disable_designer" title="Disable future changes in schema designer" ng-true-value="'true'" ng-false-value="'false'"/>
+ <label for="disable_designer" class="checkbox" title="Disable future changes in schema designer">Disable future changes by the schema designer?</label>
+ </div>
+ <div ng-repeat="error in publishErrors" ng-show="publishErrors" class="clearfix note error"><span>{{error}}</span></div>
+ <div class="clearfix publish-buttons">
+ <button type="submit" class="action publish-button" ng-click="doPublish()"><span>Publish</span></button>
+ <button type="reset" class="reset" ng-click="togglePublish()"><span>Cancel</span></button>
+ </div>
+ </div>
+
+ <div id="confirm-dialog" class="action add" data-rel="add" ng-show="showConfirmEditSchema" style="display:block;">
+ <p class="clearfix warn"><span>Warning: You've chosen to load an existing schema that is already being used by active collections. Making changes to the '{{confirmSchema}}' schema will impact existing documents in these collections. Please proceed with caution.<br/><br/></span></p>
+ <p class="clearfix">Collections using the {{confirmSchema}} schema:</p>
+ <ul id="confirm-colls"><li class="clearfix" ng-repeat="coll in collectionsForConfig">{{coll}}</li></ul>
+ <p class="clearfix buttons">
+ <button id="confirm-edit" type="submit" class="submit" ng-click="confirmEditSchema()"><span>Proceed, I understand the risks</span></button>
+ <button type="reset" class="reset" ng-click="cancelEditSchema()"><span>Cancel</span></button>
+ </p>
+ </div>
+
+ <div class="action add" data-rel="add" ng-show="showNewSchema" style="display:block;">
+ <p ng-show="firstSchemaMessage" class="clearfix">Get started building your search app by defining a new schema. A schema defines all the fields in your search index. Solr comes with a <b>_default</b> schema that provides a good starting point to customize based on your specific data.<br/><br/></p>
+ <form>
+ <p class="clearfix"><label for="add_schema">New Schema:</label><input type="text" name="name" id="add_schema" ng-model="newSchema"></p>
+ <div class="clearfix"><label for="copy_schema">Copy From:</label>
+ <select id="copy_schema"
+ ng-model="copyFrom"
+ chosen
+ ng-options="schema for schema in schemasWithDefault"></select><a ng-click="showHelp('copyFromHelp')"><img class="help-ico" src="img/ico/question-white.png"/></a>
+ <div id="copyFromHelp" class="help" ng-show="helpId === 'copyFromHelp'" escape-pressed="hideAll()">
+ <div class="help">
+ <p>Copy an existing Configset (schema, solrconfig.xml, and supporting files) as the starting point for your new schema. The <b>_default</b> Configset includes a schema with field guessing enabled, dynamic fields for common field types, and field types for analyzing text data for a number of common languages supported by Lucene. For more information about Configsets, see: <div class="help-anchor"><a target="_blank" href="https://solr.apache.org/guide/config-sets.html">Configsets</a></div></p>
+ </div>
+ </div>
+ </div>
+ <p class="clearfix note error" ng-show="addMessage"><span>{{addMessage}}</span></p>
+ <p class="clearfix buttons">
+ <button type="submit" class="submit" ng-click="addSchema()"><span>Add Schema</span></button>
+ <button type="reset" class="reset" ng-click="cancelAddSchema()"><span>Cancel</span></button>
+ </p>
+ </form>
+ </div>
+ </div>
+ </div>
+ </div>
+
+ <div class="clearfix">
+
+ <div class="designer-left">
+ <div class="block" id="sample-docs">
+ <h2><span>Sample Documents</span></h2>
+ <div class="content">
+ <form id="sample-docs-form">
+ <div id="document-container">
+ <div id="file-upload">
+ <input type="file" id="upload-file" name="upload-file" file-model="fileUpload"/><a ng-click="showHelp('sampleDocsHelp')"><img class="help-ico" src="img/ico/question-white.png"/></a>
+ <div id="sampleDocsHelp" class="help" ng-show="helpId === 'sampleDocsHelp'">
+ <div class="help">
+ <p>Upload a JSON, CSS, or XML file containing sample documents or simply paste some sample documents into the text area below; the Schema Designer supports a maximum of 5MB and 1,000 documents.
+ <br/><br/>Click on the <b>Analyze Documents</b> button to have Solr determine the schema by looking at the sample values for each field. Sample documents are stored on the server so you can make changes to the schema and Schema Designer will automatically re-index the sample documents to apply the changes.
+ </p>
+ </div>
+ </div>
+ </div>
+ <label for="document">or Paste Sample Document(s):</label>
+ <textarea ng-model="sampleDocuments" name="document" id="document" title="Sample Documents" rows="20" cols="44" placeholder="" ng-change="onSampleDocumentsChanged()"></textarea>
+ </div>
+ <div id="analyze-buttons">
+ <p class="clearfix note" ng-show="sampleMessage && !fileUpload && !sampleDocuments"><span>{{sampleMessage}}</span></p>
+ <button type="submit" ng-click="doAnalyze()" id="analyze" ng-show="currentSchema && (hasDocsOnServer || sampleDocuments || fileUpload)"><span>Analyze Documents</span></button>
+ </div>
+ </form>
+ </div>
+ </div>
+
+ </div>
+
+ <div class="designer-middle">
+ <div class="block" id="schema-editor">
+ <h2><span>Schema Editor</span></h2>
+ <div id="tree-filters" ng-show="showSchemaActions">
+ Filter fields by:&nbsp;<select id="tree-filters-cat"
+ ng-model="treeFilter"
+ ng-change="applyTreeFilterOption()">
+ <option value=""></option>
+ <option value="type">type</option>
+ <option value="feature">feature</option>
+ </select>
+ <select id="tree-filters-options"
+ type="text"
+ ng-model="treeFilterOption"
+ ng-change="onTreeFilterOptionChanged()"
+ ng-options="option for option in treeFilterOptions"></select>
+ <span id="treeFilterFeatureEnabledSpan" ng-show="treeFilter === 'feature'">
+ <input id="treeFilterFeatureEnabledInput" type="checkbox" ng-model="treeFilterFeatureEnabled" ng-change="onTreeFilterOptionChanged()" ng-true-value="'true'" ng-false-value="'false'"/>
+ <label class="checkbox" for="treeFilterFeatureEnabledInput" title="enabled">enabled</label>
+ </span>
+ </div>
+ <div id="tree-content" class="content clearfix" ng-show="showSchemaActions">
+ <jstree class="tree" on-select="onSelectSchemaTreeNode(url)" id="schemaJsTree" data="schemaTree"></jstree>
+ </div>
+ <div id="tree-node-content" ng-show="showFieldDetails">
+ <div id="field-top-right">
+ <button id="updateSelected" class="action" ng-click="updateSelected()" ng-show="selectedUpdated"><span>Update {{ selectedType }}</span></button>
+ <!-- <button id="deleteSelected" class="action" ng-click="deleteSelected()" ng-show="selectedNode"><span>Delete {{ selectedType }}</span></button> -->
+ </div>
+
+ <div id="schema-root-details" ng-show="isSchemaRoot">
+ <div class="field-form-right">
+ <label class="checkbox" title="Schema">Schema:</label>
+ <p id="currentSchemaLabel">{{currentSchema}}</p>
+ </div>
+ <div class="field-form-right">
+ <label for="unique_key" class="checkbox" title="Unique Key Field">Unique Key Field:</label>
+ <select id="unique_key"
+ placeholder-text-single="'Unique Key Field'"
+ ng-model="updateUniqueKeyField"
+ chosen
+ disabled
+ ng-change="updateUniqueKey()"
+ ng-options="field for field in possibleIdFields"></select>
+ </div>
+ <div class="field-form-right">
+ <label for="languages">Language(s):</label>
+ <select ng-model="languages" id="languages" ng-change="changeLanguages()" placeholder="Languages" size="12" multiple>
+ <option value="*">-- All --</option>
+ <option value="ar">Arabic</option>
+ <option value="hy">Armenian</option>
+ <option value="eu">Basque</option>
+ <option value="bg">Bulgarian</option>
+ <option value="ca">Catalan</option>
+ <option value="cjk">CJK bigram</option>
+ <option value="cz">Czech</option>
+ <option value="da">Danish</option>
+ <option value="nl">Dutch</option>
+ <option value="en">English</option>
+ <option value="et">Estonian</option>
+ <option value="fa">Farsi</option>
+ <option value="fi">Finnish</option>
+ <option value="fr">French</option>
+ <option value="gl">Galician</option>
+ <option value="de">German</option>
+ <option value="el">Greek</option>
+ <option value="hi">Hindi</option>
+ <option value="hu">Hungarian</option>
+ <option value="id">Indonesian</option>
+ <option value="ga">Irish</option>
+ <option value="it">Italian</option>
+ <option value="ja">Japanese</option>
+ <option value="ko">Korean</option>
+ <option value="lv">Latvian</option>
+ <option value="no">Norwegian</option>
+ <option value="pt">Portuguese</option>
+ <option value="ro">Romanian</option>
+ <option value="ru">Russian</option>
+ <option value="es">Spanish</option>
+ <option value="sv">Swedish</option>
+ <option value="th">Thai</option>
+ <option value="tr">Turkish</option>
+ </select><a ng-click="showHelp('languagesHelp')"><img class="help-ico" src="img/ico/question-white.png"/></a>
+ <div id="languagesHelp" class="help" ng-show="helpId === 'languagesHelp'" escape-pressed="hideAll()">
+ <div class="help">
+ <p>By default, schemas copied from the <i>_default</i> schema include field types (and associated files) for all built-in languages. Select the languages needed for your application and the designer will remove all the unnecessary field types for languages you don't need. For more information about text analysis and languages, see:
+ <div class="help-anchor">
+ <a target="_blank" href="https://solr.apache.org/guide/language-analysis.html#language-specific-factories">Language-Specific Factories</a>
+ </div>
+ </p>
+ </div>
+ </div>
+ </div>
+ <div class="field-form-checkbox">
+ <input ng-change="rootChanged()" id="include_dynamic" type="checkbox" ng-model="enableDynamicFields" ng-true-value="'true'" ng-false-value="'false'">
+ <label for="include_dynamic" class="checkbox" title="Include dynamic fields">Include dynamic fields?</label>
+ <a ng-click="showHelp('dynamicFieldsHelp')"><img class="help-ico" src="img/ico/question-white.png"/></a>
+ <div id="dynamicFieldsHelp" class="help" ng-show="helpId === 'dynamicFieldsHelp'">
+ <div class="help-wide">
+ <p>Dynamic fields allow Solr to index fields that you did not explicitly define in your schema. Dynamic fields can make your application less brittle by providing some flexibility in the documents you can add to Solr. It is recommended to keep the default set of dynamic fields enabled for your schema. Unchecking this option removes all dynamic fields from your schema. For more information about dynamic fields, see:
+ <div class="help-anchor"><a target="_blank" href="https://solr.apache.org/guide/dynamic-fields.html">Dynamic Fields</a></div>
+ </p>
+ </div>
+ </div>
+ </div>
+ <div class="field-form-checkbox">
+ <input ng-change="rootChanged()" id="enable_guessing" type="checkbox" ng-model="enableFieldGuessing" ng-true-value="'true'" ng-false-value="'false'">
+ <label for="enable_guessing" class="checkbox" title="Enable field guessing">Enable field guessing?</label>
+ <a ng-click="showHelp('fieldGuessingHelp')"><img class="help-ico" src="img/ico/question-white.png"/></a>
+ <div id="fieldGuessingHelp" class="help" ng-show="helpId === 'fieldGuessingHelp'">
+ <div class="help-wide">
+ <p>Field guessing (aka "schemaless mode") allows Solr to detect the "best" field type for unknown fields encountered during indexing. Field guessing also performs some field transformations, such as removing spaces from field names. If you use the schema designer to create your schema based on sample documents, you may not need to enable this feature.
+ However, with this feature disabled, you need to make sure the incoming data matches the schema exactly or indexing errors may occur. For more information about schemaless mode, see:
+ <div class="help-anchor"><a target="_blank" href="https://solr.apache.org/guide/schemaless-mode.html">Schemaless Mode</a></div>
+ </p>
+ </div>
+ </div>
+ </div>
+ <div class="field-form-checkbox">
+ <input ng-change="rootChanged()" id="enable_nested" type="checkbox" ng-model="enableNestedDocs" ng-true-value="'true'" ng-false-value="'false'">
+ <label for="enable_nested" class="checkbox" title="Enable field guessing">Enable nested child documents?</label>
+ <a ng-click="showHelp('nestedDocsHelp')"><img class="help-ico" src="img/ico/question-white.png"/></a>
+ <div id="nestedDocsHelp" class="help" ng-show="helpId === 'nestedDocsHelp'">
+ <div class="help-wide">
+ <p>Enabling this feature adds the <b>_root_</b> and <b>_nest_path_</b> fields to your schema.<br/><br/>For more information about indexing nested child documents, see:
+ <div class="help-anchor"><a target="_blank" href="https://solr.apache.org/guide/indexing-nested-documents.html">Indexing Nested Child Documents</a></div>
+ </p>
+ </div>
+ </div>
+ </div>
+ </div>
+ <div id="file-node" ng-show="selectedFile">
+ <textarea name="file-node-text" id="file-node-text" ng-model="fileNodeText" rows="24" cols="62"></textarea>
+ <div id="save-file-buttons" class="clearfix">
+ <button id="save-file" type="submit" class="submit" ng-click="updateFile()"><span>Save</span></button>
+ </div>
+ <p class="clearfix note error" ng-show="updateFileError"><span>{{updateFileError}}</span></p>
+ </div>
+ <div id="leaf-node" ng-show="isLeafNode">
+ <form>
+ <p class="clearfix note error" ng-show="updateSelectedError"><span>{{updateSelectedError}}</span></p>
+ <div id="field-top">
+ <div id="field-top-left">
+ <div class="field-top-form">{{selectedType}}: <b>{{selectedNode.name}}</b></div>
+ <div class="field-top-form" ng-show="selectedType === 'Field'">
+ <label for="field_type">Type:</label>
+ <select chosen
+ type="text"
+ id="field_type"
+ ng-model="selectedNode.type"
+ ng-change="onFieldTypeChanged(type)"
+ ng-options="type for type in types"></select><a ng-click="showHelp('fieldTypeHelp')"><img class="help-ico" src="img/ico/question-white.png"/></a>
+ <div id="fieldTypeHelp" class="help" ng-show="helpId === 'fieldTypeHelp'">
+ <div class="help-wider">
+ <p>The field type defines how Solr should interpret data in a field and how the field can be queried. The properties on each field are inherited from the field type if not explicitly overridden in the field definition. In many cases, it's preferable to choose a different field type instead of overriding the field properties. For example, there's typically a multi-valued version of each type, such as <b>strings</b> is the multi-valued version of the <b>string</b> type. For more information about field types, see:
+ <div class="help-anchor">
+ <a target="_blank" href="https://solr.apache.org/guide/field-type-definitions-and-properties.html#field-type-definitions-and-properties">Field Type Definitions and Properties</a>
+ </div>
+ </p>
+ </div>
+ </div>
+ </div>
+ <div class="field-top-form" ng-show="selectedType === 'Field'">
+ <label for="field_default" title="Default Value">Default Value:</label>
+ <input id="field_default" type="text" ng-model="selectedNode.default" ng-click="markSelectedUpdated($event)">
+ </div>
+ <div class="field-top-form" ng-show="selectedType === 'Type'">
+ <label for="type_class" title="Class">Class</label>
+ <input id="type_class" type="text" ng-model="selectedNode.class" ng-click="markSelectedUpdated($event)" placeholder="Java Class">
+ </div>
+ </div>
+ </div>
+ <div id="field-bottom">
+ <div id="field-form-left">
+ <div class="field-form">
+ <input id="indexed" type="checkbox" ng-model="selectedNode.indexed" ng-click="markSelectedUpdated($event)" value="true">
+ <label for="indexed" class="checkbox" title="Indexed">Indexed</label>
+ </div>
+ <div class="field-form">
+ <input id="stored" type="checkbox" ng-model="selectedNode.stored" ng-click="markSelectedUpdated($event)" value="true">
+ <label for="stored" class="checkbox" title="Stored">Stored</label>
+ </div>
+ <div class="field-form">
+ <input id="multiValued" type="checkbox" ng-model="selectedNode.multiValued" ng-click="markSelectedUpdated($event)" value="true">
+ <label for="multiValued" class="checkbox" title="Multi-Valued">Multi-Valued</label>
+ </div>
+ <div class="field-form">
+ <input id="docValues" type="checkbox" ng-model="selectedNode.docValues" ng-click="markSelectedUpdated($event)" ng-disabled="isDisabled('not-text')" value="true">
+ <label for="docValues" class="checkbox" title="Doc Values">Doc Values</label>
+ </div>
+ <div class="field-form">
+ <input id="useDocValuesAsStored" type="checkbox" ng-model="selectedNode.useDocValuesAsStored" ng-click="markSelectedUpdated($event)" ng-disabled="isDisabled('docValues')" value="true">
+ <label for="useDocValuesAsStored" class="checkbox" title="Use Doc Values as Stored">Use Doc Values as Stored</label>
+ </div>
+ <div class="field-form" ng-show="selectedType == 'Field'">
+ <input id="required" type="checkbox" ng-model="selectedNode.required" ng-click="markSelectedUpdated($event)" value="true">
+ <label for="required" class="checkbox" title="Required">Required</label>
+ </div>
+ <div class="field-form">
+ <input id="tokenized" type="checkbox" ng-model="selectedNode.tokenized" ng-click="markSelectedUpdated($event)" ng-disabled="isDisabled('indexed')" value="true">
+ <label for="tokenized" class="checkbox" title="Tokenized">Tokenized</label>
+ </div>
+ <div class="field-form">
+ <input id="sortMissingLast" type="checkbox" ng-model="selectedNode.sortMissingLast" ng-click="markSelectedUpdated($event)" value="true">
+ <label for="sortMissingLast" class="checkbox" title="Sort Missing Last">Sort Missing Last</label>
+ </div>
+ <div class="field-form">
+ <input id="uninvertible" type="checkbox" ng-model="selectedNode.uninvertible" ng-click="markSelectedUpdated($event)" ng-disabled="isDisabled('indexed')" value="true">
+ <label for="uninvertible" class="checkbox" title="Uninvertible">Uninvertible</label>
+ </div>
+ </div>
+ <div id="field-form-right">
+ <div class="field-form">
+ <input id="termVectors" type="checkbox" ng-model="selectedNode.termVectors" ng-click="markSelectedUpdated($event)" ng-disabled="isDisabled('indexed')" value="true">
+ <label for="termVectors" class="checkbox" title="Term Vectors">Term Vectors</label>
+ </div>
+ <div class="field-form">
+ <input id="termPositions" type="checkbox" ng-model="selectedNode.termPositions" ng-click="markSelectedUpdated($event)" ng-disabled="isDisabled('termVectors')" value="true">
+ <label for="termPositions" class="checkbox" title="Term Positions">Term Positions</label>
+ </div>
+ <div class="field-form">
+ <input id="termOffsets" type="checkbox" ng-model="selectedNode.termOffsets" ng-click="markSelectedUpdated($event)" ng-disabled="isDisabled('termVectors')" value="true">
+ <label for="termOffsets" class="checkbox" title="Term Offsets">Term Offsets</label>
+ </div>
+ <div class="field-form">
+ <input id="termPayloads" type="checkbox" ng-model="selectedNode.termPayloads" ng-click="markSelectedUpdated($event)" ng-disabled="isDisabled('termVectors')" value="true">
+ <label for="termPayloads" class="checkbox" title="Term Payloads">Term Payloads</label>
+ </div>
+ <div class="field-form">
+ <input id="omitNorms" type="checkbox" ng-model="selectedNode.omitNorms" ng-click="markSelectedUpdated($event)" ng-disabled="isDisabled('indexed')" value="true">
+ <label for="omitNorms" class="checkbox" title="Omit Norms">Omit Norms</label>
+ </div>
+ <div class="field-form">
+ <input id="omitPositions" type="checkbox" ng-model="selectedNode.omitPositions" ng-click="markSelectedUpdated($event)" ng-disabled="isDisabled('indexed')" value="true">
+ <label for="omitPositions" class="checkbox" title="Omit Positions">Omit Positions</label>
+ </div>
+ <div class="field-form">
+ <input id="omitTermFreqAndPositions" type="checkbox" ng-model="selectedNode.omitTermFreqAndPositions" ng-click="markSelectedUpdated($event)" ng-disabled="isDisabled('indexed')"
+ value="true">
+ <label for="omitTermFreqAndPositions" class="checkbox" title="Omit Term Frequencies & Positions">Omit Term Frequencies & Positions</label>
+ </div>
+ <div class="field-form">
+ <input id="storeOffsetsWithPositions" type="checkbox" ng-model="selectedNode.storeOffsetsWithPositions" ng-click="markSelectedUpdated($event)" ng-disabled="isDisabled('indexed')"
+ value="true">
+ <label for="storeOffsetsWithPositions" class="checkbox" title="Store Offsets with Positions">Store Offsets with Positions</label>
+ </div>
+ <div class="field-form">
+ <input id="large" type="checkbox" ng-model="selectedNode.large" ng-click="markSelectedUpdated($event)" ng-disabled="isDisabled('stored')" value="true">
+ <label for="large" class="checkbox" title="Large">Large</label>
+ </div>
+ </div>
+ <div id="field-copy" ng-show="selectedType === 'Field'">
+ <div class="field-form">
+ <label for="copy_dest" class="checkbox" title="Copy to">Copy to:</label>
+ <input type="text" id="copy_dest" ng-model="selectedNode.copyDest" ng-change="markSelectedUpdated($event)" placeholder="specify your destination field(s) (comma-delimited)">
+ <a ng-click="showHelp('copyToHelp')"><img class="help-ico" src="img/ico/question-white.png"/></a>
+ <div id="copyToHelp" class="help" ng-show="helpId === 'copyToHelp'">
+ <div class="help-top">
+ <p>Copy fields allow you to apply several distinct field types to a single piece of incoming information during indexing. For instance, a common approach is to copy important string fields into the catch-all '_text_' field to enable full-text matches on terms contained in string fields. Provide a comma-delimited list of fields to copy into during indexing. For more information about copy fields, see:
+ <div class="help-anchor">
+ <a target="_blank" href="https://solr.apache.org/guide/8_8/copying-fields.html">Copy Fields</a>
+ </div>
+ </p>
+ </div>
+ </div>
+ </div>
+ </div>
+ <div id="field-analysis" ng-show="analysis.data.tokenized">
+ <div id="edit-analysis-json-link" ng-show="!selectedNode.type"><a ng-click="toggleShowAnalysisJson()" class="toggle">{{editAnalysis}}</a></div>
+ <div id="show-analysis-json" ng-show="showAnalysisJson">
+ <textarea id="analysis-json" ng-change="markSelectedUpdated($event)" ng-model="analysisJsonText" rows="6" cols="60"></textarea>
+ </div>
+ <div id="hide-analysis-json" ng-show="!showAnalysisJson">
+ <ul class="analyzer">
+ <li class="clearfix {{analyzer.key}}" ng-class="{open: analyzer.show}" ng-repeat="analyzer in analysis.analyzers">
+ <p><span>{{analyzer.name}}&nbsp;Analyzer:</span></p>
+ <dl>
+ <dt><a ng-click="toggleAnalyzer(analyzer)" class="toggle">{{analyzer.detail.className}}</a></dt>
+ </dl>
+ <ul ng-show="analyzer.show">
+ <li class="clearfix {{componentType.key}} data" ng-repeat="componentType in analyzer.detail.componentTypes" ng-show="componentType.components">
+ <p>{{componentType.label}}:</p>
+ <dl>
+ <dt ng-repeat-start="component in componentType.components">{{component.className}}</dt>
+ <dd ng-repeat-end ng-repeat="arg in component.args" ng-class="{'ico-1': arg.booleanValue, 'ico-0': arg.booleanValue==false}">
+ {{arg.name}}<span ng-show="arg.value && arg.name != 'words' && arg.name != 'protected'">: {{arg.value}}</span><span ng-show="arg.value && (arg.name === 'words' || arg.name === 'protected')">: <a class="file-link" ng-click="loadFile($event)">{{arg.value}}</a></span>
+ </dd>
+ </dl>
+ </li>
+ </ul>
+ </li>
+ </ul>
+ </div>
+ </div>
+ </div>
+ </form>
+ </div>
+ <div class="container-node" ng-show="containerNode === 'fields' || containerNode === 'dynamicFields'">
+ <table border="0" cellspacing="0" cellpadding="0">
+ <tbody>
+ <tr ng-class="{odd:$odd}">
+ <th class="prop-hdr">Name <a ng-click="showHelp('fieldTableHelp')"><img class="help-ico" src="img/ico/question-white.png"/></a>
+ <div id="fieldTableHelp" class="help" ng-show="helpId === 'fieldTableHelp'">
+ <div class="help">
+ <p>This table provides a list of {{containerNodeLabel}} defined for your schema. Click on the name in the first column to edit properties. Click on <b>{{containerNodeLabel}}</b> in the Schema Editor tree on the left to return to this table view.<br/><br/>For more information about field properties, see: <div class="help-anchor"><a target="_blank" href="https://solr.apache.org/guide/defining-fields.html">Field Properties</a></div></p>
+ </div>
+ </div>
+ </th>
+ <th class="prop-hdr">Type</th>
+ <th class="prop-hdr-flag">Indexed</th>
+ <th class="prop-hdr-flag">Stored</th>
+ <th class="prop-hdr-flag">MultiValued</th>
+ <th class="prop-hdr-flag">DocValues</th>
+ <th class="prop-hdr-flag">Tokenized</th>
+ </tr>
+ <tr ng-repeat="f in containerNodes" ng-class="{odd:$odd}">
+ <td class="prop"><a ng-click="selectField($event)">{{f.name}}</a></td>
+ <td class="prop">{{f.type}}</td>
+ <td class="prop-flag"><span ng-show="f.indexed"><img src="img/ico/tick.png"/></span></td>
+ <td class="prop-flag"><span ng-show="f.stored"><img src="img/ico/tick.png"/></span></td>
+ <td class="prop-flag"><span ng-show="f.multiValued"><img src="img/ico/tick.png"/></span></td>
+ <td class="prop-flag"><span ng-show="f.docValues"><img src="img/ico/tick.png"/></span></td>
+ <td class="prop-flag"><span ng-show="f.tokenized"><img src="img/ico/tick.png"/></span></td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ <div class="container-node" ng-show="containerNode === 'fieldTypes'">
+ <table border="0" cellspacing="0" cellpadding="0">
+ <tbody>
+ <tr ng-class="{odd:$odd}">
+ <th class="prop-hdr">Type <a ng-click="showHelp('fieldTypeTableHelp')"><img class="help-ico" src="img/ico/question-white.png"/></a>
+ <div id="fieldTypeTableHelp" class="help" ng-show="helpId === 'fieldTypeTableHelp'">
+ <div class="help">
+ <p>This table provides a list of field types defined for your schema. Click on the type name in the first column to edit properties for that type. Changing properties for a type impacts all fields linked to the type. Click on <b>Field Types</b> in the Schema Editor tree on the left to return to this table view.<br/><br/>For more information about field types, see: <div class="help-anchor"><a target="_blank" href="https://solr.apache.org/guide/field-type-definitions-and-properties.html">Field Type Definitions and Properties</a></div></p>
+ </div>
+ </div></th>
+ <th class="prop-hdr">Class</th>
+ <th class="prop-hdr-flag">Indexed</th>
+ <th class="prop-hdr-flag">Stored</th>
+ <th class="prop-hdr-flag">MultiValued</th>
+ <th class="prop-hdr-flag">DocValues</th>
+ <th class="prop-hdr-flag">Tokenized</th>
+ </tr>
+ <tr ng-repeat="f in containerNodes" ng-class="{odd:$odd}">
+ <td class="prop"><a ng-click="selectFieldType($event)">{{f.name}}</a></td>
+ <td class="prop">{{f.class}}</td>
+ <td class="prop-flag"><span ng-show="f.indexed"><img src="img/ico/tick.png"/></span></td>
+ <td class="prop-flag"><span ng-show="f.stored"><img src="img/ico/tick.png"/></span></td>
+ <td class="prop-flag"><span ng-show="f.multiValued"><img src="img/ico/tick.png"/></span></td>
+ <td class="prop-flag"><span ng-show="f.docValues"><img src="img/ico/tick.png"/></span></td>
+ <td class="prop-flag"><span ng-show="f.tokenized"><img src="img/ico/tick.png"/></span></td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ </div>
+ </div>
+ </div>
+
+ <div class="designer-right">
+ <div class="block" id="analysis">
+ <h2><span>Text Analysis</span></h2>
+ <div class="block analysis-error" id="analysis-handler-missing" ng-show="isHandlerMissing">
+ <div class="head">This Functionality requires the <code>/analysis/field</code> Handler to be registered and active!</div>
+ </div>
+ <div class="block analysis-error" id="analysis-error" ng-show="analysisError">
+ <div class="body">{{analysisError}}</div>
+ </div>
+ <div id="field-analysis-holder" ng-show="showAnalysis">
+ <div class="field-form">
+ <label for="analysis_select_doc_id">Sample Doc ID</label>
+ <select id="analysis_select_doc_id"
+ placeholder-text-single="'ID'"
+ ng-model="sampleDocId"
+ chosen
+ ng-change="updateSampleDocId()"
+ ng-options="id for id in sampleDocIds"></select>
+ </div>
+ <div class="field-form">
+ <label for="analysis_fieldvalue_index">{{selectedNode.name}}</label>
+ <textarea name="analysis.fieldvalue" id="analysis_fieldvalue_index" ng-model="indexText" rows="2" cols="40"></textarea>
+ </div>
+ <div class="verbose_output" ng-class="{active:analysisVerbose}">
+ <a ng-click="toggleVerbose()">Detailed Output</a>
+ </div>
+ <div id="analysis-result" class="clearfix">
+ <div ng-class="key" ng-repeat="(key, type) in result">
+ <table border="0" cellspacing="0" cellpadding="0">
+ <tbody ng-repeat="component in type">
+ <tr class="step">
+ <td class="part analyzer">
+ <div>
+ <abbr title="{{component.name}}">{{component.short}}</abbr>
+ </div>
+ </td>
+
+ <td class="part legend" ng-show="analysisVerbose">
+ <div class="holder">
+ <table border="0" cellspacing="0" cellpadding="0">
+ <tr>
+ <td>
+ <table border="0" cellspacing="0" cellpadding="0">
+ <tr ng-repeat="caption in component.captions" class="{{generate_class_name( short_key )}}"><td>{{ caption }}</td></tr>
+ </table>
+ </td>
+ </tr>
+ </table>
+ </div>
+ </td>
+ <td class="part data" ng-class="{spacer:token.blank}" colspan="1" ng-repeat="token in component.tokens track by token.index">
+ <div class="holder" ng-hide="token.blank">
+ <table border="0" cellspacing="0" cellpadding="0">
+ <tbody>
+ <tr class="details">
+ <td class="details">
+ <table border="0" cellspacing="0" cellpadding="0">
+ <tbody>
+ <tr class="{{value.name}} {{value.extraclass}}" ng-repeat="value in token.keys" ng-show="analysisVerbose || value.name=='text'">
+ <td>{{value.value}}</td>
+ </tr>
+ </tbody>
+ </table>
+ </td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ <div class="holder" ng-show="token.blank">&nbsp;</div>
+ </td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+
+ <div class="clearfix">
+ <div class="query-tester-left">
+ <div class="block" id="query-tester">
+ <h2><span>Query Tester</span></h2>
+ <div class="content">
+ <div id="query-form" ng-show="showSchemaActions">
+ <div class="field-form-right">
+ <label for="q" title="The query string.">Query</label>
+ <textarea name="q" ng-model="query.q" id="q" rows="1" cols="30" title="The query string.">*:*</textarea>
+ <a ng-click="showHelp('qHelp')"><img class="help-ico" src="img/ico/question-white.png"/></a>
+ <div id="qHelp" class="help" ng-show="helpId === 'qHelp'">
+ <div class="help">
+ <p>Main query parameter <b>q</b> specifies the ranked matching criteria for documents. If you don't specify a field name in the query, the default text field, typically <b>_text_</b>, is used.
+ If the query results do not contain expected documents, check how the default text field is populated, typically using copy field directives in the schema.
+ <br/><br/>For more information about query parameters, see:
+ <div class="help-anchor">
+ <a target="_blank" href="https://solr.apache.org/guide/query-syntax-and-parsing.html">Query Syntax and Parsing</a>
+ </div>
+ </p>
+ </div>
+ </div>
+ </div>
+ <div class="field-form-right">
+ <label for="sort" title="Sort field">Sort by</label>
+ <select chosen placeholder-text-single="'Select Field'" id="sort" ng-model="query.sortBy" ng-options="field for field in sortableFields"></select> asc <input type="radio" ng-model="query.sortDir" value="asc"/> desc <input type="radio" ng-model="query.sortDir" value="desc"/>
+ <a ng-click="showHelp('querySortHelp')"><img class="help-ico" src="img/ico/question-white.png"/></a>
+ <div id="querySortHelp" class="help" ng-show="helpId === 'querySortHelp'">
+ <div class="help">
+ <p>Sort fields must be single-valued and indexed (or have docValues enabled). If an expected field is not present in this list, check the active properties in the schema.
+ <br/><br/>For more information about query parameters, see:
+ <div class="help-anchor">
+ <a target="_blank" href="https://solr.apache.org/guide/query-syntax-and-parsing.html">Query Syntax and Parsing</a>
+ </div>
+ </p>
+ </div>
+ </div>
+ </div>
+ <div class="field-form-right">
+ <label for="ff" title="Facet field">Facet fields</label>
+ <select multiple ng-model="selectedFacets" id="ff" size="7" ng-options="ff for ff in facetFields"></select>
+ <a ng-click="showHelp('queryFacetHelp')"><img class="help-ico" src="img/ico/question-white.png"/></a>
+ <div id="queryFacetHelp" class="help" ng-show="helpId === 'queryFacetHelp'">
+ <div class="help">
+ Fields must be indexed or have docValues enabled to compute facets.<br/><br/>For more information about faceting, see:
+ <div class="help-anchor"><a target="_blank" href="https://solr.apache.org/guide/faceting.html">Faceting</a></div>
+ </div>
+ </div>
+ </div>
+ <div class="field-form-right">
+ <label for="hl" title="Highlight">Highlighting</label>
+ <select chosen placeholder-text-single="'Select Field'" id="hl" ng-model="query.highlight" ng-options="field for field in hlFields"></select>
+ </div>
+ <div class="field-form-right">
+ <label for="rawParams" title="Additional params">Additional params</label>
+ <textarea name="q" ng-model="rawParams" id="rawParams" rows="1" cols="30" title="Additional URL params"></textarea>
+ </div>
+ <div id="query-buttons">
+ <button type="submit" ng-click="doQuery()" id="queryButton"><span>Run Query</span></button>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+ <div class="query-tester-right">
+ <div class="block" id="query-results">
+ <h2><span>Query Results</span></h2>
+ <div class="content" ng-show="queryResultsTree">
+ <div id="results-tree-left">
+ <jstree class="tree" on-select="onSelectQueryResultsNode(url)" id="queryResultsJsTree" data="queryResultsTree"></jstree>
+ </div>
+ <div id="results-tree-right">
+ <div id="editDocumentsToolbar" ng-show="(selectedResultsNode === '/' || selectedResultsNode === 'docs') && queryDocs">
+ <button id="editDocuments" class="action" ng-click="editDocuments()"><span>Edit Documents</span></button><a ng-click="showHelp('editDocsHelp')"><img class="help-ico" src="img/ico/question-white.png"/></a>
+ <div id="editDocsHelp" class="help" ng-show="helpId === 'editDocsHelp'">
+ <div class="help-small">Copy the documents from the query results to the Sample Documents text area as JSON. This allows you to refine your schema by changing the sample data sourced from the query results.</div>
+ </div>
+ </div>
+ <div id="selected-hit" ng-show="resultsData">
+ <table border="0" cellspacing="0" cellpadding="0">
+ <tbody>
+ <tr ng-repeat="row in resultsData" ng-class="{odd:$odd}">
+ <td class="hit-field"><span ng-show="!row.type">{{row.name}}</span><span ng-show="row.type === 'f'"><a ng-click="selectField($event)">{{row.name}}</a></span></td>
+ <td class="hit-value" width="500">{{row.value}}</td>
+ </tr>
+ </tbody>
+ </table>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+ </div>
+
+</div>
+
+</div>