Apache Solr open-source search software
SOLR-10808: Enable docValues by default on specific fields (#2606)
Houston Putmanb44bdc198d83
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 8787dd60653..99533e5f97c 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -153,6 +153,14 @@ New FeaturesImprovements---------------------+* SOLR-10808, SOLR-12963: The Solr schema version has been increased to 1.7.+ Starting in schema version 1.7, most fields/fieldTypes that support docValues will have them enabled by default.+ These field types include primitive (Numeric, Date, Bool, String, Enum, UUID), sorting (SortableTextField, SortableBinaryField, CollationField, ICUCollationField) and LatLonPointSpacialField.+ This behavior can be reverted by setting the 'docValues' parameter for a field or a field type to false, the default for schema versions 1.6 and below.+ Also in schema version 1.7, all fields/fieldTypes will be unable to be uninverted by default.+ This behavior can be reverted by setting the 'uninvertible' parameter for a field or a field type to true, the default for schema versions 1.6 and below.+ (Houston Putman, hossman)+* SOLR-17137: Enable Prometheus exporter to communicate with SSL protected Solr. (Eivind Bergstøl via Eric Pugh)* SOLR-16921: use -solrUrl to derive the zk host connection for bin/solr zk subcommands (Eric Pugh)@@ -183,13 +191,6 @@ ImprovementsStatuses are now removed 5 minutes after the read of a completed/failed status. Helps collectionasync backup/restore and other operations scale to 100+ shards. (Pierre Salagnac, David Smiley)-* SOLR-10808, SOLR-12963: The Solr schema version has been increased to 1.7.- Starting in schema version 1.7, all fields/fieldTypes that support docValues will have them enabled by default.- This behavior can be reverted by setting the 'docValues' parameter for a field or a field type to false, the default for schema versions 1.6 and below.- Also in schema version 1.7, all fields/fieldTypes will be unable to be uninverted by default.- This behavior can be reverted by setting the 'uninvertible' parameter for a field or a field type to true, the default for schema versions 1.6 and below.- (Houston Putman, hossman)-* SOLR-10808 : The Solr schema version has been increased to 1.7. Since schema version 1.7, all fields/fieldTypes thatsupport docValues will have them enabled by default. This behavior can be reverted by setting'docValues' parameter for a field or a field type to false, the default for schema versions 1.6 and below. (Houston Putman)diff --git a/solr/core/src/java/org/apache/solr/schema/BinaryField.java b/solr/core/src/java/org/apache/solr/schema/BinaryField.javaindex 37bfc653236..94bc94fd86f 100644--- a/solr/core/src/java/org/apache/solr/schema/BinaryField.java+++ b/solr/core/src/java/org/apache/solr/schema/BinaryField.java@@ -152,11 +152,6 @@ public class BinaryField extends FieldType {protected void checkSupportsDocValues() { // we support DocValues}- @Override- protected boolean doesTypeSupportDocValues() {- return true;- }-@Overridepublic Object toNativeType(Object val) {if (val instanceof byte[]) {diff --git a/solr/core/src/java/org/apache/solr/schema/CollationField.java b/solr/core/src/java/org/apache/solr/schema/CollationField.javaindex 739bae5f211..91f67d459e7 100644--- a/solr/core/src/java/org/apache/solr/schema/CollationField.java+++ b/solr/core/src/java/org/apache/solr/schema/CollationField.java@@ -246,7 +246,7 @@ public class CollationField extends FieldType {}@Override- protected boolean doesTypeSupportDocValues() {+ protected boolean enableDocValuesByDefault() {return true;}diff --git a/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java b/solr/core/src/java/org/apache/solr/schema/DenseVectorField.javaindex d83462235b0..4d528361dd4 100644--- a/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java+++ b/solr/core/src/java/org/apache/solr/schema/DenseVectorField.java@@ -179,7 +179,7 @@ public class DenseVectorField extends FloatPointField {}@Override- protected boolean doesTypeSupportDocValues() {+ protected boolean enableDocValuesByDefault() {return false;}diff --git a/solr/core/src/java/org/apache/solr/schema/FieldType.java b/solr/core/src/java/org/apache/solr/schema/FieldType.javaindex 9431691b1de..cb60b1f332a 100644--- a/solr/core/src/java/org/apache/solr/schema/FieldType.java+++ b/solr/core/src/java/org/apache/solr/schema/FieldType.java@@ -188,7 +188,7 @@ public abstract class FieldType extends FieldProperties {args.remove("compressThreshold");}if (schemaVersion >= 1.6f) properties |= USE_DOCVALUES_AS_STORED;- if (schemaVersion >= 1.7f && doesTypeSupportDocValues()) properties |= DOC_VALUES;+ if (schemaVersion >= 1.7f && enableDocValuesByDefault()) properties |= DOC_VALUES;if (schemaVersion < 1.7f) properties |= UNINVERTIBLE;@@ -1161,15 +1161,17 @@ public abstract class FieldType extends FieldProperties {ErrorCode.SERVER_ERROR, "Field type " + this + " does not support doc values");}- /** Returns whether this field type supports docValues. By default none do. */- protected boolean doesTypeSupportDocValues() {- try {- // TODO: In Solr 10.0 change this such that checkSupportsDocValues() calls this method instead- checkSupportsDocValues();- return true;- } catch (Exception ignored) {- return false;- }+ /**+ * Returns whether this field type should enable docValues by default for schemaVersion >= 1.7.+ * This should not be enabled for fields that did not have docValues implemented by Solr 9.7, as+ * users may have indexed documents without docValues (since they weren't supported). Flipping the+ * default docValues values when they upgrade to a new version will break their index+ * compatibility.+ *+ * <p>New field types can enable this without issue, as long as they support docValues.+ */+ protected boolean enableDocValuesByDefault() {+ return false;}public static final String TYPE = "type";diff --git a/solr/core/src/java/org/apache/solr/schema/LatLonPointSpatialField.java b/solr/core/src/java/org/apache/solr/schema/LatLonPointSpatialField.javaindex b237b79ad37..5010ba27075 100644--- a/solr/core/src/java/org/apache/solr/schema/LatLonPointSpatialField.java+++ b/solr/core/src/java/org/apache/solr/schema/LatLonPointSpatialField.java@@ -70,7 +70,7 @@ public class LatLonPointSpatialField}@Override- protected boolean doesTypeSupportDocValues() {+ protected boolean enableDocValuesByDefault() {return true;}diff --git a/solr/core/src/java/org/apache/solr/schema/PointType.java b/solr/core/src/java/org/apache/solr/schema/PointType.javaindex 2f80fc24722..3d68af9449c 100644--- a/solr/core/src/java/org/apache/solr/schema/PointType.java+++ b/solr/core/src/java/org/apache/solr/schema/PointType.java@@ -172,11 +172,6 @@ public class PointType extends CoordinateFieldType implements SpatialQueryable {}}- @Override- protected boolean doesTypeSupportDocValues() {- return true;- }-/*** Calculates the range and creates a RangeQuery (bounding box) wrapped in a BooleanQuery (unless* the dimension is 1, one range for every dimension, AND'd together by a Booleandiff --git a/solr/core/src/java/org/apache/solr/schema/PrimitiveFieldType.java b/solr/core/src/java/org/apache/solr/schema/PrimitiveFieldType.javaindex 2ce3203ae71..cc3fb3f7ff0 100644--- a/solr/core/src/java/org/apache/solr/schema/PrimitiveFieldType.java+++ b/solr/core/src/java/org/apache/solr/schema/PrimitiveFieldType.java@@ -39,7 +39,7 @@ public abstract class PrimitiveFieldType extends FieldType {}@Override- protected boolean doesTypeSupportDocValues() {+ protected boolean enableDocValuesByDefault() {return true;}diff --git a/solr/core/src/java/org/apache/solr/schema/SortableTextField.java b/solr/core/src/java/org/apache/solr/schema/SortableTextField.javaindex 4811a1dfa59..9bc6a3f7cb9 100644--- a/solr/core/src/java/org/apache/solr/schema/SortableTextField.java+++ b/solr/core/src/java/org/apache/solr/schema/SortableTextField.java@@ -151,7 +151,7 @@ public class SortableTextField extends TextField {}@Override- protected boolean doesTypeSupportDocValues() {+ protected boolean enableDocValuesByDefault() {return true;}diff --git a/solr/core/src/test-files/solr/collection1/conf/schema-binaryfield.xml b/solr/core/src/test-files/solr/collection1/conf/schema-binaryfield.xmlindex 7d5b3f09051..51c2b26a832 100644--- a/solr/core/src/test-files/solr/collection1/conf/schema-binaryfield.xml+++ b/solr/core/src/test-files/solr/collection1/conf/schema-binaryfield.xml@@ -33,7 +33,7 @@<field name="id" type="string" indexed="true" stored="true" multiValued="false" required="true"/><field name="data" type="binary" stored="true"/>- <field name="data_dv" type="binary" stored="false" />+ <field name="data_dv" type="binary" stored="false" docValues="true" /><uniqueKey>id</uniqueKey>diff --git a/solr/modules/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java b/solr/modules/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.javaindex 87a09c56212..9d72b88ee38 100644--- a/solr/modules/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java+++ b/solr/modules/analysis-extras/src/java/org/apache/solr/schema/ICUCollationField.java@@ -316,7 +316,7 @@ public class ICUCollationField extends FieldType {}@Override- protected boolean doesTypeSupportDocValues() {+ protected boolean enableDocValuesByDefault() {return true;}diff --git a/solr/solr-ref-guide/modules/indexing-guide/pages/docvalues.adoc b/solr/solr-ref-guide/modules/indexing-guide/pages/docvalues.adocindex 1cf3edf9b92..34b836def78 100644--- a/solr/solr-ref-guide/modules/indexing-guide/pages/docvalues.adoc+++ b/solr/solr-ref-guide/modules/indexing-guide/pages/docvalues.adoc@@ -35,7 +35,21 @@ This approach promises to relieve some of the memory requirements of the fieldCa== Enabling DocValues[IMPORTANT]-DocValues are enabled by default for all field types that support them (when using a schemaVersion >= 1.7).+DocValues are enabled by default for most field types that support them *when using a schemaVersion >= 1.7*.++* Primitive Fields+** `Numeric` (not `DenseVectorField`)+** `Boolean`+** `String`+** `Date`+** `UUID`+** `Enum`+* Sorting Fields+** `CollationField`+** `ICUCollationField`+** `SortableTextField`+** `SortableBinaryField`+* `LatLonPointSpacialField` (not `PointType`, which is different than `PointField`)When using an earlier schemaVersion (\<= 1.6), you only need to enable docValues for a field that you will use it with.As with all schema design, you need to define a field type and then define fields of that type with docValues enabled.diff --git a/solr/solr-ref-guide/modules/indexing-guide/pages/field-type-definitions-and-properties.adoc b/solr/solr-ref-guide/modules/indexing-guide/pages/field-type-definitions-and-properties.adocindex f085682b6cb..ee613d53cf9 100644--- a/solr/solr-ref-guide/modules/indexing-guide/pages/field-type-definitions-and-properties.adoc+++ b/solr/solr-ref-guide/modules/indexing-guide/pages/field-type-definitions-and-properties.adoc@@ -198,7 +198,7 @@ The table below includes the default value for most `FieldType` implementations|Property |Description |Implicit Default|`indexed` |If `true`, the value of the field can be used in queries to retrieve matching documents. |`true`|`stored` |If `true`, the actual value of the field can be retrieved by queries. |`true`-|`docValues` |If `true`, the value of the field will be put in a column-oriented xref:docvalues.adoc[] structure. |`true`+|`docValues` |If `true`, the value of the field will be put in a column-oriented xref:docvalues.adoc[] structure. |`true` for xref:indexing-guide:docvalues.adoc#enabling-docvalues[_most_ fields]|`sortMissingFirst`, `sortMissingLast` |Control the placement of documents when a sort field is not present. |`false`|`multiValued` |If `true`, indicates that a single document might contain multiple values for this field type. |`false`|`uninvertible` |If `true`, indicates that an `indexed="true" docValues="false"` field can be "un-inverted" at query time to build up large in memory data structure to serve in place of xref:docvalues.adoc[]. |`false`diff --git a/solr/solr-ref-guide/modules/query-guide/pages/spatial-search.adoc b/solr/solr-ref-guide/modules/query-guide/pages/spatial-search.adocindex 868938e31e5..cb714e2f8ac 100644--- a/solr/solr-ref-guide/modules/query-guide/pages/spatial-search.adoc+++ b/solr/solr-ref-guide/modules/query-guide/pages/spatial-search.adoc@@ -27,7 +27,7 @@ Using spatial search, you can:There are four main field types available for spatial search:-* `LatLonPointSpatialField`+* `LatLonPointSpatialField` (has `docValues` enabled by default)* `PointType`* `SpatialRecursivePrefixTreeFieldType` (RPT for short), including `RptWithGeometrySpatialField`, a derivative* `BBoxField`diff --git a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adocindex ee1cae69deb..e6413f8cd15 100644--- a/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc+++ b/solr/solr-ref-guide/modules/upgrade-notes/pages/major-changes-in-solr-9.adoc@@ -70,8 +70,9 @@ Due to changes in Lucene 9, that isn't possible any more.== Solr 9.7=== SchemaVersion upgraded to 1.7The default schemaVersion is now 1.7.-With the new schema version, all fields and fieldTypes that support docValues (StrField, *PointField, BoolField, etc.)-will have docValues enabled by default.+With the new schema version, most fields and fieldTypes that support docValues will have docValues enabled by default.+See the xref:indexing-guide:docvalues.adoc#enabling-docvalues[Enabling Doc Values Section] for a complete list.+Also, fields will be unable to be uninverted by default. (`uninvertible=false`)In order to upgrade from schemaVersion 1.6 to 1.7, a re-index of all data will generally be required.@@ -81,7 +82,7 @@ https://issues.apache.org/jira/browse/LUCENE-9334[Lucene does not allow adding dHowever, if any of the following is true, a re-index is not required:* All applicable fields already have docValues enabled in the original schema-* The schema is updated to explicitly default `docValues="false"` for all fields and fieldTypes that did not have an explicit `docValues` default provided.+* The schema is updated to xref:indexing-guide:docvalues.adoc#disabling-docvalues[explicitly disable docValues] for all fields and fieldTypes that did not have an explicit `docValues` default provided.The `uninvertible=false` default has no impact on the index, so is unrelated to re-indexing concerns.diff --git a/solr/test-framework/src/java/org/apache/solr/schema/SortableBinaryField.java b/solr/test-framework/src/java/org/apache/solr/schema/SortableBinaryField.javaindex 5c6d31c43ba..28edf725d14 100644--- a/solr/test-framework/src/java/org/apache/solr/schema/SortableBinaryField.java+++ b/solr/test-framework/src/java/org/apache/solr/schema/SortableBinaryField.java@@ -37,7 +37,7 @@ public class SortableBinaryField extends BinaryField {}@Override- protected boolean doesTypeSupportDocValues() {+ protected boolean enableDocValuesByDefault() {return true;}