Apache Solr open-source search software
SOLR-15570: Include declared but unused fields from the schema in table metadata (#241)
Timothy Potter6056b6814de1
diff --git a/solr/core/src/java/org/apache/solr/handler/sql/SolrSchema.java b/solr/core/src/java/org/apache/solr/handler/sql/SolrSchema.javaindex 413bbf792e6..d527018f6db 100644--- a/solr/core/src/java/org/apache/solr/handler/sql/SolrSchema.java+++ b/solr/core/src/java/org/apache/solr/handler/sql/SolrSchema.java@@ -23,6 +23,8 @@ import java.util.HashMap;import java.util.Map;import java.util.Properties;import java.util.Set;+import java.util.stream.Collectors;+import java.util.stream.Stream;import com.google.common.collect.ImmutableMap;import org.apache.calcite.rel.type.RelDataType;@@ -111,33 +113,40 @@ class SolrSchema extends AbstractSchema implements Closeable {}}- private Map<String, LukeResponse.FieldTypeInfo> getFieldTypeInfo(final String collection) {+ private LukeResponse getSchema(final String collection) {final String zk = this.properties.getProperty("zk");PKIAuthenticationPlugin.withServerIdentity(true);try {LukeRequest lukeRequest = new LukeRequest();- lukeRequest.setShowSchema(true); // for custom type info ...+ lukeRequest.setShowSchema(true); // for empty fields and custom type info ...lukeRequest.setNumTerms(0);- return lukeRequest.process(solrClientCache.getCloudSolrClient(zk), collection).getFieldTypeInfo();+ return lukeRequest.process(solrClientCache.getCloudSolrClient(zk), collection);} catch (SolrServerException | IOException e) {throw new RuntimeException(e);} finally {PKIAuthenticationPlugin.withServerIdentity(false);}}-+RelProtoDataType getRelDataType(String collection) {// Temporary type factory, just for the duration of this method. Allowable// because we're creating a proto-type, not a type; before being used, the// proto-type will be copied into a real type factory.final RelDataTypeFactory typeFactory = new SqlTypeFactoryImpl(RelDataTypeSystem.DEFAULT);final RelDataTypeFactory.Builder fieldInfo = typeFactory.builder();- Map<String, LukeResponse.FieldInfo> luceneFieldInfoMap = getFieldInfo(collection);++ // Get fields that have data, including dynamic field instances+ Map<String, LukeResponse.FieldInfo> fieldsInUseMap = getFieldInfo(collection);++ LukeResponse schema = getSchema(collection);+ // merge the actual fields in use returned by Luke with the declared fields in the schema that are empty+ Map<String, LukeResponse.FieldInfo> combinedFields = Stream.of(fieldsInUseMap, schema.getFieldInfo())+ .flatMap(map -> map.entrySet().stream())+ .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue, (v1, v2) -> v1));- Map<String, LukeResponse.FieldTypeInfo> fieldTypeInfoMap = null; // loaded lazily if neededMap<String, Class<?>> javaClassForTypeMap = new HashMap<>(); // local cache for custom field types we've already resolved- for (Map.Entry<String, LukeResponse.FieldInfo> entry : luceneFieldInfoMap.entrySet()) {+ for (Map.Entry<String, LukeResponse.FieldInfo> entry : combinedFields.entrySet()) {LukeResponse.FieldInfo luceneFieldInfo = entry.getValue();String luceneFieldType = luceneFieldInfo.getType();@@ -173,11 +182,7 @@ class SolrSchema extends AbstractSchema implements Closeable {default:Class<?> javaClass = javaClassForTypeMap.get(luceneFieldType);if (javaClass == null) {- if (fieldTypeInfoMap == null) {- // lazily go to luke for the field type info ...- fieldTypeInfoMap = getFieldTypeInfo(collection);- }- javaClass = guessJavaClassForFieldType(fieldTypeInfoMap.get(luceneFieldType));+ javaClass = guessJavaClassForFieldType(schema.getFieldTypeInfo().get(luceneFieldType));javaClassForTypeMap.put(luceneFieldType, javaClass);}type = typeFactory.createJavaType(javaClass);diff --git a/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java b/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.javaindex c3420fd5b18..f7209a1744d 100644--- a/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java+++ b/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java@@ -2232,4 +2232,20 @@ public class TestSQLHandler extends SolrCloudTestCase {// select * w/o limit is not supported by Solr SQLexpectThrows(IOException.class, () -> expectResults("SELECT * FROM $ALIAS", -1));}++ @Test+ public void testSelectEmptyField() throws Exception {+ new UpdateRequest()+ .add("id", "01")+ .add("id", "02")+ .add("id", "03")+ .add("id", "04")+ .add("id", "05")+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);++ // stringx is declared in the schema but has no docs+ expectResults("SELECT id, stringx FROM $ALIAS", 5);+ // notafield_i matches a dynamic field pattern but has no docs, so don't allow this+ expectThrows(IOException.class, () -> expectResults("SELECT id, stringx, notafield_i FROM $ALIAS", 5));+ }}diff --git a/solr/solr-ref-guide/src/parallel-sql-interface.adoc b/solr/solr-ref-guide/src/parallel-sql-interface.adocindex 7837b956cdc..68caf06e879 100644--- a/solr/solr-ref-guide/src/parallel-sql-interface.adoc+++ b/solr/solr-ref-guide/src/parallel-sql-interface.adoc@@ -40,7 +40,7 @@ Column names in the SQL query map directly to fields in the Solr index for the cThese identifiers are case sensitive.Aliases are supported, and can be referenced in the `ORDER BY` clause.-The `*` syntax to indicate all fields is not supported in either limited or unlimited queries.+The `SELECT *` syntax to indicate all fields is only supported for queries with a `LIMIT` clause.The `score` field can be used only with queries that contain a `LIMIT` clause.For example, we could index Solr's sample documents and then construct an SQL query like this: