Apache Solr open-source search software
SOLR-15456: Get field type info from luke for custom fields instead of defaulting to String for Parallel SQL (#181)
Timothy Potterae7b276d50ed
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex a50911483b6..0227c272f46 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -149,6 +149,8 @@ when told to. The admin UI now tells it to. (Nazerke Seidan, David Smiley)This was already working for XML & "javabin"/SolrJ. Previously, omitting the ID would be confusedfor a partial/atomic update. (David Smiley)+* SOLR-15456: Get field type info from luke for custom fields instead of defaulting to String in Parallel SQL (Timothy Potter)+Other Changes----------------------* SOLR-14656: Autoscaling framework removed (Ishan Chattopadhyaya, noble, Ilan Ginzburg)diff --git a/solr/core/src/java/org/apache/solr/handler/sql/SolrEnumerator.java b/solr/core/src/java/org/apache/solr/handler/sql/SolrEnumerator.javaindex fc1ea7f63f4..944435c6366 100644--- a/solr/core/src/java/org/apache/solr/handler/sql/SolrEnumerator.java+++ b/solr/core/src/java/org/apache/solr/handler/sql/SolrEnumerator.java@@ -24,6 +24,7 @@ import org.slf4j.LoggerFactory;import java.io.IOException;import java.lang.invoke.MethodHandles;+import java.util.Date;import java.util.List;import java.util.ArrayList;import java.util.Map;@@ -87,6 +88,11 @@ class SolrEnumerator implements Enumerator<Object> {return val;}+ if (clazz.equals(Date.class)) {+ // make sure the val returned is a Date as Avatica cannot deal with string values for Timestamp fields+ val = tuple.getDate(field.getKey());+ }+if(val instanceof ArrayList) {ArrayList<?> arrayList = (ArrayList<?>) val;StringBuilder buf = new StringBuilder();diff --git a/solr/core/src/java/org/apache/solr/handler/sql/SolrFilter.java b/solr/core/src/java/org/apache/solr/handler/sql/SolrFilter.javaindex 2f817507e99..a04cdaca2de 100644--- a/solr/core/src/java/org/apache/solr/handler/sql/SolrFilter.java+++ b/solr/core/src/java/org/apache/solr/handler/sql/SolrFilter.java@@ -16,9 +16,11 @@*/package org.apache.solr.handler.sql;+import java.time.Instant;import java.util.ArrayList;import java.util.List;import java.util.Map;+import java.util.regex.Pattern;import org.apache.calcite.plan.RelOptCluster;import org.apache.calcite.plan.RelOptCost;@@ -33,12 +35,16 @@ import org.apache.calcite.rex.RexInputRef;import org.apache.calcite.rex.RexLiteral;import org.apache.calcite.rex.RexNode;import org.apache.calcite.sql.SqlKind;+import org.apache.calcite.sql.type.SqlTypeName;import org.apache.calcite.util.Pair;/*** Implementation of a {@link org.apache.calcite.rel.core.Filter} relational expression in Solr.*/class SolrFilter extends Filter implements SolrRel {++ private static final Pattern CALCITE_TIMESTAMP_REGEX = Pattern.compile("^\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}$");+SolrFilter(RelOptCluster cluster,RelTraitSet traitSet,@@ -119,9 +125,10 @@ class SolrFilter extends Filter implements SolrRel {String fieldName = gte.getKey();String query = null;if (fieldName.equals(lte.getKey()) && compareRexLiteral(gte.right, lte.right) < 0) {- query = fieldName + ":[" + gte.getValue() + " TO " + lte.getValue() + "]";+ query = fieldName + ":[" + toSolrLiteral(gte.getValue()) + " TO " + toSolrLiteral(lte.getValue()) + "]";this.negativeQuery = false; // so we don't get *:* AND range}+return query;}@@ -208,36 +215,39 @@ class SolrFilter extends Filter implements SolrRel {}Pair<String, RexLiteral> binaryTranslated = getFieldValuePair(node);+ final String key = binaryTranslated.getKey();+ RexLiteral value = binaryTranslated.getValue();switch (kind) {case EQUALS:- String terms = binaryTranslated.getValue().toString().trim();- terms = terms.replace("'", "");+ SqlTypeName fieldTypeName = ((RexCall) node).getOperands().get(0).getType().getSqlTypeName();+ String terms = toSolrLiteralForEquals(value, fieldTypeName).trim();+boolean wrappedQuotes = false;if (!terms.startsWith("(") && !terms.startsWith("[") && !terms.startsWith("{")) {terms = "\"" + terms + "\"";wrappedQuotes = true;}- String clause = binaryTranslated.getKey() + ":" + terms;+ String clause = key + ":" + terms;if (terms.contains("*") && wrappedQuotes) {clause = "{!complexphrase}" + clause;}this.negativeQuery = false;return clause;case NOT_EQUALS:- return "-(" + binaryTranslated.getKey() + ":" + binaryTranslated.getValue() + ")";+ return "-(" + key + ":" + toSolrLiteral(value) + ")";case LESS_THAN:this.negativeQuery = false;- return "(" + binaryTranslated.getKey() + ": [ * TO " + binaryTranslated.getValue() + " })";+ return "(" + key + ": [ * TO " + toSolrLiteral(value) + " })";case LESS_THAN_OR_EQUAL:this.negativeQuery = false;- return "(" + binaryTranslated.getKey() + ": [ * TO " + binaryTranslated.getValue() + " ])";+ return "(" + key + ": [ * TO " + toSolrLiteral(value) + " ])";case GREATER_THAN:this.negativeQuery = false;- return "(" + binaryTranslated.getKey() + ": { " + binaryTranslated.getValue() + " TO * ])";+ return "(" + key + ": { " + toSolrLiteral(value) + " TO * ])";case GREATER_THAN_OR_EQUAL:this.negativeQuery = false;- return "(" + binaryTranslated.getKey() + ": [ " + binaryTranslated.getValue() + " TO * ])";+ return "(" + key + ": [ " + toSolrLiteral(value) + " TO * ])";case LIKE:return translateLike(node, false);case IS_NOT_NULL:@@ -248,6 +258,39 @@ class SolrFilter extends Filter implements SolrRel {}}+ // translate to a literal string value for Solr queries, such as translating a+ // Calcite timestamp value into an ISO-8601 formatted timestamp that Solr likes+ private String toSolrLiteral(RexLiteral literal) {+ Object value2 = literal.getValue2();+ SqlTypeName typeName = literal.getTypeName();+ final String solrLiteral;+ if (value2 instanceof Long && (typeName == SqlTypeName.TIMESTAMP || typeName == SqlTypeName.TIMESTAMP_WITH_LOCAL_TIME_ZONE)) {+ // return as an ISO-8601 timestamp+ solrLiteral = Instant.ofEpochMilli((Long) value2).toString();+ } else {+ solrLiteral = value2.toString();+ }+ return solrLiteral;+ }++ // special case handling for expressions like: WHERE timestamp = '2021-06-04 04:00:00'+ // Calcite passes the right hand side as a string instead of as a Long+ private String toSolrLiteralForEquals(RexLiteral literal, SqlTypeName fieldTypeName) {+ Object value2 = literal.getValue2();+ final String solrLiteral;+ // oddly, for = criteria with a timestamp field, Calcite passes us a String instead of a Long as it does with other operators like >+ if (value2 instanceof String && fieldTypeName == SqlTypeName.TIMESTAMP && CALCITE_TIMESTAMP_REGEX.matcher((String) value2).matches()) {+ String timestamp = ((String) value2).replace(' ', 'T').replace("'", "");+ if (Character.isDigit(timestamp.charAt(timestamp.length() - 1))) {+ timestamp += "Z";+ }+ solrLiteral = timestamp;+ } else {+ solrLiteral = toSolrLiteral(literal);+ }+ return solrLiteral;+ }+protected Pair<String, RexLiteral> getFieldValuePair(RexNode node) {if (!(node instanceof RexCall)) {throw new AssertionError("expected RexCall for predicate but found: " + node);diff --git a/solr/core/src/java/org/apache/solr/handler/sql/SolrSchema.java b/solr/core/src/java/org/apache/solr/handler/sql/SolrSchema.javaindex caa30762d62..413bbf792e6 100644--- a/solr/core/src/java/org/apache/solr/handler/sql/SolrSchema.java+++ b/solr/core/src/java/org/apache/solr/handler/sql/SolrSchema.java@@ -18,10 +18,13 @@ package org.apache.solr.handler.sql;import java.io.Closeable;import java.io.IOException;+import java.util.Date;+import java.util.HashMap;import java.util.Map;import java.util.Properties;import java.util.Set;+import com.google.common.collect.ImmutableMap;import org.apache.calcite.rel.type.RelDataType;import org.apache.calcite.rel.type.RelDataTypeFactory;import org.apache.calcite.rel.type.RelDataTypeImpl;@@ -38,8 +41,11 @@ import org.apache.solr.client.solrj.response.LukeResponse;import org.apache.solr.common.cloud.Aliases;import org.apache.solr.common.cloud.ClusterState;import org.apache.solr.common.cloud.ZkStateReader;--import com.google.common.collect.ImmutableMap;+import org.apache.solr.schema.DateValueFieldType;+import org.apache.solr.schema.DoubleValueFieldType;+import org.apache.solr.schema.FloatValueFieldType;+import org.apache.solr.schema.IntValueFieldType;+import org.apache.solr.schema.LongValueFieldType;import org.apache.solr.security.PKIAuthenticationPlugin;class SolrSchema extends AbstractSchema implements Closeable {@@ -91,16 +97,28 @@ class SolrSchema extends AbstractSchema implements Closeable {return builder.build();}- private Map<String, LukeResponse.FieldInfo> getFieldInfo(String collection) {- String zk = this.properties.getProperty("zk");- CloudSolrClient cloudSolrClient = solrClientCache.getCloudSolrClient(zk);- // Send the Luke request using the server identity vs. the logged in user+ private Map<String, LukeResponse.FieldInfo> getFieldInfo(final String collection) {+ final String zk = this.properties.getProperty("zk");PKIAuthenticationPlugin.withServerIdentity(true);try {LukeRequest lukeRequest = new LukeRequest();lukeRequest.setNumTerms(0);- LukeResponse lukeResponse = lukeRequest.process(cloudSolrClient, collection);- return lukeResponse.getFieldInfo();+ return lukeRequest.process(solrClientCache.getCloudSolrClient(zk), collection).getFieldInfo();+ } catch (SolrServerException | IOException e) {+ throw new RuntimeException(e);+ } finally {+ PKIAuthenticationPlugin.withServerIdentity(false);+ }+ }++ private Map<String, LukeResponse.FieldTypeInfo> getFieldTypeInfo(final String collection) {+ final String zk = this.properties.getProperty("zk");+ PKIAuthenticationPlugin.withServerIdentity(true);+ try {+ LukeRequest lukeRequest = new LukeRequest();+ lukeRequest.setShowSchema(true); // for custom type info ...+ lukeRequest.setNumTerms(0);+ return lukeRequest.process(solrClientCache.getCloudSolrClient(zk), collection).getFieldTypeInfo();} catch (SolrServerException | IOException e) {throw new RuntimeException(e);} finally {@@ -116,12 +134,15 @@ class SolrSchema extends AbstractSchema implements Closeable {final RelDataTypeFactory.Builder fieldInfo = typeFactory.builder();Map<String, LukeResponse.FieldInfo> luceneFieldInfoMap = getFieldInfo(collection);- for(Map.Entry<String, LukeResponse.FieldInfo> entry : luceneFieldInfoMap.entrySet()) {+ Map<String, LukeResponse.FieldTypeInfo> fieldTypeInfoMap = null; // loaded lazily if needed+ Map<String, Class<?>> javaClassForTypeMap = new HashMap<>(); // local cache for custom field types we've already resolved++ for (Map.Entry<String, LukeResponse.FieldInfo> entry : luceneFieldInfoMap.entrySet()) {LukeResponse.FieldInfo luceneFieldInfo = entry.getValue();String luceneFieldType = luceneFieldInfo.getType();// SOLR-13414: Luke can return a field definition with no type in rare situations- if(luceneFieldType == null) {+ if (luceneFieldType == null) {continue;}@@ -146,8 +167,20 @@ class SolrSchema extends AbstractSchema implements Closeable {case "pdouble":type = typeFactory.createJavaType(Double.class);break;+ case "pdate":+ type = typeFactory.createJavaType(Date.class);+ break;default:- type = typeFactory.createJavaType(String.class);+ Class<?> javaClass = javaClassForTypeMap.get(luceneFieldType);+ if (javaClass == null) {+ if (fieldTypeInfoMap == null) {+ // lazily go to luke for the field type info ...+ fieldTypeInfoMap = getFieldTypeInfo(collection);+ }+ javaClass = guessJavaClassForFieldType(fieldTypeInfoMap.get(luceneFieldType));+ javaClassForTypeMap.put(luceneFieldType, javaClass);+ }+ type = typeFactory.createJavaType(javaClass);}/*@@ -159,9 +192,30 @@ class SolrSchema extends AbstractSchema implements Closeable {fieldInfo.add(entry.getKey(), type).nullable(true);}- fieldInfo.add("_query_",typeFactory.createJavaType(String.class));- fieldInfo.add("score",typeFactory.createJavaType(Double.class));+ fieldInfo.add("_query_", typeFactory.createJavaType(String.class));+ fieldInfo.add("score", typeFactory.createJavaType(Double.class));return RelDataTypeImpl.proto(fieldInfo.build());}++ private Class<?> guessJavaClassForFieldType(LukeResponse.FieldTypeInfo typeInfo) {+ Class<?> typeClass = null;+ if (typeInfo != null && !typeInfo.isTokenized() && typeInfo.getClassName() != null) {+ try {+ final Class<?> fieldTypeClass = getClass().getClassLoader().loadClass(typeInfo.getClassName());+ // a numeric type ... narrow down+ if (IntValueFieldType.class.isAssignableFrom(fieldTypeClass) || LongValueFieldType.class.isAssignableFrom(fieldTypeClass)) {+ typeClass = Long.class;+ } else if (FloatValueFieldType.class.isAssignableFrom(fieldTypeClass) || DoubleValueFieldType.class.isAssignableFrom(fieldTypeClass)) {+ typeClass = Double.class;+ } else if (DateValueFieldType.class.isAssignableFrom(fieldTypeClass)) {+ typeClass = Date.class;+ }+ } catch (ClassNotFoundException e) {+ throw new RuntimeException(e);+ }+ }+ // default to String if we could narrow it down by looking at the field type class+ return typeClass != null ? typeClass : String.class;+ }}diff --git a/solr/core/src/test-files/solr/configsets/sql/conf/schema.xml b/solr/core/src/test-files/solr/configsets/sql/conf/schema.xmlindex b5e2dea49ef..f0872a8ac61 100644--- a/solr/core/src/test-files/solr/configsets/sql/conf/schema.xml+++ b/solr/core/src/test-files/solr/configsets/sql/conf/schema.xml@@ -41,6 +41,31 @@<fieldType name="tlong" class="${solr.tests.LongFieldType}" docValues="true" precisionStep="8" positionIncrementGap="0"/><fieldType name="tdouble" class="${solr.tests.DoubleFieldType}" docValues="true" precisionStep="8" positionIncrementGap="0"/>+ <!-- Used to test proper type handling for custom field types based on built-in type classes -->+ <fieldType name="stringx" class="solr.StrField" docValues="true"/>+ <fieldType name="tintx" class="solr.TrieIntField" docValues="true"/>+ <fieldType name="pintx" class="solr.IntPointField" docValues="true"/>+ <fieldType name="tfloatx" class="solr.TrieFloatField" docValues="true"/>+ <fieldType name="pfloatx" class="solr.FloatPointField" docValues="true"/>+ <fieldType name="tlongx" class="solr.TrieLongField" docValues="true"/>+ <fieldType name="plongx" class="solr.LongPointField" docValues="true"/>+ <fieldType name="tdoublex" class="solr.TrieDoubleField" docValues="true"/>+ <fieldType name="pdoublex" class="solr.DoublePointField" docValues="true"/>+ <fieldType name="pdatex" class="solr.DatePointField" docValues="true"/>+ <field name="stringx" type="stringx" indexed="true" stored="true"/>+ <field name="tintx" type="tintx" indexed="true" stored="true"/>+ <field name="pintx" type="pintx" indexed="true" stored="true"/>+ <field name="pintxs" type="pintx" indexed="true" stored="true" multiValued="true"/>+ <field name="tfloatx" type="tfloatx" indexed="true" stored="true"/>+ <field name="pfloatx" type="pfloatx" indexed="true" stored="true"/>+ <field name="tlongx" type="tlongx" indexed="true" stored="true"/>+ <field name="plongx" type="plongx" indexed="true" stored="true"/>+ <field name="tdoublex" type="tdoublex" indexed="true" stored="true"/>+ <field name="pdoublex" type="pdoublex" indexed="true" stored="true"/>+ <field name="textx" type="text" indexed="true" stored="true"/>+ <field name="pdatex" type="pdatex" indexed="true" stored="true"/>+ <field name="pdatexs" type="pdatex" indexed="true" stored="true" multiValued="true"/>+<!-- Field type demonstrating an Analyzer failure --><fieldType name="failtype1" class="solr.TextField"><analyzer type="index">diff --git a/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java b/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.javaindex ef4d1fd288c..9df32797025 100644--- a/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java+++ b/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java@@ -20,6 +20,7 @@ import java.io.IOException;import java.util.ArrayList;import java.util.Arrays;import java.util.List;+import java.util.Locale;import org.apache.lucene.util.LuceneTestCase;import org.apache.lucene.util.LuceneTestCase.Slow;@@ -2033,6 +2034,91 @@ public class TestSQLHandler extends SolrCloudTestCase {.commit(cluster.getSolrClient(), COLLECTIONORALIAS);List<Tuple> tuples = expectResults("SELECT COUNT(1) as `the_count` FROM $ALIAS as `alias` WHERE (`alias`.`b_s`='foo' AND `alias`.`a_s` LIKE 'hell%' AND `alias`.`c_s` IS NOT NULL) HAVING (COUNT(1) > 0)", 1);- assertTrue(4L == tuples.get(0).getLong("the_count"));+ assertEquals(4L, (long) tuples.get(0).getLong("the_count"));+ }++ @Test+ public void testDateHandling() throws Exception {+ new UpdateRequest()+ .add("id", "1", "pdatex", "2021-06-01T00:00:00Z")+ .add("id", "2", "pdatex", "2021-06-02T02:00:00Z")+ .add("id", "3", "pdatex", "2021-06-03T03:00:00Z")+ .add("id", "4", "pdatex", "2021-06-04T04:00:00Z")+ .add("id", "5", "pdatex", "2021-06-01T01:01:00Z")+ .add("id", "6", "pdatex", "2021-06-02T02:02:00Z")+ .add("id", "7", "pdatex", "2021-06-03T03:03:00Z")+ .add("id", "8", "pdatex", "2021-06-04T04:04:00Z")+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);++ expectResults("SELECT id FROM $ALIAS WHERE pdatex IS NULL", 0);+ expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex IS NOT NULL", 8);+ expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex > '2021-06-02'", 6);+ expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex <= '2021-06-01'", 1);+ expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex BETWEEN '2021-06-03' AND '2021-06-05'", 4);+ expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex > '2021-06-04 04:00:00'", 1);+ expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex = '2021-06-04 04:00:00'", 1);+ expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex = CAST('2021-06-04 04:04:00' as TIMESTAMP)", 1);+ }++ @Test+ public void testAggsOnCustomFieldType() throws Exception {+ new UpdateRequest()+ .add(withMultiValuedField("pintxs", Arrays.asList(1,5),"id", "1", "tintx", "1", "pintx", "2", "tfloatx", "3.33", "pfloatx", "3.33", "tlongx", "1623875868000", "plongx", "1623875868000", "tdoublex", "3.14159265359", "pdoublex", "3.14159265359", "stringx", "A", "textx", "aaa", "pdatex", "2021-06-17T00:00:00Z"))+ .add(withMultiValuedField("pintxs", Arrays.asList(2,6),"id", "2", "tintx", "2", "pintx", "4", "tfloatx", "4.44", "pfloatx", "4.44", "tlongx", "1723875868000", "plongx", "1723875868000", "tdoublex", "6.14159265359", "pdoublex", "6.14159265359", "stringx", "B", "textx", "bbb", "pdatex", "2021-06-18T00:00:00Z"))+ .add(withMultiValuedField("pintxs", Arrays.asList(3,7),"id", "3", "tintx", "3", "pintx", "6", "tfloatx", "5.55", "pfloatx", "5.55", "tlongx", "1823875868000", "plongx", "1823875868000", "tdoublex", "9.14159265359", "pdoublex", "9.14159265359", "stringx", "C", "textx", "ccc", "pdatex", "2021-06-19T00:00:00Z"))+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);++ String dateStatsSql = "min(pdatex) as min_pdatex, max(pdatex) as max_pdatex";+ String numTypeStatsSql = toStatsSql(Arrays.asList("intx", "floatx", "longx", "doublex"));+ String sql = "SELECT min(pintxs) as min_pintxs, max(pintxs) as max_pintxs, "++ min("stringx")+", "+max("stringx")+", "+min("textx")+", "+max("textx")+", "+numTypeStatsSql+", "+dateStatsSql+" FROM $ALIAS";++ List<Tuple> tuples = expectResults(sql, 1);+ Tuple stats = tuples.get(0);+ assertEquals("A", stats.getString("min_stringx"));+ assertEquals("C", stats.getString("max_stringx"));+ assertEquals("aaa", stats.getString("min_textx"));+ assertEquals("ccc", stats.getString("max_textx"));+ assertEquals(1L, (long) stats.getLong("min_tintx"));+ assertEquals(3L, (long) stats.getLong("max_tintx"));+ assertEquals(2L, (long) stats.getLong("min_pintx"));+ assertEquals(6L, (long) stats.getLong("max_pintx"));+ assertEquals(1L, (long) stats.getLong("min_pintxs"));+ assertEquals(7L, (long) stats.getLong("max_pintxs"));+ assertEquals(1623875868000L, (long) stats.getLong("min_tlongx"));+ assertEquals(1823875868000L, (long) stats.getLong("max_tlongx"));+ assertEquals(1623875868000L, (long) stats.getLong("min_plongx"));+ assertEquals(1823875868000L, (long) stats.getLong("max_plongx"));+ final double delta = 0.00001d;+ assertEquals(3.33d, stats.getDouble("min_tfloatx"), delta);+ assertEquals(5.55d, stats.getDouble("max_tfloatx"), delta);+ assertEquals(3.33d, stats.getDouble("min_pfloatx"), delta);+ assertEquals(5.55d, stats.getDouble("max_pfloatx"), delta);+ assertEquals(3.14159265359d, stats.getDouble("min_tdoublex"), delta);+ assertEquals(9.14159265359d, stats.getDouble("max_tdoublex"), delta);+ assertEquals(3.14159265359d, stats.getDouble("min_pdoublex"), delta);+ assertEquals(9.14159265359d, stats.getDouble("max_pdoublex"), delta);+ assertNotNull(stats.getDate("min_pdatex"));+ assertNotNull(stats.getDate("max_pdatex"));+ }++ private String toStatsSql(List<String> types) {+ StringBuilder sb = new StringBuilder();+ for (String type : types) {+ if (sb.length() > 0) {+ sb.append(", ");+ }+ sb.append(min("t"+type)).append(", ").append(min("p"+type));+ sb.append(", ").append(max("t"+type)).append(", ").append(max("p"+type));+ }+ return sb.toString();+ }++ private String min(String type) {+ return String.format(Locale.ROOT, "min(%s) as min_%s", type, type);+ }++ private String max(String type) {+ return String.format(Locale.ROOT, "max(%s) as max_%s", type, type);}}