Apache Solr open-source search software
SOLR-15576: Allow filtering on ISO-8601 formatted timestamp literals in SQL WHERE clause (#247)
Timothy Potter072c9c8a74a0
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txtindex 0d17ea4900e..9570c17f959 100644--- a/solr/CHANGES.txt+++ b/solr/CHANGES.txt@@ -387,6 +387,8 @@ Improvements* SOLR-15564: Improve filtering expressions in /admin/metrics. (ab)+* SOLR-15576: Allow filtering on ISO-8601 formatted timestamp literals in SQL WHERE clause (Timothy Potter)+Optimizations---------------------* SOLR-15433: Replace transient core cache LRU by Caffeine cache. (Bruno Roustant)diff --git a/solr/core/src/java/org/apache/solr/handler/sql/SolrFilter.java b/solr/core/src/java/org/apache/solr/handler/sql/SolrFilter.javaindex 1c01b21ef3e..f5cabb7a2ca 100644--- a/solr/core/src/java/org/apache/solr/handler/sql/SolrFilter.java+++ b/solr/core/src/java/org/apache/solr/handler/sql/SolrFilter.java@@ -35,6 +35,7 @@ import org.apache.calcite.rex.*;import org.apache.calcite.sql.SqlKind;import org.apache.calcite.sql.type.SqlTypeName;import org.apache.calcite.util.Pair;+import org.apache.solr.client.solrj.util.ClientUtils;import org.slf4j.Logger;import org.slf4j.LoggerFactory;@@ -218,7 +219,8 @@ class SolrFilter extends Filter implements SolrRel {terms = terms.replace("'", "").replace('%', '*').replace('_', '?');boolean wrappedQuotes = false;if (!terms.startsWith("(") && !terms.startsWith("[") && !terms.startsWith("{")) {- terms = "\"" + terms + "\"";+ // restore the * and ? after escaping+ terms = "\"" + ClientUtils.escapeQueryChars(terms).replace("\\*", "*").replace("\\?", "?") + "\"";wrappedQuotes = true;}@@ -239,23 +241,10 @@ class SolrFilter extends Filter implements SolrRel {RexLiteral value = binaryTranslated.getValue();switch (kind) {case EQUALS:- SqlTypeName fieldTypeName = ((RexCall) node).getOperands().get(0).getType().getSqlTypeName();- String terms = toSolrLiteralForEquals(value, fieldTypeName).trim();-- boolean wrappedQuotes = false;- if (!terms.startsWith("(") && !terms.startsWith("[") && !terms.startsWith("{")) {- terms = "\"" + terms + "\"";- wrappedQuotes = true;- }-- String clause = key + ":" + terms;- if (terms.contains("*") && wrappedQuotes) {- clause = "{!complexphrase}" + clause;- }this.negativeQuery = false;- return clause;+ return toEqualsClause(key, value, node);case NOT_EQUALS:- return "-(" + key + ":" + toSolrLiteral(value) + ")";+ return "-(" + toEqualsClause(key, value, node) + ")";case LESS_THAN:this.negativeQuery = false;return "(" + key + ": [ * TO " + toSolrLiteral(value) + " })";@@ -278,6 +267,24 @@ class SolrFilter extends Filter implements SolrRel {}}+ private String toEqualsClause(String key, RexLiteral value, RexNode node) {+ SqlTypeName fieldTypeName = ((RexCall) node).getOperands().get(0).getType().getSqlTypeName();+ String terms = toSolrLiteralForEquals(value, fieldTypeName).trim();++ boolean wrappedQuotes = false;+ if (!terms.startsWith("(") && !terms.startsWith("[") && !terms.startsWith("{")) {+ terms = "\"" + ClientUtils.escapeQueryChars(terms) + "\"";+ wrappedQuotes = true;+ }++ String clause = key + ":" + terms;+ if (terms.contains("*") && wrappedQuotes) {+ clause = "{!complexphrase}" + clause;+ }++ return clause;+ }+// translate to a literal string value for Solr queries, such as translating a// Calcite timestamp value into an ISO-8601 formatted timestamp that Solr likesprivate String toSolrLiteral(RexLiteral literal) {@@ -339,15 +346,27 @@ class SolrFilter extends Filter implements SolrRel {if (a != null) {return a;}- final Pair<String, RexLiteral> b = translateBinary2(right, left);- if (b != null) {- return b;++ // we can swap these if doing an equals / not equals+ if (call.op.kind == SqlKind.EQUALS || call.op.kind == SqlKind.NOT_EQUALS) {+ final Pair<String, RexLiteral> b = translateBinary2(right, left);+ if (b != null) {+ return b;+ }}if (left.getKind() == SqlKind.CAST && right.getKind() == SqlKind.CAST) {return translateBinary2(((RexCall)left).operands.get(0), ((RexCall)right).operands.get(0));}+ // for WHERE clause like: pdatex >= '2021-07-13T15:12:10.037Z'+ if (left.getKind() == SqlKind.INPUT_REF && right.getKind() == SqlKind.CAST) {+ final RexCall cast = ((RexCall)right);+ if (cast.operands.size() == 1 && cast.operands.get(0).getKind() == SqlKind.LITERAL) {+ return translateBinary2(left, cast.operands.get(0));+ }+ }+throw new AssertionError("cannot translate call " + call);}@@ -493,10 +512,14 @@ class SolrFilter extends Filter implements SolrRel {}return a;}- final Pair<String, RexLiteral> b = translateBinary2(right, left);- if (b != null) {- return b;++ if (call.op.kind == SqlKind.EQUALS || call.op.kind == SqlKind.NOT_EQUALS) {+ final Pair<String, RexLiteral> b = translateBinary2(right, left);+ if (b != null) {+ return b;+ }}+throw new AssertionError("cannot translate call " + call);}diff --git a/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java b/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.javaindex a1a2882998a..adb7332f62a 100644--- a/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java+++ b/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java@@ -338,10 +338,10 @@ public class TestSQLHandler extends SolrCloudTestCase {.add("id", "2", "text_t", "XXXX XXXX", "str_s", "b", "field_i", "8").add("id", "3", "text_t", "XXXX XXXX", "str_s", "a", "field_i", "20").add("id", "4", "text_t", "XXXX XXXX", "str_s", "b", "field_i", "11")- .add("id", "5", "text_t", "XXXX XXXX", "str_s", "c", "field_i", "30")- .add("id", "6", "text_t", "XXXX XXXX", "str_s", "c", "field_i", "40")- .add("id", "7", "text_t", "XXXX XXXX", "str_s", "c", "field_i", "50")- .add("id", "8", "text_t", "XXXX XXXX", "str_s", "c", "field_i", "60")+ .add("id", "5", "text_t", "XXXX XXXX", "str_s", "c", "field_i", "30", "specialchars_s", "witha|pipe")+ .add("id", "6", "text_t", "XXXX XXXX", "str_s", "c", "field_i", "40", "specialchars_s", "witha\\slash")+ .add("id", "7", "text_t", "XXXX XXXX", "str_s", "c", "field_i", "50", "specialchars_s", "witha!bang")+ .add("id", "8", "text_t", "XXXX XXXX", "str_s", "c", "field_i", "60", "specialchars_s", "witha\"quote").commit(cluster.getSolrClient(), COLLECTIONORALIAS);String baseUrl = cluster.getJettySolrRunners().get(0).getBaseUrl().toString() + "/" + COLLECTIONORALIAS;@@ -452,6 +452,14 @@ public class TestSQLHandler extends SolrCloudTestCase {tuple = tuples.get(1);assertEquals("8", tuple.get("id"));+ expectResults("SELECT id FROM $ALIAS WHERE str_s = 'a'", 2);+ expectResults("SELECT id FROM $ALIAS WHERE 'a' = str_s", 2);+ expectResults("SELECT id FROM $ALIAS WHERE str_s <> 'c'", 4);+ expectResults("SELECT id FROM $ALIAS WHERE 'c' <> str_s", 4);+ expectResults("SELECT id FROM $ALIAS WHERE specialchars_s = 'witha\"quote'", 1);+ expectResults("SELECT id FROM $ALIAS WHERE specialchars_s = 'witha|pipe'", 1);+ expectResults("SELECT id FROM $ALIAS WHERE specialchars_s LIKE 'with%'", 4);+ expectResults("SELECT id FROM $ALIAS WHERE specialchars_s LIKE 'witha|%'", 1);}@Test@@ -1641,6 +1649,17 @@ public class TestSQLHandler extends SolrCloudTestCase {assert (tuple.EOF);assert (tuple.EXCEPTION);assert (tuple.getException().contains("No match found for function signature blah"));++ // verify exception message formatting with wildcard query+ sParams = mapParams(CommonParams.QT, "/sql", "aggregationMode", "map_reduce",+ "stmt",+ "select str_s from collection1 where not_a_field LIKE 'foo%'");++ solrStream = new SolrStream(baseUrl, sParams);+ tuple = getTuple(new ExceptionStream(solrStream));+ assert (tuple.EOF);+ assert (tuple.EXCEPTION);+ assert (tuple.getException().contains("Column 'not_a_field' not found in any table"));}@Test@@ -2058,6 +2077,24 @@ public class TestSQLHandler extends SolrCloudTestCase {expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex BETWEEN '2021-06-03' AND '2021-06-05'", 4);}+ @Test+ public void testISO8601TimestampFiltering() throws Exception {+ new UpdateRequest()+ .add("id", "1", "pdatex", "2021-07-13T15:12:09.037Z")+ .add("id", "2", "pdatex", "2021-07-13T15:12:10.037Z")+ .add("id", "3", "pdatex", "2021-07-13T15:12:11.037Z")+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);++ expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex >= CAST('2021-07-13 15:12:10.037' as TIMESTAMP)", 2);+ expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex >= '2021-07-13T15:12:10.037Z'", 2);+ expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex < '2021-07-13T15:12:10.037Z'", 1);+ expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex = '2021-07-13T15:12:10.037Z'", 1);+ expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex <> '2021-07-13T15:12:10.037Z'", 2);+ expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex BETWEEN '2021-07-13T15:12:09.037Z' AND '2021-07-13T15:12:10.037Z' ORDER BY pdatex ASC", 2);+ expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex >= '2021-07-13T15:12:10.037Z'", 2);+ expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex >= '2021-07-13T15:12:10.037Z' ORDER BY pdatex ASC LIMIT 10", 2);+ }+@Testpublic void testAggsOnCustomFieldType() throws Exception {new UpdateRequest()diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JDBCStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JDBCStream.javaindex 778f60cfc1a..8045e34ddc9 100644--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JDBCStream.java+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JDBCStream.java@@ -283,8 +283,7 @@ public class JDBCStream extends TupleStream implements Expressible {resultSet = statement.executeQuery(sqlQuery);resultSet.setFetchSize(fetchSize);} catch (SQLException e) {- throw new IOException(String.format(Locale.ROOT, "Failed to execute sqlQuery '%s' against JDBC connection '%s'.\n"- + e.getMessage(), sqlQuery, connectionUrl), e);+ throw new IOException(String.format(Locale.ROOT, "Failed to execute sqlQuery '%s' against JDBC connection '%s'.%nCaused by: %s", sqlQuery, connectionUrl, e.getMessage()), e);}try{