Skip to content

armgit / solr

Public Imported from GitHub
0

Apache Solr open-source search software

SOLR-15576: Allow filtering on ISO-8601 formatted timestamp literals in SQL WHERE clause (#247)

Timothy Potter072c9c8a74a0
diff --git a/solr/CHANGES.txt b/solr/CHANGES.txt
index 0d17ea4900e..9570c17f959 100644
--- a/solr/CHANGES.txt
+++ b/solr/CHANGES.txt
@@ -387,6 +387,8 @@ Improvements
* SOLR-15564: Improve filtering expressions in /admin/metrics. (ab)
+* SOLR-15576: Allow filtering on ISO-8601 formatted timestamp literals in SQL WHERE clause (Timothy Potter)
+
Optimizations
---------------------
* SOLR-15433: Replace transient core cache LRU by Caffeine cache. (Bruno Roustant)
diff --git a/solr/core/src/java/org/apache/solr/handler/sql/SolrFilter.java b/solr/core/src/java/org/apache/solr/handler/sql/SolrFilter.java
index 1c01b21ef3e..f5cabb7a2ca 100644
--- a/solr/core/src/java/org/apache/solr/handler/sql/SolrFilter.java
+++ b/solr/core/src/java/org/apache/solr/handler/sql/SolrFilter.java
@@ -35,6 +35,7 @@ import org.apache.calcite.rex.*;
import org.apache.calcite.sql.SqlKind;
import org.apache.calcite.sql.type.SqlTypeName;
import org.apache.calcite.util.Pair;
+import org.apache.solr.client.solrj.util.ClientUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -218,7 +219,8 @@ class SolrFilter extends Filter implements SolrRel {
terms = terms.replace("'", "").replace('%', '*').replace('_', '?');
boolean wrappedQuotes = false;
if (!terms.startsWith("(") && !terms.startsWith("[") && !terms.startsWith("{")) {
- terms = "\"" + terms + "\"";
+ // restore the * and ? after escaping
+ terms = "\"" + ClientUtils.escapeQueryChars(terms).replace("\\*", "*").replace("\\?", "?") + "\"";
wrappedQuotes = true;
}
@@ -239,23 +241,10 @@ class SolrFilter extends Filter implements SolrRel {
RexLiteral value = binaryTranslated.getValue();
switch (kind) {
case EQUALS:
- SqlTypeName fieldTypeName = ((RexCall) node).getOperands().get(0).getType().getSqlTypeName();
- String terms = toSolrLiteralForEquals(value, fieldTypeName).trim();
-
- boolean wrappedQuotes = false;
- if (!terms.startsWith("(") && !terms.startsWith("[") && !terms.startsWith("{")) {
- terms = "\"" + terms + "\"";
- wrappedQuotes = true;
- }
-
- String clause = key + ":" + terms;
- if (terms.contains("*") && wrappedQuotes) {
- clause = "{!complexphrase}" + clause;
- }
this.negativeQuery = false;
- return clause;
+ return toEqualsClause(key, value, node);
case NOT_EQUALS:
- return "-(" + key + ":" + toSolrLiteral(value) + ")";
+ return "-(" + toEqualsClause(key, value, node) + ")";
case LESS_THAN:
this.negativeQuery = false;
return "(" + key + ": [ * TO " + toSolrLiteral(value) + " })";
@@ -278,6 +267,24 @@ class SolrFilter extends Filter implements SolrRel {
}
}
+ private String toEqualsClause(String key, RexLiteral value, RexNode node) {
+ SqlTypeName fieldTypeName = ((RexCall) node).getOperands().get(0).getType().getSqlTypeName();
+ String terms = toSolrLiteralForEquals(value, fieldTypeName).trim();
+
+ boolean wrappedQuotes = false;
+ if (!terms.startsWith("(") && !terms.startsWith("[") && !terms.startsWith("{")) {
+ terms = "\"" + ClientUtils.escapeQueryChars(terms) + "\"";
+ wrappedQuotes = true;
+ }
+
+ String clause = key + ":" + terms;
+ if (terms.contains("*") && wrappedQuotes) {
+ clause = "{!complexphrase}" + clause;
+ }
+
+ return clause;
+ }
+
// translate to a literal string value for Solr queries, such as translating a
// Calcite timestamp value into an ISO-8601 formatted timestamp that Solr likes
private String toSolrLiteral(RexLiteral literal) {
@@ -339,15 +346,27 @@ class SolrFilter extends Filter implements SolrRel {
if (a != null) {
return a;
}
- final Pair<String, RexLiteral> b = translateBinary2(right, left);
- if (b != null) {
- return b;
+
+ // we can swap these if doing an equals / not equals
+ if (call.op.kind == SqlKind.EQUALS || call.op.kind == SqlKind.NOT_EQUALS) {
+ final Pair<String, RexLiteral> b = translateBinary2(right, left);
+ if (b != null) {
+ return b;
+ }
}
if (left.getKind() == SqlKind.CAST && right.getKind() == SqlKind.CAST) {
return translateBinary2(((RexCall)left).operands.get(0), ((RexCall)right).operands.get(0));
}
+ // for WHERE clause like: pdatex >= '2021-07-13T15:12:10.037Z'
+ if (left.getKind() == SqlKind.INPUT_REF && right.getKind() == SqlKind.CAST) {
+ final RexCall cast = ((RexCall)right);
+ if (cast.operands.size() == 1 && cast.operands.get(0).getKind() == SqlKind.LITERAL) {
+ return translateBinary2(left, cast.operands.get(0));
+ }
+ }
+
throw new AssertionError("cannot translate call " + call);
}
@@ -493,10 +512,14 @@ class SolrFilter extends Filter implements SolrRel {
}
return a;
}
- final Pair<String, RexLiteral> b = translateBinary2(right, left);
- if (b != null) {
- return b;
+
+ if (call.op.kind == SqlKind.EQUALS || call.op.kind == SqlKind.NOT_EQUALS) {
+ final Pair<String, RexLiteral> b = translateBinary2(right, left);
+ if (b != null) {
+ return b;
+ }
}
+
throw new AssertionError("cannot translate call " + call);
}
diff --git a/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java b/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java
index a1a2882998a..adb7332f62a 100644
--- a/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java
+++ b/solr/core/src/test/org/apache/solr/handler/TestSQLHandler.java
@@ -338,10 +338,10 @@ public class TestSQLHandler extends SolrCloudTestCase {
.add("id", "2", "text_t", "XXXX XXXX", "str_s", "b", "field_i", "8")
.add("id", "3", "text_t", "XXXX XXXX", "str_s", "a", "field_i", "20")
.add("id", "4", "text_t", "XXXX XXXX", "str_s", "b", "field_i", "11")
- .add("id", "5", "text_t", "XXXX XXXX", "str_s", "c", "field_i", "30")
- .add("id", "6", "text_t", "XXXX XXXX", "str_s", "c", "field_i", "40")
- .add("id", "7", "text_t", "XXXX XXXX", "str_s", "c", "field_i", "50")
- .add("id", "8", "text_t", "XXXX XXXX", "str_s", "c", "field_i", "60")
+ .add("id", "5", "text_t", "XXXX XXXX", "str_s", "c", "field_i", "30", "specialchars_s", "witha|pipe")
+ .add("id", "6", "text_t", "XXXX XXXX", "str_s", "c", "field_i", "40", "specialchars_s", "witha\\slash")
+ .add("id", "7", "text_t", "XXXX XXXX", "str_s", "c", "field_i", "50", "specialchars_s", "witha!bang")
+ .add("id", "8", "text_t", "XXXX XXXX", "str_s", "c", "field_i", "60", "specialchars_s", "witha\"quote")
.commit(cluster.getSolrClient(), COLLECTIONORALIAS);
String baseUrl = cluster.getJettySolrRunners().get(0).getBaseUrl().toString() + "/" + COLLECTIONORALIAS;
@@ -452,6 +452,14 @@ public class TestSQLHandler extends SolrCloudTestCase {
tuple = tuples.get(1);
assertEquals("8", tuple.get("id"));
+ expectResults("SELECT id FROM $ALIAS WHERE str_s = 'a'", 2);
+ expectResults("SELECT id FROM $ALIAS WHERE 'a' = str_s", 2);
+ expectResults("SELECT id FROM $ALIAS WHERE str_s <> 'c'", 4);
+ expectResults("SELECT id FROM $ALIAS WHERE 'c' <> str_s", 4);
+ expectResults("SELECT id FROM $ALIAS WHERE specialchars_s = 'witha\"quote'", 1);
+ expectResults("SELECT id FROM $ALIAS WHERE specialchars_s = 'witha|pipe'", 1);
+ expectResults("SELECT id FROM $ALIAS WHERE specialchars_s LIKE 'with%'", 4);
+ expectResults("SELECT id FROM $ALIAS WHERE specialchars_s LIKE 'witha|%'", 1);
}
@Test
@@ -1641,6 +1649,17 @@ public class TestSQLHandler extends SolrCloudTestCase {
assert (tuple.EOF);
assert (tuple.EXCEPTION);
assert (tuple.getException().contains("No match found for function signature blah"));
+
+ // verify exception message formatting with wildcard query
+ sParams = mapParams(CommonParams.QT, "/sql", "aggregationMode", "map_reduce",
+ "stmt",
+ "select str_s from collection1 where not_a_field LIKE 'foo%'");
+
+ solrStream = new SolrStream(baseUrl, sParams);
+ tuple = getTuple(new ExceptionStream(solrStream));
+ assert (tuple.EOF);
+ assert (tuple.EXCEPTION);
+ assert (tuple.getException().contains("Column 'not_a_field' not found in any table"));
}
@Test
@@ -2058,6 +2077,24 @@ public class TestSQLHandler extends SolrCloudTestCase {
expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex BETWEEN '2021-06-03' AND '2021-06-05'", 4);
}
+ @Test
+ public void testISO8601TimestampFiltering() throws Exception {
+ new UpdateRequest()
+ .add("id", "1", "pdatex", "2021-07-13T15:12:09.037Z")
+ .add("id", "2", "pdatex", "2021-07-13T15:12:10.037Z")
+ .add("id", "3", "pdatex", "2021-07-13T15:12:11.037Z")
+ .commit(cluster.getSolrClient(), COLLECTIONORALIAS);
+
+ expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex >= CAST('2021-07-13 15:12:10.037' as TIMESTAMP)", 2);
+ expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex >= '2021-07-13T15:12:10.037Z'", 2);
+ expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex < '2021-07-13T15:12:10.037Z'", 1);
+ expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex = '2021-07-13T15:12:10.037Z'", 1);
+ expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex <> '2021-07-13T15:12:10.037Z'", 2);
+ expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex BETWEEN '2021-07-13T15:12:09.037Z' AND '2021-07-13T15:12:10.037Z' ORDER BY pdatex ASC", 2);
+ expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex >= '2021-07-13T15:12:10.037Z'", 2);
+ expectResults("SELECT id, pdatex FROM $ALIAS WHERE pdatex >= '2021-07-13T15:12:10.037Z' ORDER BY pdatex ASC LIMIT 10", 2);
+ }
+
@Test
public void testAggsOnCustomFieldType() throws Exception {
new UpdateRequest()
diff --git a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JDBCStream.java b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JDBCStream.java
index 778f60cfc1a..8045e34ddc9 100644
--- a/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JDBCStream.java
+++ b/solr/solrj/src/java/org/apache/solr/client/solrj/io/stream/JDBCStream.java
@@ -283,8 +283,7 @@ public class JDBCStream extends TupleStream implements Expressible {
resultSet = statement.executeQuery(sqlQuery);
resultSet.setFetchSize(fetchSize);
} catch (SQLException e) {
- throw new IOException(String.format(Locale.ROOT, "Failed to execute sqlQuery '%s' against JDBC connection '%s'.\n"
- + e.getMessage(), sqlQuery, connectionUrl), e);
+ throw new IOException(String.format(Locale.ROOT, "Failed to execute sqlQuery '%s' against JDBC connection '%s'.%nCaused by: %s", sqlQuery, connectionUrl, e.getMessage()), e);
}
try{