Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Fix issue with select distinct, closes #379

davidmezzetti1f99694f76af
diff --git a/src/python/txtai/database/sql/expression.py b/src/python/txtai/database/sql/expression.py
index 86d54d12..88bd797c 100644
--- a/src/python/txtai/database/sql/expression.py
+++ b/src/python/txtai/database/sql/expression.py
@@ -64,8 +64,8 @@ class Expression:
transformed tokens
"""
- # Create clause index and token iterator
- index, iterator = 0, enumerate(tokens)
+ # Create clause index and token iterator. Iterator skips distinct tokens.
+ index, iterator = 0, ((x, token) for x, token in enumerate(tokens) if not Token.isdistinct(token))
for x, token in iterator:
# Check if separator, increment clause index
if Token.isseparator(token):
@@ -189,6 +189,11 @@ class Expression:
if not any(Token.isoperator(t) for t in alias) and alias[0] in ("[", "(") and alias[-1] in ("]", ")"):
alias = alias[1:-1]
+ # Strip leading distinct keyword
+ values = alias.split()
+ if len(values) > 0 and Token.isdistinct(values[0]):
+ alias = " ".join(values[1:])
+
# Resolve alias
token = self.resolver(token, alias)
diff --git a/src/python/txtai/database/sql/token.py b/src/python/txtai/database/sql/token.py
index fbcb61f9..f073430c 100644
--- a/src/python/txtai/database/sql/token.py
+++ b/src/python/txtai/database/sql/token.py
@@ -11,15 +11,18 @@ class Token:
# Similar token replacement
SIMILAR_TOKEN = "__SIMILAR__"
+ # Default distinct token
+ DISTINCT = ["distinct"]
+
+ # Default alias token
+ ALIAS = ["as"]
+
# Default list of comparison operators
OPERATORS = ["=", "!=", "<>", ">", ">=", "<", "<=", "+", "-", "*", "/", "%", "||", "not", "between", "like", "is", "null"]
# Default list of logic separators
LOGIC_SEPARATORS = ["and", "or"]
- # Default alias token
- ALIAS = ["as"]
-
# Default list of sort order operators
SORT_ORDER = ["asc", "desc"]
@@ -58,8 +61,15 @@ class Token:
prior = Token.get(tokens, x - 1)
token = tokens[x]
- # True if prior token is not a separator and not a grouping token and current token is either a column token or quoted token
- return alias and x > 0 and not Token.isseparator(prior) and not Token.isgroupstart(prior) and (Token.iscolumn(token) or Token.isquoted(token))
+ # True if prior token is not a separator, grouping token or distinct token and current token is either a column token or quoted token
+ return (
+ alias
+ and x > 0
+ and not Token.isseparator(prior)
+ and not Token.isgroupstart(prior)
+ and not Token.isdistinct(prior)
+ and (Token.iscolumn(token) or Token.isquoted(token))
+ )
@staticmethod
def isattribute(tokens, x):
@@ -129,6 +139,21 @@ class Token:
# Compound expression is defined as: <column> <operator(s)> <column>
return Token.isoperator(tokens[x]) and (Token.iscolumn(Token.get(tokens, x - 1)) or Token.iscolumn(Token.get(tokens, x + 1)))
+ @staticmethod
+ def isdistinct(token):
+ """
+ Checks if token is the distinct keyword.
+
+ Args:
+ token: token to test
+
+ Returns:
+ True if this token is a distinct keyword, False otherwise
+ """
+
+ # Token is the distinct keyword
+ return token and token.lower() in Token.DISTINCT
+
@staticmethod
def isfunction(tokens, x):
"""
diff --git a/test/python/testdatabase/testsql.py b/test/python/testdatabase/testsql.py
index 1e8aed1c..042b61d8 100644
--- a/test/python/testdatabase/testsql.py
+++ b/test/python/testdatabase/testsql.py
@@ -89,6 +89,33 @@ class TestSQL(unittest.TestCase):
self.assertSql("groupby", "select * from txtai group by [a]", 'json_extract(data, "$.a")')
self.assertSql("orderby", "select * from txtai where order by [a]", 'json_extract(data, "$.a")')
+ def testDistinct(self):
+ """
+ Test distinct expressions
+ """
+
+ # Attributes
+ self.assertSql("select", "select distinct id from txtai", "distinct s.id")
+ self.assertSql("select", "select distinct id as myid from txtai", "distinct s.id as myid")
+ self.assertSql("select", "select distinct a from txtai", 'distinct json_extract(data, "$.a") as "a"')
+ self.assertSql("select", "select distinct a.b from txtai", 'distinct json_extract(data, "$.a.b") as "a.b"')
+
+ # Bracket expression
+ self.assertSql("select", "select distinct [ab cd] from txtai", 'distinct json_extract(data, "$.ab cd") as "distinct[ab cd]"')
+
+ # Function expression
+ self.assertSql("select", "select distinct(id) from txtai", 'distinct(s.id) as "distinct(id)"')
+ self.assertSql("select", "select count(distinct id) from txtai", 'count(distinct s.id) as "count(distinct id)"')
+ self.assertSql("select", "select count(distinct a) from txtai", 'count(distinct json_extract(data, "$.a")) as "count(distinct a)"')
+ self.assertSql("select", "select count(distinct avg(id)) from txtai", 'count(distinct avg(s.id)) as "count(distinct avg(id))"')
+ self.assertSql(
+ "select", "select count(distinct avg(a)) from txtai", 'count(distinct avg(json_extract(data, "$.a"))) as "count(distinct avg(a))"'
+ )
+
+ # Compound expression
+ self.assertSql("select", "select distinct a/1 from txtai", 'distinct json_extract(data, "$.a") / 1 as "a / 1"')
+ self.assertSql("select", "select distinct(a/1) from txtai", 'distinct(json_extract(data, "$.a") / 1) as "distinct(a / 1)"')
+
def testGroupby(self):
"""
Test group by clauses