💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Fix issue with select distinct, closes #379
davidmezzetti1f99694f76af
diff --git a/src/python/txtai/database/sql/expression.py b/src/python/txtai/database/sql/expression.pyindex 86d54d12..88bd797c 100644--- a/src/python/txtai/database/sql/expression.py+++ b/src/python/txtai/database/sql/expression.py@@ -64,8 +64,8 @@ class Expression:transformed tokens"""- # Create clause index and token iterator- index, iterator = 0, enumerate(tokens)+ # Create clause index and token iterator. Iterator skips distinct tokens.+ index, iterator = 0, ((x, token) for x, token in enumerate(tokens) if not Token.isdistinct(token))for x, token in iterator:# Check if separator, increment clause indexif Token.isseparator(token):@@ -189,6 +189,11 @@ class Expression:if not any(Token.isoperator(t) for t in alias) and alias[0] in ("[", "(") and alias[-1] in ("]", ")"):alias = alias[1:-1]+ # Strip leading distinct keyword+ values = alias.split()+ if len(values) > 0 and Token.isdistinct(values[0]):+ alias = " ".join(values[1:])+# Resolve aliastoken = self.resolver(token, alias)diff --git a/src/python/txtai/database/sql/token.py b/src/python/txtai/database/sql/token.pyindex fbcb61f9..f073430c 100644--- a/src/python/txtai/database/sql/token.py+++ b/src/python/txtai/database/sql/token.py@@ -11,15 +11,18 @@ class Token:# Similar token replacementSIMILAR_TOKEN = "__SIMILAR__"+ # Default distinct token+ DISTINCT = ["distinct"]++ # Default alias token+ ALIAS = ["as"]+# Default list of comparison operatorsOPERATORS = ["=", "!=", "<>", ">", ">=", "<", "<=", "+", "-", "*", "/", "%", "||", "not", "between", "like", "is", "null"]# Default list of logic separatorsLOGIC_SEPARATORS = ["and", "or"]- # Default alias token- ALIAS = ["as"]-# Default list of sort order operatorsSORT_ORDER = ["asc", "desc"]@@ -58,8 +61,15 @@ class Token:prior = Token.get(tokens, x - 1)token = tokens[x]- # True if prior token is not a separator and not a grouping token and current token is either a column token or quoted token- return alias and x > 0 and not Token.isseparator(prior) and not Token.isgroupstart(prior) and (Token.iscolumn(token) or Token.isquoted(token))+ # True if prior token is not a separator, grouping token or distinct token and current token is either a column token or quoted token+ return (+ alias+ and x > 0+ and not Token.isseparator(prior)+ and not Token.isgroupstart(prior)+ and not Token.isdistinct(prior)+ and (Token.iscolumn(token) or Token.isquoted(token))+ )@staticmethoddef isattribute(tokens, x):@@ -129,6 +139,21 @@ class Token:# Compound expression is defined as: <column> <operator(s)> <column>return Token.isoperator(tokens[x]) and (Token.iscolumn(Token.get(tokens, x - 1)) or Token.iscolumn(Token.get(tokens, x + 1)))+ @staticmethod+ def isdistinct(token):+ """+ Checks if token is the distinct keyword.++ Args:+ token: token to test++ Returns:+ True if this token is a distinct keyword, False otherwise+ """++ # Token is the distinct keyword+ return token and token.lower() in Token.DISTINCT+@staticmethoddef isfunction(tokens, x):"""diff --git a/test/python/testdatabase/testsql.py b/test/python/testdatabase/testsql.pyindex 1e8aed1c..042b61d8 100644--- a/test/python/testdatabase/testsql.py+++ b/test/python/testdatabase/testsql.py@@ -89,6 +89,33 @@ class TestSQL(unittest.TestCase):self.assertSql("groupby", "select * from txtai group by [a]", 'json_extract(data, "$.a")')self.assertSql("orderby", "select * from txtai where order by [a]", 'json_extract(data, "$.a")')+ def testDistinct(self):+ """+ Test distinct expressions+ """++ # Attributes+ self.assertSql("select", "select distinct id from txtai", "distinct s.id")+ self.assertSql("select", "select distinct id as myid from txtai", "distinct s.id as myid")+ self.assertSql("select", "select distinct a from txtai", 'distinct json_extract(data, "$.a") as "a"')+ self.assertSql("select", "select distinct a.b from txtai", 'distinct json_extract(data, "$.a.b") as "a.b"')++ # Bracket expression+ self.assertSql("select", "select distinct [ab cd] from txtai", 'distinct json_extract(data, "$.ab cd") as "distinct[ab cd]"')++ # Function expression+ self.assertSql("select", "select distinct(id) from txtai", 'distinct(s.id) as "distinct(id)"')+ self.assertSql("select", "select count(distinct id) from txtai", 'count(distinct s.id) as "count(distinct id)"')+ self.assertSql("select", "select count(distinct a) from txtai", 'count(distinct json_extract(data, "$.a")) as "count(distinct a)"')+ self.assertSql("select", "select count(distinct avg(id)) from txtai", 'count(distinct avg(s.id)) as "count(distinct avg(id))"')+ self.assertSql(+ "select", "select count(distinct avg(a)) from txtai", 'count(distinct avg(json_extract(data, "$.a"))) as "count(distinct avg(a))"'+ )++ # Compound expression+ self.assertSql("select", "select distinct a/1 from txtai", 'distinct json_extract(data, "$.a") / 1 as "a / 1"')+ self.assertSql("select", "select distinct(a/1) from txtai", 'distinct(json_extract(data, "$.a") / 1) as "distinct(a / 1)"')+def testGroupby(self):"""Test group by clauses