💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add support for user defined column aliases and identifier/expression bracket clauses, closes #186 and closes #187
davidmezzetti5ff161f31f9d
diff --git a/src/python/txtai/database/base.py b/src/python/txtai/database/base.pyindex c0ac6a87..be730875 100644--- a/src/python/txtai/database/base.py+++ b/src/python/txtai/database/base.py@@ -5,7 +5,7 @@ Database moduleimport loggingfrom .encoder import EncoderFactory-from .sql import Expression, SQL, SQLException+from .sql import SQL, SQLException, Token# Logging configurationlogger = logging.getLogger(__name__)@@ -143,7 +143,7 @@ class Database:if "select" in query and similarity:for x in range(len(similarity)):- token = f"{Expression.SIMILAR_TOKEN}{x}"+ token = f"{Token.SIMILAR_TOKEN}{x}"if where and token in where:where = where.replace(token, self.embed(similarity, x))elif similarity:@@ -169,14 +169,14 @@ class Database:return self.sql(query)- def resolve(self, name, alias=False, compound=False):+ def resolve(self, name, alias=None):"""- Resolves a query column name with the database column name.+ Resolves a query column name with the database column name. This method also builds alias expressions+ if alias is set.Args:name: query column name- alias: True if an alias clause should be added, defaults to False- compound: True if this column is part of a compound expression, defaults to False+ alias: alias name, defaults to NoneReturns:database column namediff --git a/src/python/txtai/database/sql/__init__.py b/src/python/txtai/database/sql/__init__.pyindex 862f8931..869716e0 100644--- a/src/python/txtai/database/sql/__init__.py+++ b/src/python/txtai/database/sql/__init__.py@@ -5,3 +5,4 @@ SQL importsfrom .aggregate import Aggregatefrom .base import SQL, SQLExceptionfrom .expression import Expression+from .token import Tokendiff --git a/src/python/txtai/database/sql/aggregate.py b/src/python/txtai/database/sql/aggregate.pyindex dfb65aba..5e73db41 100644--- a/src/python/txtai/database/sql/aggregate.py+++ b/src/python/txtai/database/sql/aggregate.py@@ -32,27 +32,18 @@ class Aggregate(SQL):# Parse queryquery = super().__call__(query)- # Detect if this is a SQL query+ # Check if this is a SQL queryif "select" in query:# Get list of unique and aggregate columns. If no aggregate columns or order by found, skip- columns = query["select"]+ columns = list(results[0].keys())aggcolumns = self.aggcolumns(columns)if aggcolumns or query["orderby"]:- # Query formatted columns- qcolumns = list(results[0].keys())-- # Combine internal column names with initial result values for consistent processing- results = [dict(zip(columns, result.values())) for result in results]-# Merge aggregate columnsif aggcolumns:results = self.aggregate(query, results, columns, aggcolumns)- # Sort results- results = self.orderby(query, results) if query["orderby"] else self.defaultsort(results)-- # Combine query formatted columns with result values for final results- return [dict(zip(qcolumns, result.values())) for result in results]+ # Sort results and return+ return self.orderby(query, results) if query["orderby"] else self.defaultsort(results)# Otherwise, run default sortreturn self.defaultsort(results)@@ -128,7 +119,7 @@ class Aggregate(SQL):Args:query: input queryresults: query results- columns: list of select colum+ columns: list of select columnsReturns:results grouped using group by clausediff --git a/src/python/txtai/database/sql/base.py b/src/python/txtai/database/sql/base.pyindex ac8dd518..c85529c5 100644--- a/src/python/txtai/database/sql/base.py+++ b/src/python/txtai/database/sql/base.py@@ -48,17 +48,17 @@ class SQL:# Tokenize querytokens, positions = self.tokenize(query)- # Similar queries- similar = []+ # Alias clauses and similar queries+ aliases, similar = {}, []# Parse SQL clausesclauses = {- "select": self.parse(tokens, positions, "select", alias=True),- "where": self.parse(tokens, positions, "where", similar=similar),- "groupby": self.parse(tokens, positions, "group", offset=2),- "having": self.parse(tokens, positions, "having"),- "orderby": self.parse(tokens, positions, "order", offset=2),- "limit": self.parse(tokens, positions, "limit"),+ "select": self.parse(tokens, positions, "select", alias=True, aliases=aliases),+ "where": self.parse(tokens, positions, "where", aliases=aliases, similar=similar),+ "groupby": self.parse(tokens, positions, "group", offset=2, aliases=aliases),+ "having": self.parse(tokens, positions, "having", aliases=aliases),+ "orderby": self.parse(tokens, positions, "order", offset=2, aliases=aliases),+ "limit": self.parse(tokens, positions, "limit", aliases=aliases),}# Add parsed similar queries, if any@@ -69,20 +69,19 @@ class SQL:return clauses if clauses else {"similar": [[query]]}# pylint: disable=W0613- def defaultresolve(self, name, alias=False, compound=False):+ def defaultresolve(self, name, alias=None):"""Default resolve function. Performs no processing, only returns name.Args:name: query column name- alias: True if an alias clause should be added, defaults to False- compound: True if this column is part of a compound expression, defaults to False+ alias: alias name, defaults to NoneReturns:- name if alias is False, otherwise empty string+ name"""- return name if not alias else ""+ return namedef issql(self, query):"""@@ -121,7 +120,7 @@ class SQL:# Identify sql clause token positionspositions = {}- # Get position of clause keywords. For multi-term clauses, validate next token(s) match as well+ # Get position of clause keywords. For multi-term clauses, validate next token matches as wellfor x, token in enumerate(tokens):t = token.lower()if t not in positions and t in SQL.CLAUSES and (t not in ["group", "order"] or (x + 1 < len(tokens) and tokens[x + 1] == "by")):@@ -129,7 +128,7 @@ class SQL:return (tokens, positions)- def parse(self, tokens, positions, name, offset=1, alias=False, similar=None):+ def parse(self, tokens, positions, name, offset=1, alias=False, aliases=None, similar=None):"""Runs query column name to database column name mappings for clauses. This method will alsoparse SIMILAR() function calls, extract parameters for those calls and leave a placeholder@@ -141,6 +140,7 @@ class SQL:name: current query clause nameoffset: how many tokens are in the clause namealias: True if terms in the clause should be aliased (i.e. column as alias)+ aliases: dict of generated aliases, if present these tokens should NOT be resolvedsimilar: list where parsed similar clauses should be storedReturns:@@ -157,7 +157,7 @@ class SQL:clause = tokens[positions[name] + offset : end]# Parse and resolve parameters- clause = self.expression(clause, alias, similar)+ clause = self.expression(clause, alias, aliases, similar)return clausediff --git a/src/python/txtai/database/sql/expression.py b/src/python/txtai/database/sql/expression.pyindex 3bf9491d..86d54d12 100644--- a/src/python/txtai/database/sql/expression.py+++ b/src/python/txtai/database/sql/expression.py@@ -2,24 +2,14 @@Expression module"""+from .token import Token+class Expression:"""Parses expression statements and runs a set of substitution/formatting rules."""- # Similar token replacement- SIMILAR_TOKEN = "__SIMILAR__"-- # Default list of comparison operators- OPERATORS = ["=", "!=", "<>", ">", ">=", "<", "<=", "+", "-", "*", "/", "%", "not", "between", "like", "is", "null"]-- # Default list of logic separators- LOGIC_SEPARATORS = ["and", "or"]-- # Default list of sort order operators- SORT_ORDER = ["asc", "desc"]-def __init__(self, resolver, tolist):"""Creates a new expression parser.@@ -32,60 +22,84 @@ class Expression:self.resolver = resolverself.tolist = tolist- def __call__(self, tokens, alias=False, similar=None):+ def __call__(self, tokens, alias=False, aliases=None, similar=None):"""- Parses a list of tokens, replaces query column names with database column names and- adds similar query placeholders. Returns rewritten clause.+ Parses and formats a list of tokens as follows:+ - Replaces query column names with database column names+ - Adds similar query placeholders and extracts similar function parameters+ - Rewrites expression and returnsArgs:tokens: input expression- alias: if True, column aliases should be generated+ alias: if True, column aliases should be generated and added to aliases dict+ aliases: dict of generated aliases, if present these tokens should NOT be resolvedsimilar: list of similar queries, if present new similar queries are appended to this listReturns:- rewritten clause as a string+ rewritten clause"""- # Resolve column name tokens- tokens = self.resolve(list(tokens), alias, similar)+ # Processes token expressions and applies a set of transformation rules+ transformed = self.process(list(tokens), alias, aliases, similar)++ # Re-write alias expression and return+ if alias and not self.tolist:+ return self.buildalias(transformed, tokens, aliases)- # Re-write query and return- return self.buildlist(tokens) if self.tolist is True else self.buildtext(tokens)+ # Re-write input expression and return+ return self.buildlist(transformed) if self.tolist is True else self.buildtext(transformed)- def resolve(self, tokens, alias, similar):+ def process(self, tokens, alias, aliases, similar):"""- Resolves query column names with database column names.+ Replaces query column names with database column names, adds similar query placeholders and+ extracts similar function parameters.Args:tokens: input expression- alias: if True, column aliases should be generated+ alias: if True, column aliases should be generated and added to aliases dict+ aliases: dict of generated aliases, if present these tokens should NOT be resolvedsimilar: list of similar queries, if present new similar queries are appended to this listReturns:- resolved tokens+ transformed tokens"""- iterator = enumerate(tokens)+ # Create clause index and token iterator+ index, iterator = 0, enumerate(tokens)for x, token in iterator:+ # Check if separator, increment clause index+ if Token.isseparator(token):+ index += 1++ # Check if token is a square bracket+ elif Token.isbracket(token):+ # Resolve bracket expression+ self.bracket(iterator, tokens, x)+# Check if token is a similar function- if self.issimilar(tokens, x, similar):+ elif Token.issimilar(tokens, x, similar):# Resolve similar expressionself.similar(iterator, tokens, x, similar)# Check if token is a function- elif self.isfunction(tokens, x):+ elif Token.isfunction(tokens, x):# Resolve function expression- self.function(iterator, tokens, token)+ self.function(iterator, tokens, token, aliases)- # Check for attribute column not part of a compound expression- elif self.iscolumn(token) and not self.isoperator(self.get(tokens, x + 1)):+ # Check for alias expression+ elif Token.isalias(tokens, x, alias):+ # Process alias expression+ self.alias(iterator, tokens, x, aliases, index)++ # Check for attribute expression+ elif Token.isattribute(tokens, x):# Resolve attribute expression- self.attribute(tokens, x, alias)+ self.attribute(tokens, x, aliases)- # Check for compound expressions. Need to resolve left and/or right hand side- elif self.isoperator(token) and (self.iscolumn(self.get(tokens, x - 1)) or self.iscolumn(self.get(tokens, x + 1))):+ # Check for compound expression+ elif Token.iscompound(tokens, x):# Resolve compound expression- self.compound(iterator, tokens, x, alias)+ self.compound(iterator, tokens, x, aliases)# Remove replaced tokensreturn [token for token in tokens if token]@@ -105,7 +119,7 @@ class Expression:text = ""for token in tokens:# Write token with whitespace rules applied- text += self.wrapspace(text, token)+ text += Token.wrapspace(text, token)# Remove any leading/trailing whitespace and returnreturn text.strip()@@ -125,10 +139,12 @@ class Expression:parts, current, parens, brackets = [], [], 0, 0for token in tokens:+ # Create new partif token == "," and not parens and not brackets:- parts.append("".join(current))+ parts.append(self.buildtext(current))current = []else:+ # Accumulate tokensif token == "(":parens += 1elif token == ")":@@ -137,29 +153,89 @@ class Expression:brackets += 1elif token == "]":brackets -= 1- elif self.issortorder(token):+ elif Token.issortorder(token):token = f" {token}"current.append(token)+ # Add last partif current:- parts.append("".join(current))+ parts.append(self.buildtext(current))return parts- def issimilar(self, tokens, x, similar):+ def buildalias(self, transformed, tokens, aliases):"""- Checks if tokens[x] is a similar() function.+ Builds new alias text expression from transformed and input tokens.Args:- tokens: input tokens- x: current position- similar: list of similar clauses+ transformed: transformed tokens+ tokens: original input tokens+ aliases: dict of column aliasesReturns:- True if tokens[x] is a similar clause+ alias text expression+ """++ # Convert tokens to expressions+ transformed = self.buildlist(transformed)+ tokens = self.buildlist(tokens)++ expression = []+ for x, token in enumerate(transformed):+ if x not in aliases.values():+ alias = tokens[x]++ # Strip leading/trailing brackets from alias name that doesn't have operators+ if not any(Token.isoperator(t) for t in alias) and alias[0] in ("[", "(") and alias[-1] in ("]", ")"):+ alias = alias[1:-1]++ # Resolve alias+ token = self.resolver(token, alias)++ expression.append(token)++ # Build alias text expression+ return ", ".join(expression)++ def bracket(self, iterator, tokens, x):"""+ Consumes a [bracket] expression.++ Args:+ iterator: tokens iterator+ tokens: input tokens+ x: current position+ """++ # Function parameters+ params = []++ # Clear token from stream+ token = tokens[x]+ tokens[x] = None- return similar is not None and tokens[x].lower() == "similar" and self.get(tokens, x + 1) == "("+ # Bracket counter (current token is an open bracket)+ brackets = 1++ # Read until token is a end bracket+ while token and (token != "]" or brackets > 0):+ x, token = next(iterator, (None, None))++ # Increase/decrease bracket counter+ if token == "[":+ brackets += 1+ elif token == "]":+ brackets -= 1++ # Accumulate tokens+ if token != "]" or brackets > 0:+ params.append(token)++ # Clear token from stream+ tokens[x] = None++ # Set last token to resolved bracket expression+ tokens[x] = self.resolve(self.buildtext(params), None)def similar(self, iterator, tokens, x, similar):"""@@ -170,234 +246,135 @@ class Expression:iterator: tokens iteratortokens: input tokensx: current position- similar: list of similar clauses+ similar: list where similar function call parameters are stored"""- # Get function parameters+ # Function parametersparams = []- # Clear token from stream, it will be replaced by a placeholder for the function call+ # Clear token from streamtoken = tokens[x]tokens[x] = None+ # Read until token is a closing parenwhile token and token != ")":x, token = next(iterator, (None, None))if token and token not in ["(", ",", ")"]:- # Strip quotes+ # Strip quotes and accumulate tokensparams.append(token.replace("'", "").replace('"', ""))# Clear token from streamtokens[x] = None# Add placeholder for embedding similarity results- tokens[x] = f"{Expression.SIMILAR_TOKEN}{len(similar)}"+ tokens[x] = f"{Token.SIMILAR_TOKEN}{len(similar)}"# Save parameterssimilar.append(params)- def isfunction(self, tokens, x):+ def function(self, iterator, tokens, token, aliases):"""- Checks if tokens[x] is a function.+ Resolves column names within the function's parameters.Args:+ iterator: tokens iteratortokens: input tokens- x: current position-- Returns:- True if tokens[x] is a function, False otherwise+ token: current token+ aliases: dict of generated aliases, if present these tokens should NOT be resolved"""- # Check if token is a functios- return not self.isoperator(tokens[x]) and self.get(tokens, x + 1) == "("+ # Consume function parameters+ while token and token != ")":+ x, token = next(iterator, (None, None))+ if Token.isfunction(tokens, x):+ # Resolve function parameters that are functions+ self.function(iterator, tokens, token, aliases)+ elif Token.isattribute(tokens, x):+ # Resolve attributes+ self.attribute(tokens, x, aliases)+ elif Token.iscompound(tokens, x):+ # Resolve compound expressions+ self.compound(iterator, tokens, x, aliases)- def function(self, iterator, tokens, token):+ def alias(self, iterator, tokens, x, aliases, index):"""- Resolves column names within the function's parameters.+ Reads an alias clause and stores it in aliases.Args:iterator: tokens iteratortokens: input tokens- token: current token+ x: current position+ aliases: dict where aliases are stored - stores {alias: clause index}+ index: clause index, used to match aliases with columns"""- # Consume function parameters- while token and token != ")":+ token = tokens[x]++ # If this is an alias token, get next token+ if token in Token.ALIAS:x, token = next(iterator, (None, None))- if self.isfunction(tokens, x):- # Resolve function parameters that are functions- self.function(iterator, tokens, token)- elif self.iscolumn(token):- # Resolve function parameter- tokens[x] = self.resolver(tokens[x])- def attribute(self, tokens, x, alias):+ # Consume tokens until end of stream or a separator is found. Evaluate next token to prevent consuming here.+ while x + 1 < len(tokens) and not Token.isseparator(Token.get(tokens, x + 1)):+ x, token = next(iterator, (None, None))++ # Add normalized alias and clause index+ aliases[Token.normalize(token)] = index++ def attribute(self, tokens, x, aliases):"""Resolves an attribute column name.Args:tokens: input tokensx: current token position- alias: if True, column aliases should be generated+ aliases: dict of generated aliases, if present these tokens should NOT be resolved"""- # Alias name- name = tokens[x]-# Resolve attribute expression- tokens[x] = self.resolver(tokens[x])-- # Add alias- if alias:- alias = self.resolver(name, alias=True)- tokens[x] += alias if alias else ""+ tokens[x] = self.resolve(tokens[x], aliases)- def compound(self, iterator, tokens, x, alias):+ def compound(self, iterator, tokens, x, aliases):"""- Resolves a compound operator (left side <operator(s)> right side).+ Resolves column names in a compound expression (left side <operator(s)> right side).Args:iterator: tokens iteratortokens: input tokensx: current token position- alias: if True, column aliases should be generated+ aliases: dict of generated aliases, if present these tokens should NOT be resolved"""- # Alias name- name = None-# Resolve left side (left side already had function processing applied through standard loop)- if self.iscolumn(tokens[x - 1]):- name = tokens[x - 1]- tokens[x - 1] = self.resolver(tokens[x - 1])+ if Token.iscolumn(tokens[x - 1]):+ tokens[x - 1] = self.resolve(tokens[x - 1], aliases)# Consume operator(s), handle both single and compound operators, i.e. column NOT LIKE 1token = tokens[x]- while token and self.isoperator(token):+ while token and Token.isoperator(token):x, token = next(iterator, (None, None))# Resolve right side- if token and self.iscolumn(token):+ if token and Token.iscolumn(token):# Need to process functions since it hasn't went through the standard loop yet- if self.isfunction(tokens, x):- self.function(iterator, tokens, token)+ if Token.isfunction(tokens, x):+ self.function(iterator, tokens, token, aliases)else:- name = token- tokens[x] = self.resolver(token)-- # Add alias to last token- if name and alias:- alias = self.resolver(name, alias=True, compound=True)- tokens[x] += alias if alias else ""-- def iscolumn(self, token):- """- Checks if token is a column name.-- Args:- token: token to test-- Returns:- True if this token is a column name token, False otherwise- """-- return token and not self.isoperator(token) and not self.islogicseparator(token) and not self.isliteral(token) and not self.issortorder(token)-- def isoperator(self, token):- """- Checks if token is an operator token.-- Args:- token: token to test-- Returns:- True if this token is an operator, False otherwise- """-- return token and token.lower() in Expression.OPERATORS-- def islogicseparator(self, token):- """- Checks if token is a logic separator token.-- Args:- token: token to test-- Returns:- True if this token is a logic separator, False otherwise- """-- return token and token.lower() in Expression.LOGIC_SEPARATORS-- def issortorder(self, token):- """- Checks if token is a sort order token.-- Args:- token: token to test-- Returns:- True if this token is a sort order operator, False otherwise- """-- return token and token.lower() in Expression.SORT_ORDER-- def isliteral(self, token):- """- Checks if token is a literal. Literals are wrapped in quotes, parens, wildcards or numeric.-- Args:- token: token to test-- Returns:- True if this token is a literal, False otherwise- """-- return token and (token.startswith(("'", '"', ",", "(", ")", "[", "]", "*")) or token.replace(".", "", 1).isdigit())+ tokens[x] = self.resolve(token, aliases)- def wrapspace(self, text, token):+ def resolve(self, token, aliases):"""- Applies whitespace wrapping rules to token.+ Resolves this token's value if it is not an alias.Args:- text: current text buffer- token: token to add+ token: token to resolve+ aliases: dict of generated aliases, if present these tokens should NOT be resolvedReturns:- token with whitespace rules applied+ resolved token value"""- # Wildcards have no whitespace. Need special case since * is also multiply which does have whitespace.- if token in ["*"] and (not text or text.endswith((" ", "("))):+ if aliases and Token.normalize(token) in aliases:return token- # Operator whitespace- if self.isoperator(token) or self.islogicseparator(token) or token.lower() in ["in"]:- return f" {token} " if not text.endswith(" ") else f"{token} "-- # Comma whitespace- if token in [","]:- return f"{token} "-- # No whitespace if any of the following is True- if not text or text.endswith((" ", "(", "[")) or token in ["(", "[", ")", "]"]:- return token-- # Default is to add leading whitespace- return f" {token}"-- def get(self, tokens, x):- """- Gets token at position x. This method will validate position is valid within tokens.-- Args:- tokens: input tokens- x: position to retrieve-- Returns:- tokens[x] if x is a valid position, None otherwise- """-- if 0 <= x < len(tokens):- return tokens[x]-- return None+ return self.resolver(token)diff --git a/src/python/txtai/database/sql/token.py b/src/python/txtai/database/sql/token.pynew file mode 100644index 00000000..a8d5b5ed--- /dev/null+++ b/src/python/txtai/database/sql/token.py@@ -0,0 +1,317 @@+"""+Token module+"""+++class Token:+ """+ Methods to check for token type.+ """++ # Similar token replacement+ SIMILAR_TOKEN = "__SIMILAR__"++ # Default list of comparison operators+ OPERATORS = ["=", "!=", "<>", ">", ">=", "<", "<=", "+", "-", "*", "/", "%", "not", "between", "like", "is", "null"]++ # Default list of logic separators+ LOGIC_SEPARATORS = ["and", "or"]++ # Default alias token+ ALIAS = ["as"]++ # Default list of sort order operators+ SORT_ORDER = ["asc", "desc"]++ @staticmethod+ def get(tokens, x):+ """+ Gets token at position x. This method will validate position is valid within tokens.++ Args:+ tokens: input tokens+ x: position to retrieve++ Returns:+ tokens[x] if x is a valid position, None otherwise+ """++ if 0 <= x < len(tokens):+ return tokens[x]++ return None++ @staticmethod+ def isalias(tokens, x, alias):+ """+ Checks if tokens[x] is an alias keyword.++ Args:+ tokens: input tokens+ x: current position+ alias: if column alias processing is enabled++ Returns:+ True if tokens[x] is an alias token, False otherwise+ """++ prior = Token.get(tokens, x - 1)+ token = tokens[x]++ # True if prior token is not a separator and not a grouping token and current token is either a column token or quoted token+ return alias and x > 0 and not Token.isseparator(prior) and not Token.isgroupstart(prior) and (Token.iscolumn(token) or Token.isquoted(token))++ @staticmethod+ def isattribute(tokens, x):+ """+ Checks if tokens[x] is an attribute.++ Args:+ tokens: input tokens+ x: current position++ Returns:+ True if tokens[x] is an attribute, False otherwise+ """++ # True if token is a column and next token is not an operator+ return Token.iscolumn(tokens[x]) and not Token.isoperator(Token.get(tokens, x + 1))++ @staticmethod+ def isbracket(token):+ """+ Checks if token is an open bracket.++ Args:+ token: token to test++ Returns:+ True if token is an open bracket, False otherwise+ """++ # Token is a bracket+ return token == "["++ @staticmethod+ def iscolumn(token):+ """+ Checks if token is a column name.++ Args:+ token: token to test++ Returns:+ True if this token is a column name token, False otherwise+ """++ # Columns are not operators, logic separators, literals or sort order tokens+ return (+ token+ and not Token.isoperator(token)+ and not Token.islogicseparator(token)+ and not Token.isliteral(token)+ and not Token.issortorder(token)+ )++ @staticmethod+ def iscompound(tokens, x):+ """+ Checks if tokens[x] is a compound expression.++ Args:+ tokens: input tokens+ x: current position++ Returns:+ True if tokens[x] is a compound expression, False otherwise+ """++ # Compound expression is defined as: <column> <operator(s)> <column>+ return Token.isoperator(tokens[x]) and (Token.iscolumn(Token.get(tokens, x - 1)) or Token.iscolumn(Token.get(tokens, x + 1)))++ @staticmethod+ def isfunction(tokens, x):+ """+ Checks if tokens[x] is a function.++ Args:+ tokens: input tokens+ x: current position++ Returns:+ True if tokens[x] is a function, False otherwise+ """++ # True if a column token is followed by an open paren+ return Token.iscolumn(tokens[x]) and Token.get(tokens, x + 1) == "("++ @staticmethod+ def isgroupstart(token):+ """+ Checks if token is a group start token.++ Args:+ token: token to test++ Returns:+ True if token is a group start token, False otherwise+ """++ # Token is a paren+ return token == "("++ @staticmethod+ def isliteral(token):+ """+ Checks if token is a literal.++ Args:+ token: token to test++ Returns:+ True if this token is a literal, False otherwise+ """++ # Literals are wrapped in quotes, parens, wildcards or numeric.+ return token and (token.startswith(("'", '"', ",", "(", ")", "*")) or token.replace(".", "", 1).isdigit())++ @staticmethod+ def islogicseparator(token):+ """+ Checks if token is a logic separator token.++ Args:+ token: token to test++ Returns:+ True if this token is a logic separator, False otherwise+ """++ # Token is a logic separator+ return token and token.lower() in Token.LOGIC_SEPARATORS++ @staticmethod+ def isoperator(token):+ """+ Checks if token is an operator token.++ Args:+ token: token to test++ Returns:+ True if this token is an operator, False otherwise+ """++ # Token is an operator+ return token and token.lower() in Token.OPERATORS++ @staticmethod+ def isquoted(token):+ """+ Checks if token is quoted.++ Args:+ token: token to test++ Returns:+ True if this token is quoted, False otherwise+ """++ # Token is quoted+ return token.startswith(("'", '"')) and token.endswith(("'", '"'))++ @staticmethod+ def isseparator(token):+ """+ Checks if token is a separator token.++ Args:+ token to test++ Returns:+ True if this token is a separator, False otherwise+ """++ # Token is a comma+ return token == ","++ @staticmethod+ def issimilar(tokens, x, similar):+ """+ Checks if tokens[x] is a similar() function.++ Args:+ tokens: input tokens+ x: current position+ similar: list where similar function call parameters are stored, can be None in which case similar processing is skipped++ Returns:+ True if tokens[x] is a similar clause+ """++ # True if a "similar" token is followed by an open paren+ return similar is not None and tokens[x].lower() == "similar" and Token.get(tokens, x + 1) == "("++ @staticmethod+ def issortorder(token):+ """+ Checks if token is a sort order token.++ Args:+ token: token to test++ Returns:+ True if this token is a sort order operator, False otherwise+ """++ # Token is a sort order operator+ return token and token.lower() in Token.SORT_ORDER++ @staticmethod+ def normalize(token):+ """+ Applies a normalization algorithm to the input token as follows:+ - Strip single and double quotes+ - Make lowercase++ Args:+ token: input token++ Returns:+ normalized token+ """++ # Lowercase, replace and return+ return token.lower().replace("'", "").replace('"', "")++ @staticmethod+ def wrapspace(text, token):+ """+ Applies whitespace wrapping rules to token.++ Args:+ text: current text buffer+ token: token to add++ Returns:+ token with whitespace rules applied+ """++ # Wildcards have no whitespace. Need special case since * is also multiply which does have whitespace.+ if token in ["*"] and (not text or text.endswith((" ", "("))):+ return token++ # Operator whitespace+ if Token.isoperator(token) or Token.islogicseparator(token) or token.lower() in ["in"]:+ return f" {token} " if not text.endswith(" ") else f"{token} "++ # Comma whitespace+ if Token.isseparator(token):+ return f"{token} "++ # No whitespace if any of the following is True+ if not text or text.endswith((" ", "(", "[")) or token in ["(", "[", ")", "]"] or token.startswith("."):+ return token++ # Default is to add leading whitespace+ return f" {token}"diff --git a/src/python/txtai/database/sqlite.py b/src/python/txtai/database/sqlite.pyindex 4625a0a0..dda50b92 100644--- a/src/python/txtai/database/sqlite.py+++ b/src/python/txtai/database/sqlite.py@@ -224,19 +224,23 @@ class SQLite(Database):# Format and return resultsreturn self.cursor.fetchall()- def resolve(self, name, alias=False, compound=False):+ def resolve(self, name, alias=None):# Standard column namessections = ["indexid", "id", "tags", "entry"]noprefix = ["data", "object", "score", "text"]- # Alias JSON column expressions+ # Alias expressionif alias:- # Only apply aliases to non-standard columns or compound expressions- if name not in sections + noprefix or compound:- return f' as "{name}"'+ # Skip if name matches alias or alias is a standard column name+ if name == alias or alias in sections:+ return name- # No alias- return None+ # Build alias clause+ return f'{name} as "{alias}"'++ # Name is already resolved, skip+ if name.startswith("json_extract(data") or any(f"s.{s}" == name for s in sections):+ return name# Standard columns - need prefixesif name.lower() in sections:diff --git a/test/python/testapi/testcluster.py b/test/python/testapi/testcluster.pyindex e8592e2d..538b1dab 100644--- a/test/python/testapi/testcluster.py+++ b/test/python/testapi/testcluster.py@@ -38,8 +38,12 @@ class RequestHandler(BaseHTTPRequestHandler):if self.path == "/count":response = 26elif self.path.startswith("/search?query=select"):- if "group" in self.path:+ if "group+by+id" in self.path:+ response = [{"count(*)": 26}]+ elif "group+by+text" in self.path:response = [{"count(*)": 12, "text": "This is a test"}, {"count(*)": 14, "text": "And another test"}]+ elif "group+by+txt" in self.path:+ response = [{"count(*)": 12, "txt": "This is a test"}, {"count(*)": 14, "txt": "And another test"}]else:if self.server.server_port == 8002:response = [{"count(*)": 12, "min(indexid)": 0, "max(indexid)": 11, "avg(indexid)": 6.3}]@@ -185,15 +189,15 @@ class TestCluster(unittest.TestCase):Test cluster SQL statement"""- query = urllib.parse.quote("select count(*), min(indexid), max(indexid), avg(indexid) from txtai where text=[This is a test]")+ query = urllib.parse.quote("select count(*), min(indexid), max(indexid), avg(indexid) from txtai where text='This is a test'")self.assertEqual(self.client.get(f"search?query={query}").json(), [{"count(*)": 28, "min(indexid)": 0, "max(indexid)": 14, "avg(indexid)": 6.5}])- query = urllib.parse.quote("select count(*), text from txtai group by text order by count(*) desc")+ query = urllib.parse.quote("select count(*), text txt from txtai group by txt order by count(*) desc")self.assertEqual(self.client.get(f"search?query={query}").json(),- [{"count(*)": 28, "text": "And another test"}, {"count(*)": 24, "text": "This is a test"}],+ [{"count(*)": 28, "txt": "And another test"}, {"count(*)": 24, "txt": "This is a test"}],)query = urllib.parse.quote("select count(*), text from txtai group by text order by count(*) asc")diff --git a/test/python/testdatabase/testsql.py b/test/python/testdatabase/testsql.pyindex 94596b8f..e9c49515 100644--- a/test/python/testdatabase/testsql.py+++ b/test/python/testdatabase/testsql.py@@ -24,6 +24,35 @@ class TestSQL(unittest.TestCase):cls.sql = SQL(cls.db)+ def testAlias(self):+ """+ Test alias clauses+ """++ self.assertSql("select", "select a as a1 from txtai", 'json_extract(data, "$.a") as a1')+ self.assertSql("select", "select a 'a1' from txtai", "json_extract(data, \"$.a\") 'a1'")+ self.assertSql("select", 'select a "a1" from txtai', 'json_extract(data, "$.a") "a1"')+ self.assertSql("select", "select a a1 from txtai", 'json_extract(data, "$.a") a1')+ self.assertSql(+ "select",+ "select a, b as b1, c, d + 1 as 'd1' from txtai",+ 'json_extract(data, "$.a") as "a", json_extract(data, "$.b") as b1, '+ + 'json_extract(data, "$.c") as "c", json_extract(data, "$.d") + 1 as \'d1\'',+ )+ self.assertSql("select", "select id as myid from txtai", "s.id as myid")+ self.assertSql("select", "select length(a) t from txtai", 'length(json_extract(data, "$.a")) t')++ self.assertSql("where", "select id as myid from txtai where myid != 3 and a != 1", 'myid != 3 and json_extract(data, "$.a") != 1')+ self.assertSql("where", "select txt T from txtai where t LIKE '%abc%'", "t LIKE '%abc%'")+ self.assertSql("where", "select txt 'T' from txtai where t LIKE '%abc%'", "t LIKE '%abc%'")+ self.assertSql("where", "select txt \"T\" from txtai where t LIKE '%abc%'", "t LIKE '%abc%'")+ self.assertSql("where", "select txt as T from txtai where t LIKE '%abc%'", "t LIKE '%abc%'")+ self.assertSql("where", "select txt as 'T' from txtai where t LIKE '%abc%'", "t LIKE '%abc%'")+ self.assertSql("where", "select txt as \"T\" from txtai where t LIKE '%abc%'", "t LIKE '%abc%'")++ self.assertSql("groupby", "select id as myid, count(*) from txtai group by myid, a", 'myid, json_extract(data, "$.a")')+ self.assertSql("orderby", "select id as myid from txtai order by myid, a", 'myid, json_extract(data, "$.a")')+def testBadSQL(self):"""Test invalid SQL@@ -41,6 +70,25 @@ class TestSQL(unittest.TestCase):with self.assertRaises(SQLException):self.db.search("select a b c from txtai where id match id")+ def testBracket(self):+ """+ Test bracket expressions+ """++ self.assertSql("select", "select [a] from txtai", 'json_extract(data, "$.a") as "a"')+ self.assertSql("select", "select [a] ab from txtai", 'json_extract(data, "$.a") ab')+ self.assertSql("select", "select [abc] from txtai", 'json_extract(data, "$.abc") as "abc"')+ self.assertSql("select", "select [id], text, score from txtai", "s.id, text, score")+ self.assertSql("select", "select [ab cd], text, score from txtai", 'json_extract(data, "$.ab cd") as "ab cd", text, score')+ self.assertSql("select", "select [a[0]] from txtai", 'json_extract(data, "$.a[0]") as "a[0]"')+ self.assertSql("select", "select [a[0].ab] from txtai", 'json_extract(data, "$.a[0].ab") as "a[0].ab"')+ self.assertSql("select", "select [a[0].c[0]] from txtai", 'json_extract(data, "$.a[0].c[0]") as "a[0].c[0]"')++ self.assertSql("where", "select * from txtai where [a b] < 1 or a > 1", 'json_extract(data, "$.a b") < 1 or json_extract(data, "$.a") > 1')+ self.assertSql("where", "select [a[0].c[0]] a from txtai where a < 1", "a < 1")+ self.assertSql("groupby", "select * from txtai group by [a]", 'json_extract(data, "$.a")')+ self.assertSql("orderby", "select * from txtai where order by [a]", 'json_extract(data, "$.a")')+def testGroupby(self):"""Test group by clauses@@ -92,7 +140,7 @@ class TestSQL(unittest.TestCase):self.assertSql("select", "select id, indexid, tags from txtai", "s.id, s.indexid, s.tags")self.assertSql("select", "select id, indexid, flag from txtai", 's.id, s.indexid, json_extract(data, "$.flag") as "flag"')self.assertSql("select", "select id, indexid, a.b.c from txtai", 's.id, s.indexid, json_extract(data, "$.a.b.c") as "a.b.c"')- self.assertSql("select", "select 'id', [id], (id) from txtai", "'id', [s.id], (s.id)")+ self.assertSql("select", "select 'id', [id], (id) from txtai", "'id', s.id, (s.id)")self.assertSql("select", "select * from txtai", "*")def testSelectCompound(self):@@ -100,12 +148,41 @@ class TestSQL(unittest.TestCase):Test compound select clauses"""- self.assertSql("select", "select a + 1 from txtai", 'json_extract(data, "$.a") + 1 as "a"')- self.assertSql("select", "select 1 * a from txtai", '1 * json_extract(data, "$.a") as "a"')- self.assertSql("select", "select a/1 from txtai", 'json_extract(data, "$.a") / 1 as "a"')- self.assertSql("select", "select avg(a-b) from txtai", 'avg(json_extract(data, "$.a") - json_extract(data, "$.b"))')+ self.assertSql("select", "select a + 1 from txtai", 'json_extract(data, "$.a") + 1 as "a + 1"')+ self.assertSql("select", "select 1 * a from txtai", '1 * json_extract(data, "$.a") as "1 * a"')+ self.assertSql("select", "select a/1 from txtai", 'json_extract(data, "$.a") / 1 as "a / 1"')+ self.assertSql("select", "select avg(a-b) from txtai", 'avg(json_extract(data, "$.a") - json_extract(data, "$.b")) as "avg(a - b)"')self.assertSql("select", "select distinct(text) from txtai", "distinct(text)")- self.assertSql("select", "select id, score, (a/2)*3 from txtai", 's.id, score, (json_extract(data, "$.a") / 2) * 3')+ self.assertSql("select", "select id, score, (a/2)*3 from txtai", 's.id, score, (json_extract(data, "$.a") / 2) * 3 as "(a / 2) * 3"')+ self.assertSql("select", "select id, score, (a/2*3) from txtai", 's.id, score, (json_extract(data, "$.a") / 2 * 3) as "(a / 2 * 3)"')+ self.assertSql(+ "select",+ "select func(func2(indexid + 1), a) from txtai",+ 'func(func2(s.indexid + 1), json_extract(data, "$.a")) as "func(func2(indexid + 1), a)"',+ )+ self.assertSql("select", "select func(func2(indexid + 1), a) a from txtai", 'func(func2(s.indexid + 1), json_extract(data, "$.a")) a')++ def testSimilar(self):+ """+ Test similar functions+ """++ prefix = "select * from txtai "++ self.assertSql("where", prefix + "where similar('abc')", "__SIMILAR__0")+ self.assertSql("similar", prefix + "where similar('abc')", [["abc"]])++ self.assertSql("where", prefix + "where similar('abc') AND id = 1", "__SIMILAR__0 AND s.id = 1")+ self.assertSql("similar", prefix + "where similar('abc')", [["abc"]])++ self.assertSql("where", prefix + "where similar('abc') and similar('def')", "__SIMILAR__0 and __SIMILAR__1")+ self.assertSql("similar", prefix + "where similar('abc') and similar('def')", [["abc"], ["def"]])++ self.assertSql("where", prefix + "where similar('abc', 1000)", "__SIMILAR__0")+ self.assertSql("similar", prefix + "where similar('abc', 1000)", [["abc", "1000"]])++ self.assertSql("where", prefix + "where similar('abc', 1000) and similar('def', 10)", "__SIMILAR__0 and __SIMILAR__1")+ self.assertSql("similar", prefix + "where similar('abc', 1000) and similar('def', 10)", [["abc", "1000"], ["def", "10"]])def testWhereBasic(self):"""@@ -115,6 +192,7 @@ class TestSQL(unittest.TestCase):prefix = "select * from txtai "self.assertSql("where", prefix + "where a = b", 'json_extract(data, "$.a") = json_extract(data, "$.b")')+ self.assertSql("where", prefix + "where abc = def", 'json_extract(data, "$.abc") = json_extract(data, "$.def")')self.assertSql("where", prefix + "where a = b.value", 'json_extract(data, "$.a") = json_extract(data, "$.b.value")')self.assertSql("where", prefix + "where a = 1", 'json_extract(data, "$.a") = 1')self.assertSql("where", prefix + "WHERE 1 = a", '1 = json_extract(data, "$.a")')@@ -145,28 +223,6 @@ class TestSQL(unittest.TestCase):self.assertSql("where", prefix + "where f(id) = b(id)", "f(s.id) = b(s.id)")self.assertSql("where", prefix + "WHERE f(id)", "f(s.id)")- def testWhereSimilar(self):- """- Test similar functions- """-- prefix = "select * from txtai "-- self.assertSql("where", prefix + "where similar('abc')", "__SIMILAR__0")- self.assertSql("similar", prefix + "where similar('abc')", [["abc"]])-- self.assertSql("where", prefix + "where similar('abc') AND id = 1", "__SIMILAR__0 AND s.id = 1")- self.assertSql("similar", prefix + "where similar('abc')", [["abc"]])-- self.assertSql("where", prefix + "where similar('abc') and similar('def')", "__SIMILAR__0 and __SIMILAR__1")- self.assertSql("similar", prefix + "where similar('abc') and similar('def')", [["abc"], ["def"]])-- self.assertSql("where", prefix + "where similar('abc', 1000)", "__SIMILAR__0")- self.assertSql("similar", prefix + "where similar('abc', 1000)", [["abc", "1000"]])-- self.assertSql("where", prefix + "where similar('abc', 1000) and similar('def', 10)", "__SIMILAR__0 and __SIMILAR__1")- self.assertSql("similar", prefix + "where similar('abc', 1000) and similar('def', 10)", [["abc", "1000"], ["def", "10"]])-def assertSql(self, clause, query, expected):"""Helper method to assert a query clause is as expected.