Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Add support for user defined column aliases and identifier/expression bracket clauses, closes #186 and closes #187

davidmezzetti5ff161f31f9d
diff --git a/src/python/txtai/database/base.py b/src/python/txtai/database/base.py
index c0ac6a87..be730875 100644
--- a/src/python/txtai/database/base.py
+++ b/src/python/txtai/database/base.py
@@ -5,7 +5,7 @@ Database module
import logging
from .encoder import EncoderFactory
-from .sql import Expression, SQL, SQLException
+from .sql import SQL, SQLException, Token
# Logging configuration
logger = logging.getLogger(__name__)
@@ -143,7 +143,7 @@ class Database:
if "select" in query and similarity:
for x in range(len(similarity)):
- token = f"{Expression.SIMILAR_TOKEN}{x}"
+ token = f"{Token.SIMILAR_TOKEN}{x}"
if where and token in where:
where = where.replace(token, self.embed(similarity, x))
elif similarity:
@@ -169,14 +169,14 @@ class Database:
return self.sql(query)
- def resolve(self, name, alias=False, compound=False):
+ def resolve(self, name, alias=None):
"""
- Resolves a query column name with the database column name.
+ Resolves a query column name with the database column name. This method also builds alias expressions
+ if alias is set.
Args:
name: query column name
- alias: True if an alias clause should be added, defaults to False
- compound: True if this column is part of a compound expression, defaults to False
+ alias: alias name, defaults to None
Returns:
database column name
diff --git a/src/python/txtai/database/sql/__init__.py b/src/python/txtai/database/sql/__init__.py
index 862f8931..869716e0 100644
--- a/src/python/txtai/database/sql/__init__.py
+++ b/src/python/txtai/database/sql/__init__.py
@@ -5,3 +5,4 @@ SQL imports
from .aggregate import Aggregate
from .base import SQL, SQLException
from .expression import Expression
+from .token import Token
diff --git a/src/python/txtai/database/sql/aggregate.py b/src/python/txtai/database/sql/aggregate.py
index dfb65aba..5e73db41 100644
--- a/src/python/txtai/database/sql/aggregate.py
+++ b/src/python/txtai/database/sql/aggregate.py
@@ -32,27 +32,18 @@ class Aggregate(SQL):
# Parse query
query = super().__call__(query)
- # Detect if this is a SQL query
+ # Check if this is a SQL query
if "select" in query:
# Get list of unique and aggregate columns. If no aggregate columns or order by found, skip
- columns = query["select"]
+ columns = list(results[0].keys())
aggcolumns = self.aggcolumns(columns)
if aggcolumns or query["orderby"]:
- # Query formatted columns
- qcolumns = list(results[0].keys())
-
- # Combine internal column names with initial result values for consistent processing
- results = [dict(zip(columns, result.values())) for result in results]
-
# Merge aggregate columns
if aggcolumns:
results = self.aggregate(query, results, columns, aggcolumns)
- # Sort results
- results = self.orderby(query, results) if query["orderby"] else self.defaultsort(results)
-
- # Combine query formatted columns with result values for final results
- return [dict(zip(qcolumns, result.values())) for result in results]
+ # Sort results and return
+ return self.orderby(query, results) if query["orderby"] else self.defaultsort(results)
# Otherwise, run default sort
return self.defaultsort(results)
@@ -128,7 +119,7 @@ class Aggregate(SQL):
Args:
query: input query
results: query results
- columns: list of select colum
+ columns: list of select columns
Returns:
results grouped using group by clause
diff --git a/src/python/txtai/database/sql/base.py b/src/python/txtai/database/sql/base.py
index ac8dd518..c85529c5 100644
--- a/src/python/txtai/database/sql/base.py
+++ b/src/python/txtai/database/sql/base.py
@@ -48,17 +48,17 @@ class SQL:
# Tokenize query
tokens, positions = self.tokenize(query)
- # Similar queries
- similar = []
+ # Alias clauses and similar queries
+ aliases, similar = {}, []
# Parse SQL clauses
clauses = {
- "select": self.parse(tokens, positions, "select", alias=True),
- "where": self.parse(tokens, positions, "where", similar=similar),
- "groupby": self.parse(tokens, positions, "group", offset=2),
- "having": self.parse(tokens, positions, "having"),
- "orderby": self.parse(tokens, positions, "order", offset=2),
- "limit": self.parse(tokens, positions, "limit"),
+ "select": self.parse(tokens, positions, "select", alias=True, aliases=aliases),
+ "where": self.parse(tokens, positions, "where", aliases=aliases, similar=similar),
+ "groupby": self.parse(tokens, positions, "group", offset=2, aliases=aliases),
+ "having": self.parse(tokens, positions, "having", aliases=aliases),
+ "orderby": self.parse(tokens, positions, "order", offset=2, aliases=aliases),
+ "limit": self.parse(tokens, positions, "limit", aliases=aliases),
}
# Add parsed similar queries, if any
@@ -69,20 +69,19 @@ class SQL:
return clauses if clauses else {"similar": [[query]]}
# pylint: disable=W0613
- def defaultresolve(self, name, alias=False, compound=False):
+ def defaultresolve(self, name, alias=None):
"""
Default resolve function. Performs no processing, only returns name.
Args:
name: query column name
- alias: True if an alias clause should be added, defaults to False
- compound: True if this column is part of a compound expression, defaults to False
+ alias: alias name, defaults to None
Returns:
- name if alias is False, otherwise empty string
+ name
"""
- return name if not alias else ""
+ return name
def issql(self, query):
"""
@@ -121,7 +120,7 @@ class SQL:
# Identify sql clause token positions
positions = {}
- # Get position of clause keywords. For multi-term clauses, validate next token(s) match as well
+ # Get position of clause keywords. For multi-term clauses, validate next token matches as well
for x, token in enumerate(tokens):
t = token.lower()
if t not in positions and t in SQL.CLAUSES and (t not in ["group", "order"] or (x + 1 < len(tokens) and tokens[x + 1] == "by")):
@@ -129,7 +128,7 @@ class SQL:
return (tokens, positions)
- def parse(self, tokens, positions, name, offset=1, alias=False, similar=None):
+ def parse(self, tokens, positions, name, offset=1, alias=False, aliases=None, similar=None):
"""
Runs query column name to database column name mappings for clauses. This method will also
parse SIMILAR() function calls, extract parameters for those calls and leave a placeholder
@@ -141,6 +140,7 @@ class SQL:
name: current query clause name
offset: how many tokens are in the clause name
alias: True if terms in the clause should be aliased (i.e. column as alias)
+ aliases: dict of generated aliases, if present these tokens should NOT be resolved
similar: list where parsed similar clauses should be stored
Returns:
@@ -157,7 +157,7 @@ class SQL:
clause = tokens[positions[name] + offset : end]
# Parse and resolve parameters
- clause = self.expression(clause, alias, similar)
+ clause = self.expression(clause, alias, aliases, similar)
return clause
diff --git a/src/python/txtai/database/sql/expression.py b/src/python/txtai/database/sql/expression.py
index 3bf9491d..86d54d12 100644
--- a/src/python/txtai/database/sql/expression.py
+++ b/src/python/txtai/database/sql/expression.py
@@ -2,24 +2,14 @@
Expression module
"""
+from .token import Token
+
class Expression:
"""
Parses expression statements and runs a set of substitution/formatting rules.
"""
- # Similar token replacement
- SIMILAR_TOKEN = "__SIMILAR__"
-
- # Default list of comparison operators
- OPERATORS = ["=", "!=", "<>", ">", ">=", "<", "<=", "+", "-", "*", "/", "%", "not", "between", "like", "is", "null"]
-
- # Default list of logic separators
- LOGIC_SEPARATORS = ["and", "or"]
-
- # Default list of sort order operators
- SORT_ORDER = ["asc", "desc"]
-
def __init__(self, resolver, tolist):
"""
Creates a new expression parser.
@@ -32,60 +22,84 @@ class Expression:
self.resolver = resolver
self.tolist = tolist
- def __call__(self, tokens, alias=False, similar=None):
+ def __call__(self, tokens, alias=False, aliases=None, similar=None):
"""
- Parses a list of tokens, replaces query column names with database column names and
- adds similar query placeholders. Returns rewritten clause.
+ Parses and formats a list of tokens as follows:
+ - Replaces query column names with database column names
+ - Adds similar query placeholders and extracts similar function parameters
+ - Rewrites expression and returns
Args:
tokens: input expression
- alias: if True, column aliases should be generated
+ alias: if True, column aliases should be generated and added to aliases dict
+ aliases: dict of generated aliases, if present these tokens should NOT be resolved
similar: list of similar queries, if present new similar queries are appended to this list
Returns:
- rewritten clause as a string
+ rewritten clause
"""
- # Resolve column name tokens
- tokens = self.resolve(list(tokens), alias, similar)
+ # Processes token expressions and applies a set of transformation rules
+ transformed = self.process(list(tokens), alias, aliases, similar)
+
+ # Re-write alias expression and return
+ if alias and not self.tolist:
+ return self.buildalias(transformed, tokens, aliases)
- # Re-write query and return
- return self.buildlist(tokens) if self.tolist is True else self.buildtext(tokens)
+ # Re-write input expression and return
+ return self.buildlist(transformed) if self.tolist is True else self.buildtext(transformed)
- def resolve(self, tokens, alias, similar):
+ def process(self, tokens, alias, aliases, similar):
"""
- Resolves query column names with database column names.
+ Replaces query column names with database column names, adds similar query placeholders and
+ extracts similar function parameters.
Args:
tokens: input expression
- alias: if True, column aliases should be generated
+ alias: if True, column aliases should be generated and added to aliases dict
+ aliases: dict of generated aliases, if present these tokens should NOT be resolved
similar: list of similar queries, if present new similar queries are appended to this list
Returns:
- resolved tokens
+ transformed tokens
"""
- iterator = enumerate(tokens)
+ # Create clause index and token iterator
+ index, iterator = 0, enumerate(tokens)
for x, token in iterator:
+ # Check if separator, increment clause index
+ if Token.isseparator(token):
+ index += 1
+
+ # Check if token is a square bracket
+ elif Token.isbracket(token):
+ # Resolve bracket expression
+ self.bracket(iterator, tokens, x)
+
# Check if token is a similar function
- if self.issimilar(tokens, x, similar):
+ elif Token.issimilar(tokens, x, similar):
# Resolve similar expression
self.similar(iterator, tokens, x, similar)
# Check if token is a function
- elif self.isfunction(tokens, x):
+ elif Token.isfunction(tokens, x):
# Resolve function expression
- self.function(iterator, tokens, token)
+ self.function(iterator, tokens, token, aliases)
- # Check for attribute column not part of a compound expression
- elif self.iscolumn(token) and not self.isoperator(self.get(tokens, x + 1)):
+ # Check for alias expression
+ elif Token.isalias(tokens, x, alias):
+ # Process alias expression
+ self.alias(iterator, tokens, x, aliases, index)
+
+ # Check for attribute expression
+ elif Token.isattribute(tokens, x):
# Resolve attribute expression
- self.attribute(tokens, x, alias)
+ self.attribute(tokens, x, aliases)
- # Check for compound expressions. Need to resolve left and/or right hand side
- elif self.isoperator(token) and (self.iscolumn(self.get(tokens, x - 1)) or self.iscolumn(self.get(tokens, x + 1))):
+ # Check for compound expression
+ elif Token.iscompound(tokens, x):
# Resolve compound expression
- self.compound(iterator, tokens, x, alias)
+ self.compound(iterator, tokens, x, aliases)
# Remove replaced tokens
return [token for token in tokens if token]
@@ -105,7 +119,7 @@ class Expression:
text = ""
for token in tokens:
# Write token with whitespace rules applied
- text += self.wrapspace(text, token)
+ text += Token.wrapspace(text, token)
# Remove any leading/trailing whitespace and return
return text.strip()
@@ -125,10 +139,12 @@ class Expression:
parts, current, parens, brackets = [], [], 0, 0
for token in tokens:
+ # Create new part
if token == "," and not parens and not brackets:
- parts.append("".join(current))
+ parts.append(self.buildtext(current))
current = []
else:
+ # Accumulate tokens
if token == "(":
parens += 1
elif token == ")":
@@ -137,29 +153,89 @@ class Expression:
brackets += 1
elif token == "]":
brackets -= 1
- elif self.issortorder(token):
+ elif Token.issortorder(token):
token = f" {token}"
current.append(token)
+ # Add last part
if current:
- parts.append("".join(current))
+ parts.append(self.buildtext(current))
return parts
- def issimilar(self, tokens, x, similar):
+ def buildalias(self, transformed, tokens, aliases):
"""
- Checks if tokens[x] is a similar() function.
+ Builds new alias text expression from transformed and input tokens.
Args:
- tokens: input tokens
- x: current position
- similar: list of similar clauses
+ transformed: transformed tokens
+ tokens: original input tokens
+ aliases: dict of column aliases
Returns:
- True if tokens[x] is a similar clause
+ alias text expression
+ """
+
+ # Convert tokens to expressions
+ transformed = self.buildlist(transformed)
+ tokens = self.buildlist(tokens)
+
+ expression = []
+ for x, token in enumerate(transformed):
+ if x not in aliases.values():
+ alias = tokens[x]
+
+ # Strip leading/trailing brackets from alias name that doesn't have operators
+ if not any(Token.isoperator(t) for t in alias) and alias[0] in ("[", "(") and alias[-1] in ("]", ")"):
+ alias = alias[1:-1]
+
+ # Resolve alias
+ token = self.resolver(token, alias)
+
+ expression.append(token)
+
+ # Build alias text expression
+ return ", ".join(expression)
+
+ def bracket(self, iterator, tokens, x):
"""
+ Consumes a [bracket] expression.
+
+ Args:
+ iterator: tokens iterator
+ tokens: input tokens
+ x: current position
+ """
+
+ # Function parameters
+ params = []
+
+ # Clear token from stream
+ token = tokens[x]
+ tokens[x] = None
- return similar is not None and tokens[x].lower() == "similar" and self.get(tokens, x + 1) == "("
+ # Bracket counter (current token is an open bracket)
+ brackets = 1
+
+ # Read until token is a end bracket
+ while token and (token != "]" or brackets > 0):
+ x, token = next(iterator, (None, None))
+
+ # Increase/decrease bracket counter
+ if token == "[":
+ brackets += 1
+ elif token == "]":
+ brackets -= 1
+
+ # Accumulate tokens
+ if token != "]" or brackets > 0:
+ params.append(token)
+
+ # Clear token from stream
+ tokens[x] = None
+
+ # Set last token to resolved bracket expression
+ tokens[x] = self.resolve(self.buildtext(params), None)
def similar(self, iterator, tokens, x, similar):
"""
@@ -170,234 +246,135 @@ class Expression:
iterator: tokens iterator
tokens: input tokens
x: current position
- similar: list of similar clauses
+ similar: list where similar function call parameters are stored
"""
- # Get function parameters
+ # Function parameters
params = []
- # Clear token from stream, it will be replaced by a placeholder for the function call
+ # Clear token from stream
token = tokens[x]
tokens[x] = None
+ # Read until token is a closing paren
while token and token != ")":
x, token = next(iterator, (None, None))
if token and token not in ["(", ",", ")"]:
- # Strip quotes
+ # Strip quotes and accumulate tokens
params.append(token.replace("'", "").replace('"', ""))
# Clear token from stream
tokens[x] = None
# Add placeholder for embedding similarity results
- tokens[x] = f"{Expression.SIMILAR_TOKEN}{len(similar)}"
+ tokens[x] = f"{Token.SIMILAR_TOKEN}{len(similar)}"
# Save parameters
similar.append(params)
- def isfunction(self, tokens, x):
+ def function(self, iterator, tokens, token, aliases):
"""
- Checks if tokens[x] is a function.
+ Resolves column names within the function's parameters.
Args:
+ iterator: tokens iterator
tokens: input tokens
- x: current position
-
- Returns:
- True if tokens[x] is a function, False otherwise
+ token: current token
+ aliases: dict of generated aliases, if present these tokens should NOT be resolved
"""
- # Check if token is a functios
- return not self.isoperator(tokens[x]) and self.get(tokens, x + 1) == "("
+ # Consume function parameters
+ while token and token != ")":
+ x, token = next(iterator, (None, None))
+ if Token.isfunction(tokens, x):
+ # Resolve function parameters that are functions
+ self.function(iterator, tokens, token, aliases)
+ elif Token.isattribute(tokens, x):
+ # Resolve attributes
+ self.attribute(tokens, x, aliases)
+ elif Token.iscompound(tokens, x):
+ # Resolve compound expressions
+ self.compound(iterator, tokens, x, aliases)
- def function(self, iterator, tokens, token):
+ def alias(self, iterator, tokens, x, aliases, index):
"""
- Resolves column names within the function's parameters.
+ Reads an alias clause and stores it in aliases.
Args:
iterator: tokens iterator
tokens: input tokens
- token: current token
+ x: current position
+ aliases: dict where aliases are stored - stores {alias: clause index}
+ index: clause index, used to match aliases with columns
"""
- # Consume function parameters
- while token and token != ")":
+ token = tokens[x]
+
+ # If this is an alias token, get next token
+ if token in Token.ALIAS:
x, token = next(iterator, (None, None))
- if self.isfunction(tokens, x):
- # Resolve function parameters that are functions
- self.function(iterator, tokens, token)
- elif self.iscolumn(token):
- # Resolve function parameter
- tokens[x] = self.resolver(tokens[x])
- def attribute(self, tokens, x, alias):
+ # Consume tokens until end of stream or a separator is found. Evaluate next token to prevent consuming here.
+ while x + 1 < len(tokens) and not Token.isseparator(Token.get(tokens, x + 1)):
+ x, token = next(iterator, (None, None))
+
+ # Add normalized alias and clause index
+ aliases[Token.normalize(token)] = index
+
+ def attribute(self, tokens, x, aliases):
"""
Resolves an attribute column name.
Args:
tokens: input tokens
x: current token position
- alias: if True, column aliases should be generated
+ aliases: dict of generated aliases, if present these tokens should NOT be resolved
"""
- # Alias name
- name = tokens[x]
-
# Resolve attribute expression
- tokens[x] = self.resolver(tokens[x])
-
- # Add alias
- if alias:
- alias = self.resolver(name, alias=True)
- tokens[x] += alias if alias else ""
+ tokens[x] = self.resolve(tokens[x], aliases)
- def compound(self, iterator, tokens, x, alias):
+ def compound(self, iterator, tokens, x, aliases):
"""
- Resolves a compound operator (left side <operator(s)> right side).
+ Resolves column names in a compound expression (left side <operator(s)> right side).
Args:
iterator: tokens iterator
tokens: input tokens
x: current token position
- alias: if True, column aliases should be generated
+ aliases: dict of generated aliases, if present these tokens should NOT be resolved
"""
- # Alias name
- name = None
-
# Resolve left side (left side already had function processing applied through standard loop)
- if self.iscolumn(tokens[x - 1]):
- name = tokens[x - 1]
- tokens[x - 1] = self.resolver(tokens[x - 1])
+ if Token.iscolumn(tokens[x - 1]):
+ tokens[x - 1] = self.resolve(tokens[x - 1], aliases)
# Consume operator(s), handle both single and compound operators, i.e. column NOT LIKE 1
token = tokens[x]
- while token and self.isoperator(token):
+ while token and Token.isoperator(token):
x, token = next(iterator, (None, None))
# Resolve right side
- if token and self.iscolumn(token):
+ if token and Token.iscolumn(token):
# Need to process functions since it hasn't went through the standard loop yet
- if self.isfunction(tokens, x):
- self.function(iterator, tokens, token)
+ if Token.isfunction(tokens, x):
+ self.function(iterator, tokens, token, aliases)
else:
- name = token
- tokens[x] = self.resolver(token)
-
- # Add alias to last token
- if name and alias:
- alias = self.resolver(name, alias=True, compound=True)
- tokens[x] += alias if alias else ""
-
- def iscolumn(self, token):
- """
- Checks if token is a column name.
-
- Args:
- token: token to test
-
- Returns:
- True if this token is a column name token, False otherwise
- """
-
- return token and not self.isoperator(token) and not self.islogicseparator(token) and not self.isliteral(token) and not self.issortorder(token)
-
- def isoperator(self, token):
- """
- Checks if token is an operator token.
-
- Args:
- token: token to test
-
- Returns:
- True if this token is an operator, False otherwise
- """
-
- return token and token.lower() in Expression.OPERATORS
-
- def islogicseparator(self, token):
- """
- Checks if token is a logic separator token.
-
- Args:
- token: token to test
-
- Returns:
- True if this token is a logic separator, False otherwise
- """
-
- return token and token.lower() in Expression.LOGIC_SEPARATORS
-
- def issortorder(self, token):
- """
- Checks if token is a sort order token.
-
- Args:
- token: token to test
-
- Returns:
- True if this token is a sort order operator, False otherwise
- """
-
- return token and token.lower() in Expression.SORT_ORDER
-
- def isliteral(self, token):
- """
- Checks if token is a literal. Literals are wrapped in quotes, parens, wildcards or numeric.
-
- Args:
- token: token to test
-
- Returns:
- True if this token is a literal, False otherwise
- """
-
- return token and (token.startswith(("'", '"', ",", "(", ")", "[", "]", "*")) or token.replace(".", "", 1).isdigit())
+ tokens[x] = self.resolve(token, aliases)
- def wrapspace(self, text, token):
+ def resolve(self, token, aliases):
"""
- Applies whitespace wrapping rules to token.
+ Resolves this token's value if it is not an alias.
Args:
- text: current text buffer
- token: token to add
+ token: token to resolve
+ aliases: dict of generated aliases, if present these tokens should NOT be resolved
Returns:
- token with whitespace rules applied
+ resolved token value
"""
- # Wildcards have no whitespace. Need special case since * is also multiply which does have whitespace.
- if token in ["*"] and (not text or text.endswith((" ", "("))):
+ if aliases and Token.normalize(token) in aliases:
return token
- # Operator whitespace
- if self.isoperator(token) or self.islogicseparator(token) or token.lower() in ["in"]:
- return f" {token} " if not text.endswith(" ") else f"{token} "
-
- # Comma whitespace
- if token in [","]:
- return f"{token} "
-
- # No whitespace if any of the following is True
- if not text or text.endswith((" ", "(", "[")) or token in ["(", "[", ")", "]"]:
- return token
-
- # Default is to add leading whitespace
- return f" {token}"
-
- def get(self, tokens, x):
- """
- Gets token at position x. This method will validate position is valid within tokens.
-
- Args:
- tokens: input tokens
- x: position to retrieve
-
- Returns:
- tokens[x] if x is a valid position, None otherwise
- """
-
- if 0 <= x < len(tokens):
- return tokens[x]
-
- return None
+ return self.resolver(token)
diff --git a/src/python/txtai/database/sql/token.py b/src/python/txtai/database/sql/token.py
new file mode 100644
index 00000000..a8d5b5ed
--- /dev/null
+++ b/src/python/txtai/database/sql/token.py
@@ -0,0 +1,317 @@
+"""
+Token module
+"""
+
+
+class Token:
+ """
+ Methods to check for token type.
+ """
+
+ # Similar token replacement
+ SIMILAR_TOKEN = "__SIMILAR__"
+
+ # Default list of comparison operators
+ OPERATORS = ["=", "!=", "<>", ">", ">=", "<", "<=", "+", "-", "*", "/", "%", "not", "between", "like", "is", "null"]
+
+ # Default list of logic separators
+ LOGIC_SEPARATORS = ["and", "or"]
+
+ # Default alias token
+ ALIAS = ["as"]
+
+ # Default list of sort order operators
+ SORT_ORDER = ["asc", "desc"]
+
+ @staticmethod
+ def get(tokens, x):
+ """
+ Gets token at position x. This method will validate position is valid within tokens.
+
+ Args:
+ tokens: input tokens
+ x: position to retrieve
+
+ Returns:
+ tokens[x] if x is a valid position, None otherwise
+ """
+
+ if 0 <= x < len(tokens):
+ return tokens[x]
+
+ return None
+
+ @staticmethod
+ def isalias(tokens, x, alias):
+ """
+ Checks if tokens[x] is an alias keyword.
+
+ Args:
+ tokens: input tokens
+ x: current position
+ alias: if column alias processing is enabled
+
+ Returns:
+ True if tokens[x] is an alias token, False otherwise
+ """
+
+ prior = Token.get(tokens, x - 1)
+ token = tokens[x]
+
+ # True if prior token is not a separator and not a grouping token and current token is either a column token or quoted token
+ return alias and x > 0 and not Token.isseparator(prior) and not Token.isgroupstart(prior) and (Token.iscolumn(token) or Token.isquoted(token))
+
+ @staticmethod
+ def isattribute(tokens, x):
+ """
+ Checks if tokens[x] is an attribute.
+
+ Args:
+ tokens: input tokens
+ x: current position
+
+ Returns:
+ True if tokens[x] is an attribute, False otherwise
+ """
+
+ # True if token is a column and next token is not an operator
+ return Token.iscolumn(tokens[x]) and not Token.isoperator(Token.get(tokens, x + 1))
+
+ @staticmethod
+ def isbracket(token):
+ """
+ Checks if token is an open bracket.
+
+ Args:
+ token: token to test
+
+ Returns:
+ True if token is an open bracket, False otherwise
+ """
+
+ # Token is a bracket
+ return token == "["
+
+ @staticmethod
+ def iscolumn(token):
+ """
+ Checks if token is a column name.
+
+ Args:
+ token: token to test
+
+ Returns:
+ True if this token is a column name token, False otherwise
+ """
+
+ # Columns are not operators, logic separators, literals or sort order tokens
+ return (
+ token
+ and not Token.isoperator(token)
+ and not Token.islogicseparator(token)
+ and not Token.isliteral(token)
+ and not Token.issortorder(token)
+ )
+
+ @staticmethod
+ def iscompound(tokens, x):
+ """
+ Checks if tokens[x] is a compound expression.
+
+ Args:
+ tokens: input tokens
+ x: current position
+
+ Returns:
+ True if tokens[x] is a compound expression, False otherwise
+ """
+
+ # Compound expression is defined as: <column> <operator(s)> <column>
+ return Token.isoperator(tokens[x]) and (Token.iscolumn(Token.get(tokens, x - 1)) or Token.iscolumn(Token.get(tokens, x + 1)))
+
+ @staticmethod
+ def isfunction(tokens, x):
+ """
+ Checks if tokens[x] is a function.
+
+ Args:
+ tokens: input tokens
+ x: current position
+
+ Returns:
+ True if tokens[x] is a function, False otherwise
+ """
+
+ # True if a column token is followed by an open paren
+ return Token.iscolumn(tokens[x]) and Token.get(tokens, x + 1) == "("
+
+ @staticmethod
+ def isgroupstart(token):
+ """
+ Checks if token is a group start token.
+
+ Args:
+ token: token to test
+
+ Returns:
+ True if token is a group start token, False otherwise
+ """
+
+ # Token is a paren
+ return token == "("
+
+ @staticmethod
+ def isliteral(token):
+ """
+ Checks if token is a literal.
+
+ Args:
+ token: token to test
+
+ Returns:
+ True if this token is a literal, False otherwise
+ """
+
+ # Literals are wrapped in quotes, parens, wildcards or numeric.
+ return token and (token.startswith(("'", '"', ",", "(", ")", "*")) or token.replace(".", "", 1).isdigit())
+
+ @staticmethod
+ def islogicseparator(token):
+ """
+ Checks if token is a logic separator token.
+
+ Args:
+ token: token to test
+
+ Returns:
+ True if this token is a logic separator, False otherwise
+ """
+
+ # Token is a logic separator
+ return token and token.lower() in Token.LOGIC_SEPARATORS
+
+ @staticmethod
+ def isoperator(token):
+ """
+ Checks if token is an operator token.
+
+ Args:
+ token: token to test
+
+ Returns:
+ True if this token is an operator, False otherwise
+ """
+
+ # Token is an operator
+ return token and token.lower() in Token.OPERATORS
+
+ @staticmethod
+ def isquoted(token):
+ """
+ Checks if token is quoted.
+
+ Args:
+ token: token to test
+
+ Returns:
+ True if this token is quoted, False otherwise
+ """
+
+ # Token is quoted
+ return token.startswith(("'", '"')) and token.endswith(("'", '"'))
+
+ @staticmethod
+ def isseparator(token):
+ """
+ Checks if token is a separator token.
+
+ Args:
+ token to test
+
+ Returns:
+ True if this token is a separator, False otherwise
+ """
+
+ # Token is a comma
+ return token == ","
+
+ @staticmethod
+ def issimilar(tokens, x, similar):
+ """
+ Checks if tokens[x] is a similar() function.
+
+ Args:
+ tokens: input tokens
+ x: current position
+ similar: list where similar function call parameters are stored, can be None in which case similar processing is skipped
+
+ Returns:
+ True if tokens[x] is a similar clause
+ """
+
+ # True if a "similar" token is followed by an open paren
+ return similar is not None and tokens[x].lower() == "similar" and Token.get(tokens, x + 1) == "("
+
+ @staticmethod
+ def issortorder(token):
+ """
+ Checks if token is a sort order token.
+
+ Args:
+ token: token to test
+
+ Returns:
+ True if this token is a sort order operator, False otherwise
+ """
+
+ # Token is a sort order operator
+ return token and token.lower() in Token.SORT_ORDER
+
+ @staticmethod
+ def normalize(token):
+ """
+ Applies a normalization algorithm to the input token as follows:
+ - Strip single and double quotes
+ - Make lowercase
+
+ Args:
+ token: input token
+
+ Returns:
+ normalized token
+ """
+
+ # Lowercase, replace and return
+ return token.lower().replace("'", "").replace('"', "")
+
+ @staticmethod
+ def wrapspace(text, token):
+ """
+ Applies whitespace wrapping rules to token.
+
+ Args:
+ text: current text buffer
+ token: token to add
+
+ Returns:
+ token with whitespace rules applied
+ """
+
+ # Wildcards have no whitespace. Need special case since * is also multiply which does have whitespace.
+ if token in ["*"] and (not text or text.endswith((" ", "("))):
+ return token
+
+ # Operator whitespace
+ if Token.isoperator(token) or Token.islogicseparator(token) or token.lower() in ["in"]:
+ return f" {token} " if not text.endswith(" ") else f"{token} "
+
+ # Comma whitespace
+ if Token.isseparator(token):
+ return f"{token} "
+
+ # No whitespace if any of the following is True
+ if not text or text.endswith((" ", "(", "[")) or token in ["(", "[", ")", "]"] or token.startswith("."):
+ return token
+
+ # Default is to add leading whitespace
+ return f" {token}"
diff --git a/src/python/txtai/database/sqlite.py b/src/python/txtai/database/sqlite.py
index 4625a0a0..dda50b92 100644
--- a/src/python/txtai/database/sqlite.py
+++ b/src/python/txtai/database/sqlite.py
@@ -224,19 +224,23 @@ class SQLite(Database):
# Format and return results
return self.cursor.fetchall()
- def resolve(self, name, alias=False, compound=False):
+ def resolve(self, name, alias=None):
# Standard column names
sections = ["indexid", "id", "tags", "entry"]
noprefix = ["data", "object", "score", "text"]
- # Alias JSON column expressions
+ # Alias expression
if alias:
- # Only apply aliases to non-standard columns or compound expressions
- if name not in sections + noprefix or compound:
- return f' as "{name}"'
+ # Skip if name matches alias or alias is a standard column name
+ if name == alias or alias in sections:
+ return name
- # No alias
- return None
+ # Build alias clause
+ return f'{name} as "{alias}"'
+
+ # Name is already resolved, skip
+ if name.startswith("json_extract(data") or any(f"s.{s}" == name for s in sections):
+ return name
# Standard columns - need prefixes
if name.lower() in sections:
diff --git a/test/python/testapi/testcluster.py b/test/python/testapi/testcluster.py
index e8592e2d..538b1dab 100644
--- a/test/python/testapi/testcluster.py
+++ b/test/python/testapi/testcluster.py
@@ -38,8 +38,12 @@ class RequestHandler(BaseHTTPRequestHandler):
if self.path == "/count":
response = 26
elif self.path.startswith("/search?query=select"):
- if "group" in self.path:
+ if "group+by+id" in self.path:
+ response = [{"count(*)": 26}]
+ elif "group+by+text" in self.path:
response = [{"count(*)": 12, "text": "This is a test"}, {"count(*)": 14, "text": "And another test"}]
+ elif "group+by+txt" in self.path:
+ response = [{"count(*)": 12, "txt": "This is a test"}, {"count(*)": 14, "txt": "And another test"}]
else:
if self.server.server_port == 8002:
response = [{"count(*)": 12, "min(indexid)": 0, "max(indexid)": 11, "avg(indexid)": 6.3}]
@@ -185,15 +189,15 @@ class TestCluster(unittest.TestCase):
Test cluster SQL statement
"""
- query = urllib.parse.quote("select count(*), min(indexid), max(indexid), avg(indexid) from txtai where text=[This is a test]")
+ query = urllib.parse.quote("select count(*), min(indexid), max(indexid), avg(indexid) from txtai where text='This is a test'")
self.assertEqual(
self.client.get(f"search?query={query}").json(), [{"count(*)": 28, "min(indexid)": 0, "max(indexid)": 14, "avg(indexid)": 6.5}]
)
- query = urllib.parse.quote("select count(*), text from txtai group by text order by count(*) desc")
+ query = urllib.parse.quote("select count(*), text txt from txtai group by txt order by count(*) desc")
self.assertEqual(
self.client.get(f"search?query={query}").json(),
- [{"count(*)": 28, "text": "And another test"}, {"count(*)": 24, "text": "This is a test"}],
+ [{"count(*)": 28, "txt": "And another test"}, {"count(*)": 24, "txt": "This is a test"}],
)
query = urllib.parse.quote("select count(*), text from txtai group by text order by count(*) asc")
diff --git a/test/python/testdatabase/testsql.py b/test/python/testdatabase/testsql.py
index 94596b8f..e9c49515 100644
--- a/test/python/testdatabase/testsql.py
+++ b/test/python/testdatabase/testsql.py
@@ -24,6 +24,35 @@ class TestSQL(unittest.TestCase):
cls.sql = SQL(cls.db)
+ def testAlias(self):
+ """
+ Test alias clauses
+ """
+
+ self.assertSql("select", "select a as a1 from txtai", 'json_extract(data, "$.a") as a1')
+ self.assertSql("select", "select a 'a1' from txtai", "json_extract(data, \"$.a\") 'a1'")
+ self.assertSql("select", 'select a "a1" from txtai', 'json_extract(data, "$.a") "a1"')
+ self.assertSql("select", "select a a1 from txtai", 'json_extract(data, "$.a") a1')
+ self.assertSql(
+ "select",
+ "select a, b as b1, c, d + 1 as 'd1' from txtai",
+ 'json_extract(data, "$.a") as "a", json_extract(data, "$.b") as b1, '
+ + 'json_extract(data, "$.c") as "c", json_extract(data, "$.d") + 1 as \'d1\'',
+ )
+ self.assertSql("select", "select id as myid from txtai", "s.id as myid")
+ self.assertSql("select", "select length(a) t from txtai", 'length(json_extract(data, "$.a")) t')
+
+ self.assertSql("where", "select id as myid from txtai where myid != 3 and a != 1", 'myid != 3 and json_extract(data, "$.a") != 1')
+ self.assertSql("where", "select txt T from txtai where t LIKE '%abc%'", "t LIKE '%abc%'")
+ self.assertSql("where", "select txt 'T' from txtai where t LIKE '%abc%'", "t LIKE '%abc%'")
+ self.assertSql("where", "select txt \"T\" from txtai where t LIKE '%abc%'", "t LIKE '%abc%'")
+ self.assertSql("where", "select txt as T from txtai where t LIKE '%abc%'", "t LIKE '%abc%'")
+ self.assertSql("where", "select txt as 'T' from txtai where t LIKE '%abc%'", "t LIKE '%abc%'")
+ self.assertSql("where", "select txt as \"T\" from txtai where t LIKE '%abc%'", "t LIKE '%abc%'")
+
+ self.assertSql("groupby", "select id as myid, count(*) from txtai group by myid, a", 'myid, json_extract(data, "$.a")')
+ self.assertSql("orderby", "select id as myid from txtai order by myid, a", 'myid, json_extract(data, "$.a")')
+
def testBadSQL(self):
"""
Test invalid SQL
@@ -41,6 +70,25 @@ class TestSQL(unittest.TestCase):
with self.assertRaises(SQLException):
self.db.search("select a b c from txtai where id match id")
+ def testBracket(self):
+ """
+ Test bracket expressions
+ """
+
+ self.assertSql("select", "select [a] from txtai", 'json_extract(data, "$.a") as "a"')
+ self.assertSql("select", "select [a] ab from txtai", 'json_extract(data, "$.a") ab')
+ self.assertSql("select", "select [abc] from txtai", 'json_extract(data, "$.abc") as "abc"')
+ self.assertSql("select", "select [id], text, score from txtai", "s.id, text, score")
+ self.assertSql("select", "select [ab cd], text, score from txtai", 'json_extract(data, "$.ab cd") as "ab cd", text, score')
+ self.assertSql("select", "select [a[0]] from txtai", 'json_extract(data, "$.a[0]") as "a[0]"')
+ self.assertSql("select", "select [a[0].ab] from txtai", 'json_extract(data, "$.a[0].ab") as "a[0].ab"')
+ self.assertSql("select", "select [a[0].c[0]] from txtai", 'json_extract(data, "$.a[0].c[0]") as "a[0].c[0]"')
+
+ self.assertSql("where", "select * from txtai where [a b] < 1 or a > 1", 'json_extract(data, "$.a b") < 1 or json_extract(data, "$.a") > 1')
+ self.assertSql("where", "select [a[0].c[0]] a from txtai where a < 1", "a < 1")
+ self.assertSql("groupby", "select * from txtai group by [a]", 'json_extract(data, "$.a")')
+ self.assertSql("orderby", "select * from txtai where order by [a]", 'json_extract(data, "$.a")')
+
def testGroupby(self):
"""
Test group by clauses
@@ -92,7 +140,7 @@ class TestSQL(unittest.TestCase):
self.assertSql("select", "select id, indexid, tags from txtai", "s.id, s.indexid, s.tags")
self.assertSql("select", "select id, indexid, flag from txtai", 's.id, s.indexid, json_extract(data, "$.flag") as "flag"')
self.assertSql("select", "select id, indexid, a.b.c from txtai", 's.id, s.indexid, json_extract(data, "$.a.b.c") as "a.b.c"')
- self.assertSql("select", "select 'id', [id], (id) from txtai", "'id', [s.id], (s.id)")
+ self.assertSql("select", "select 'id', [id], (id) from txtai", "'id', s.id, (s.id)")
self.assertSql("select", "select * from txtai", "*")
def testSelectCompound(self):
@@ -100,12 +148,41 @@ class TestSQL(unittest.TestCase):
Test compound select clauses
"""
- self.assertSql("select", "select a + 1 from txtai", 'json_extract(data, "$.a") + 1 as "a"')
- self.assertSql("select", "select 1 * a from txtai", '1 * json_extract(data, "$.a") as "a"')
- self.assertSql("select", "select a/1 from txtai", 'json_extract(data, "$.a") / 1 as "a"')
- self.assertSql("select", "select avg(a-b) from txtai", 'avg(json_extract(data, "$.a") - json_extract(data, "$.b"))')
+ self.assertSql("select", "select a + 1 from txtai", 'json_extract(data, "$.a") + 1 as "a + 1"')
+ self.assertSql("select", "select 1 * a from txtai", '1 * json_extract(data, "$.a") as "1 * a"')
+ self.assertSql("select", "select a/1 from txtai", 'json_extract(data, "$.a") / 1 as "a / 1"')
+ self.assertSql("select", "select avg(a-b) from txtai", 'avg(json_extract(data, "$.a") - json_extract(data, "$.b")) as "avg(a - b)"')
self.assertSql("select", "select distinct(text) from txtai", "distinct(text)")
- self.assertSql("select", "select id, score, (a/2)*3 from txtai", 's.id, score, (json_extract(data, "$.a") / 2) * 3')
+ self.assertSql("select", "select id, score, (a/2)*3 from txtai", 's.id, score, (json_extract(data, "$.a") / 2) * 3 as "(a / 2) * 3"')
+ self.assertSql("select", "select id, score, (a/2*3) from txtai", 's.id, score, (json_extract(data, "$.a") / 2 * 3) as "(a / 2 * 3)"')
+ self.assertSql(
+ "select",
+ "select func(func2(indexid + 1), a) from txtai",
+ 'func(func2(s.indexid + 1), json_extract(data, "$.a")) as "func(func2(indexid + 1), a)"',
+ )
+ self.assertSql("select", "select func(func2(indexid + 1), a) a from txtai", 'func(func2(s.indexid + 1), json_extract(data, "$.a")) a')
+
+ def testSimilar(self):
+ """
+ Test similar functions
+ """
+
+ prefix = "select * from txtai "
+
+ self.assertSql("where", prefix + "where similar('abc')", "__SIMILAR__0")
+ self.assertSql("similar", prefix + "where similar('abc')", [["abc"]])
+
+ self.assertSql("where", prefix + "where similar('abc') AND id = 1", "__SIMILAR__0 AND s.id = 1")
+ self.assertSql("similar", prefix + "where similar('abc')", [["abc"]])
+
+ self.assertSql("where", prefix + "where similar('abc') and similar('def')", "__SIMILAR__0 and __SIMILAR__1")
+ self.assertSql("similar", prefix + "where similar('abc') and similar('def')", [["abc"], ["def"]])
+
+ self.assertSql("where", prefix + "where similar('abc', 1000)", "__SIMILAR__0")
+ self.assertSql("similar", prefix + "where similar('abc', 1000)", [["abc", "1000"]])
+
+ self.assertSql("where", prefix + "where similar('abc', 1000) and similar('def', 10)", "__SIMILAR__0 and __SIMILAR__1")
+ self.assertSql("similar", prefix + "where similar('abc', 1000) and similar('def', 10)", [["abc", "1000"], ["def", "10"]])
def testWhereBasic(self):
"""
@@ -115,6 +192,7 @@ class TestSQL(unittest.TestCase):
prefix = "select * from txtai "
self.assertSql("where", prefix + "where a = b", 'json_extract(data, "$.a") = json_extract(data, "$.b")')
+ self.assertSql("where", prefix + "where abc = def", 'json_extract(data, "$.abc") = json_extract(data, "$.def")')
self.assertSql("where", prefix + "where a = b.value", 'json_extract(data, "$.a") = json_extract(data, "$.b.value")')
self.assertSql("where", prefix + "where a = 1", 'json_extract(data, "$.a") = 1')
self.assertSql("where", prefix + "WHERE 1 = a", '1 = json_extract(data, "$.a")')
@@ -145,28 +223,6 @@ class TestSQL(unittest.TestCase):
self.assertSql("where", prefix + "where f(id) = b(id)", "f(s.id) = b(s.id)")
self.assertSql("where", prefix + "WHERE f(id)", "f(s.id)")
- def testWhereSimilar(self):
- """
- Test similar functions
- """
-
- prefix = "select * from txtai "
-
- self.assertSql("where", prefix + "where similar('abc')", "__SIMILAR__0")
- self.assertSql("similar", prefix + "where similar('abc')", [["abc"]])
-
- self.assertSql("where", prefix + "where similar('abc') AND id = 1", "__SIMILAR__0 AND s.id = 1")
- self.assertSql("similar", prefix + "where similar('abc')", [["abc"]])
-
- self.assertSql("where", prefix + "where similar('abc') and similar('def')", "__SIMILAR__0 and __SIMILAR__1")
- self.assertSql("similar", prefix + "where similar('abc') and similar('def')", [["abc"], ["def"]])
-
- self.assertSql("where", prefix + "where similar('abc', 1000)", "__SIMILAR__0")
- self.assertSql("similar", prefix + "where similar('abc', 1000)", [["abc", "1000"]])
-
- self.assertSql("where", prefix + "where similar('abc', 1000) and similar('def', 10)", "__SIMILAR__0 and __SIMILAR__1")
- self.assertSql("similar", prefix + "where similar('abc', 1000) and similar('def', 10)", [["abc", "1000"], ["def", "10"]])
-
def assertSql(self, clause, query, expected):
"""
Helper method to assert a query clause is as expected.