💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add DuckDB support, closes #462
davidmezzettid4dc811f181e
diff --git a/setup.py b/setup.pyindex 946309ec..1744f765 100644--- a/setup.py+++ b/setup.py@@ -24,7 +24,7 @@ extras["cloud"] = ["apache-libcloud>=3.3.1"]extras["console"] = ["rich>=12.0.1"]-extras["database"] = ["pillow>=7.1.2"]+extras["database"] = ["duckdb>=0.7.1", "pillow>=7.1.2"]extras["graph"] = ["networkx>=2.6.3", "python-louvain>=0.16"]diff --git a/src/python/txtai/database/__init__.py b/src/python/txtai/database/__init__.pyindex 6ac4f3a8..4655c4f4 100644--- a/src/python/txtai/database/__init__.py+++ b/src/python/txtai/database/__init__.py@@ -3,7 +3,9 @@ Database imports"""from .base import Database+from .duckdb import DuckDBfrom .encoder import *from .factory import DatabaseFactory+from .filedb import FileDBfrom .sqlite import SQLitefrom .sql import *diff --git a/src/python/txtai/database/duckdb.py b/src/python/txtai/database/duckdb.pynew file mode 100644index 00000000..1ae614d5--- /dev/null+++ b/src/python/txtai/database/duckdb.py@@ -0,0 +1,79 @@+"""+DuckDB module+"""++import os++from tempfile import TemporaryDirectory++import duckdb++from .filedb import FileDB+++class DuckDB(FileDB):+ """+ Database instance backed by DuckDB.+ """++ def connect(self, path=":memory:"):+ # Create connection and start a transaction+ # pylint: disable=I1101+ connection = duckdb.connect(path)+ connection.begin()++ return connection++ def getcursor(self):+ return self.connection++ def rows(self):+ # Yield rows from batches of query results+ batch = 256+ rows = self.cursor.fetchmany(batch)+ while rows:+ yield from rows+ rows = self.cursor.fetchmany(batch)++ def addfunctions(self):+ # DuckDB doesn't currently support scalar functions+ return++ def copy(self, path):+ # Delete existing file, if necessary+ if os.path.exists(path):+ os.remove(path)++ # Create database connection+ # pylint: disable=I1101+ connection = duckdb.connect(path)++ # List of tables+ tables = ["documents", "objects", "sections"]++ with TemporaryDirectory() as directory:+ # Export existing tables+ for table in tables:+ self.connection.execute(f"COPY {table} TO '{directory}/{table}.parquet' (FORMAT parquet)")++ # Create initial schema+ for schema in [FileDB.CREATE_DOCUMENTS, FileDB.CREATE_OBJECTS, FileDB.CREATE_SECTIONS % "sections"]:+ connection.execute(schema)++ # Import tables into new schema+ for table in tables:+ connection.execute(f"COPY {table} FROM '{directory}/{table}.parquet' (FORMAT parquet)")++ # Create indexes and sync data to database file+ connection.execute(FileDB.CREATE_SECTIONS_INDEX)+ connection.execute("CHECKPOINT")++ # Delete empty WAL files+ wal = f"{path}.wal"+ if os.path.exists(wal) and os.path.getsize(wal) == 0:+ os.remove(wal)++ # Start transaction+ connection.begin()++ return connectiondiff --git a/src/python/txtai/database/factory.py b/src/python/txtai/database/factory.pyindex ee4db61a..7ac92d82 100644--- a/src/python/txtai/database/factory.py+++ b/src/python/txtai/database/factory.py@@ -4,6 +4,7 @@ Factory modulefrom ..util import Resolver+from .duckdb import DuckDBfrom .sqlite import SQLite@@ -37,6 +38,8 @@ class DatabaseFactory:# Create document database instanceif content == "sqlite":database = SQLite(config)+ elif content == "duckdb":+ database = DuckDB(config)elif content:database = DatabaseFactory.resolve(content, config)diff --git a/src/python/txtai/database/filedb.py b/src/python/txtai/database/filedb.pynew file mode 100644index 00000000..f48f55f2--- /dev/null+++ b/src/python/txtai/database/filedb.py@@ -0,0 +1,532 @@+"""+FileDB module+"""++import datetime+import json++from .base import Database+++# pylint: disable=R0904+class FileDB(Database):+ """+ Base file database class.+ """++ # Temporary table for working with id batches+ CREATE_BATCH = """+ CREATE TEMP TABLE IF NOT EXISTS batch (+ indexid INTEGER,+ id TEXT,+ batch INTEGER+ )+ """++ DELETE_BATCH = "DELETE FROM batch"+ INSERT_BATCH_INDEXID = "INSERT INTO batch (indexid, batch) VALUES (?, ?)"+ INSERT_BATCH_ID = "INSERT INTO batch (id, batch) VALUES (?, ?)"++ # Temporary table for joining similarity scores+ CREATE_SCORES = """+ CREATE TEMP TABLE IF NOT EXISTS scores (+ indexid INTEGER PRIMARY KEY,+ score REAL+ )+ """++ DELETE_SCORES = "DELETE FROM scores"+ INSERT_SCORE = "INSERT INTO scores VALUES (?, ?)"++ # Documents - stores full content+ CREATE_DOCUMENTS = """+ CREATE TABLE IF NOT EXISTS documents (+ id TEXT PRIMARY KEY,+ data JSON,+ tags TEXT,+ entry DATETIME+ )+ """++ INSERT_DOCUMENT = "INSERT OR REPLACE INTO documents VALUES (?, ?, ?, ?)"+ DELETE_DOCUMENTS = "DELETE FROM documents WHERE id IN (SELECT id FROM batch)"++ # Objects - stores binary content+ CREATE_OBJECTS = """+ CREATE TABLE IF NOT EXISTS objects (+ id TEXT PRIMARY KEY,+ object BLOB,+ tags TEXT,+ entry DATETIME+ )+ """++ INSERT_OBJECT = "INSERT OR REPLACE INTO objects VALUES (?, ?, ?, ?)"+ DELETE_OBJECTS = "DELETE FROM objects WHERE id IN (SELECT id FROM batch)"++ # Sections - stores section text+ CREATE_SECTIONS = """+ CREATE TABLE IF NOT EXISTS %s (+ indexid INTEGER PRIMARY KEY,+ id TEXT,+ text TEXT,+ tags TEXT,+ entry DATETIME+ )+ """++ CREATE_SECTIONS_INDEX = "CREATE INDEX section_id ON sections(id)"+ INSERT_SECTION = "INSERT INTO sections VALUES (?, ?, ?, ?, ?)"+ DELETE_SECTIONS = "DELETE FROM sections WHERE id IN (SELECT id FROM batch)"+ COPY_SECTIONS = (+ "INSERT INTO %s SELECT (select count(*) - 1 from sections s1 where s.indexid >= s1.indexid) indexid, "+ + "s.id, %s AS text, s.tags, s.entry FROM sections s LEFT JOIN documents d ON s.id = d.id ORDER BY indexid"+ )+ STREAM_SECTIONS = "SELECT s.id, s.text, object, s.tags FROM %s s LEFT JOIN objects o ON s.id = o.id ORDER BY indexid"+ DROP_SECTIONS = "DROP TABLE sections"+ RENAME_SECTIONS = "ALTER TABLE %s RENAME TO sections"++ # Queries+ SELECT_IDS = "SELECT indexid, id FROM sections WHERE id in (SELECT id FROM batch)"++ # Partial sql clauses+ TABLE_CLAUSE = (+ "SELECT %s FROM sections s "+ + "LEFT JOIN documents d ON s.id = d.id "+ + "LEFT JOIN objects o ON s.id = o.id "+ + "LEFT JOIN scores sc ON s.indexid = sc.indexid"+ )+ IDS_CLAUSE = "s.indexid in (SELECT indexid from batch WHERE batch=%s)"++ def __init__(self, config):+ """+ Creates a new Database.++ Args:+ config: database configuration parameters+ """++ super().__init__(config)++ # Database connection handle+ self.connection = None+ self.cursor = None+ self.path = None++ def load(self, path):+ # Load an existing database. Thread locking must be handled externally.+ self.connection = self.connect(path)+ self.cursor = self.getcursor()+ self.path = path++ # Register custom functions+ self.addfunctions()++ def insert(self, documents, index=0):+ # Initialize connection if not open+ self.initialize()++ # Get entry date+ entry = datetime.datetime.now()++ # Insert documents+ for uid, document, tags in documents:+ if isinstance(document, dict):+ # Insert document and use return value for sections table+ document = self.insertdocument(uid, document, tags, entry)++ if document is not None:+ if isinstance(document, list):+ # Join tokens to text+ document = " ".join(document)+ elif not isinstance(document, str):+ # If object support is enabled, save object+ self.insertobject(uid, document, tags, entry)++ # Clear section text for objects, even when objects aren't inserted+ document = None++ # Save text section+ self.insertsection(index, uid, document, tags, entry)+ index += 1++ def delete(self, ids):+ if self.connection:+ # Batch ids+ self.batch(ids=ids)++ # Delete all documents, objects and sections by id+ self.cursor.execute(FileDB.DELETE_DOCUMENTS)+ self.cursor.execute(FileDB.DELETE_OBJECTS)+ self.cursor.execute(FileDB.DELETE_SECTIONS)++ def reindex(self, columns=None):+ if self.connection:+ # Working table name+ name = "rebuild"++ # Resolve and build column strings if provided+ select = "text"+ if columns:+ select = "|| ' ' ||".join([self.resolve(c) for c in columns])++ # Create new table to hold reordered sections+ self.cursor.execute(FileDB.CREATE_SECTIONS % name)++ # Copy data over+ self.cursor.execute(FileDB.COPY_SECTIONS % (name, select))++ # Stream new results+ self.cursor.execute(FileDB.STREAM_SECTIONS % name)+ for uid, text, obj, tags in self.rows():+ if not text and self.encoder and obj:+ yield (uid, self.encoder.decode(obj), tags)+ else:+ yield (uid, text, tags)++ # Swap as new table+ self.cursor.execute(FileDB.DROP_SECTIONS)+ self.cursor.execute(FileDB.RENAME_SECTIONS % name)+ self.cursor.execute(FileDB.CREATE_SECTIONS_INDEX)++ def save(self, path):+ # Temporary database+ if not self.path:+ # Save temporary database+ self.connection.commit()++ # Copy data from current to new+ connection = self.copy(path)++ # Close temporary database+ self.connection.close()++ # Point connection to new connection+ self.connection = connection+ self.cursor = self.getcursor()+ self.path = path++ # Register custom functions+ self.addfunctions()++ # Paths are equal, commit changes+ elif self.path == path:+ self.connection.commit()++ # New path is different from current path, copy data and continue using current connection+ else:+ self.copy(path).close()++ def close(self):+ # Close connection+ if self.connection:+ self.connection.close()++ def ids(self, ids):+ # Batch ids and run query+ self.batch(ids=ids)+ self.cursor.execute(FileDB.SELECT_IDS)++ # Format and return results+ return self.cursor.fetchall()++ def resolve(self, name, alias=None):+ # Standard column names+ sections = ["indexid", "id", "tags", "entry"]+ noprefix = ["data", "object", "score", "text"]++ # Alias expression+ if alias:+ # Skip if name matches alias or alias is a standard column name+ if name == alias or alias in sections:+ return name++ # Build alias clause+ return f'{name} as "{alias}"'++ # Resolve expression+ if self.expressions and name in self.expressions:+ return self.expressions[name]++ # Name is already resolved, skip+ if name.startswith("json_extract(data") or any(f"s.{s}" == name for s in sections):+ return name++ # Standard columns - need prefixes+ if name.lower() in sections:+ return f"s.{name}"++ # Standard columns - no prefixes+ if name.lower() in noprefix:+ return name++ # Other columns come from documents.data JSON+ return f"json_extract(data, '$.{name}')"++ def embed(self, similarity, batch):+ # Load similarity results id batch+ self.batch(indexids=[i for i, _ in similarity[batch]], batch=batch)++ # Average and load all similarity scores with first batch+ if not batch:+ self.scores(similarity)++ # Return ids clause placeholder+ return FileDB.IDS_CLAUSE % batch++ # pylint: disable=R0912+ def query(self, query, limit):+ # Extract query components+ select = query.get("select", self.defaults())+ where = query.get("where")+ groupby, having = query.get("groupby"), query.get("having")+ orderby, qlimit, offset = query.get("orderby"), query.get("limit"), query.get("offset")+ similarity = query.get("similar")++ # Build query text+ query = FileDB.TABLE_CLAUSE % select+ if where is not None:+ query += f" WHERE {where}"+ if groupby is not None:+ query += f" GROUP BY {groupby}"+ if having is not None:+ query += f" HAVING {having}"+ if orderby is not None:+ query += f" ORDER BY {orderby}"++ # Default ORDER BY if not provided and similarity scores are available+ if similarity and orderby is None:+ query += " ORDER BY score DESC"++ # Apply query limit+ if qlimit is not None or limit:+ query += f" LIMIT {qlimit if qlimit else limit}"++ # Apply offset+ if offset is not None:+ query += f" OFFSET {offset}"++ # Clear scores when no similar clauses present+ if not similarity:+ self.scores(None)++ # Runs a user query through execute method, which has common user query handling logic+ self.execute(self.cursor.execute, query)++ # Retrieve column list from query+ columns = [c[0] for c in self.cursor.description]++ # Map results and return+ results = []+ for row in self.rows():+ result = {}++ # Copy columns to result. In cases with duplicate column names, find one with a value+ for x, column in enumerate(columns):+ if column not in result or result[column] is None:+ # Decode object+ if self.encoder and column == "object":+ result[column] = self.encoder.decode(row[x])+ else:+ result[column] = row[x]++ results.append(result)++ return results++ def initialize(self):+ """+ Creates connection and initial database schema if no connection exists.+ """++ if not self.connection:+ # Create temporary database. Thread locking must be handled externally.+ self.connection = self.connect()+ self.cursor = self.getcursor()++ # Register custom functions+ self.addfunctions()++ # Create initial schema and indices+ self.cursor.execute(FileDB.CREATE_DOCUMENTS)+ self.cursor.execute(FileDB.CREATE_OBJECTS)+ self.cursor.execute(FileDB.CREATE_SECTIONS % "sections")+ self.cursor.execute(FileDB.CREATE_SECTIONS_INDEX)++ def insertdocument(self, uid, document, tags, entry):+ """+ Inserts a document.++ Args:+ uid: unique id+ document: input document+ tags: document tags+ entry: generated entry date++ Returns:+ section value+ """++ # Make a copy of document before changing+ document = document.copy()++ # Get and remove object field from document+ obj = document.pop("object") if "object" in document else None++ # Insert document as JSON+ if document:+ self.cursor.execute(FileDB.INSERT_DOCUMENT, [uid, json.dumps(document, allow_nan=False), tags, entry])++ # Get value of text field+ text = document.get("text")++ # If both text and object are set, insert object as it won't otherwise be used+ if text and obj:+ self.insertobject(uid, obj, tags, entry)++ # Return value to use for section - use text if available otherwise use object+ return text if text else obj++ def insertobject(self, uid, obj, tags, entry):+ """+ Inserts an object.++ Args:+ uid: unique id+ obj: input object+ tags: object tags+ entry: generated entry date+ """++ # If object support is enabled, save object+ if self.encoder:+ self.cursor.execute(FileDB.INSERT_OBJECT, [uid, self.encoder.encode(obj), tags, entry])++ def insertsection(self, index, uid, text, tags, entry):+ """+ Inserts a section.++ Args:+ index: index id+ uid: unique id+ text: section text+ tags: section tags+ entry: generated entry date+ """++ # Save text section+ self.cursor.execute(FileDB.INSERT_SECTION, [index, uid, text, tags, entry])++ def defaults(self):+ """+ Returns a list of default columns when there is no select clause.++ Returns:+ list of default columns+ """++ return "s.id, text, score"++ def batch(self, indexids=None, ids=None, batch=None):+ """+ Loads ids to a temporary batch table for efficient query processing.++ Args:+ indexids: list of indexids+ ids: list of ids+ batch: batch index, used when statement has multiple subselects+ """++ # Create or Replace temporary batch table+ self.cursor.execute(FileDB.CREATE_BATCH)++ # Delete batch when batch id is empty or for batch 0+ if not batch:+ self.cursor.execute(FileDB.DELETE_BATCH)++ if indexids:+ self.cursor.executemany(FileDB.INSERT_BATCH_INDEXID, [(i, batch) for i in indexids])+ if ids:+ self.cursor.executemany(FileDB.INSERT_BATCH_ID, [(str(uid), batch) for uid in ids])++ def scores(self, similarity):+ """+ Loads a batch of similarity scores to a temporary table for efficient query processing.++ Args:+ similarity: similarity results as [(indexid, score)]+ """++ # Create or Replace temporary scores table+ self.cursor.execute(FileDB.CREATE_SCORES)++ # Delete scores+ self.cursor.execute(FileDB.DELETE_SCORES)++ if similarity:+ # Average scores per id, needed for multiple similar() clauses+ scores = {}+ for s in similarity:+ for i, score in s:+ if i not in scores:+ scores[i] = []+ scores[i].append(score)++ # Average scores by id+ self.cursor.executemany(FileDB.INSERT_SCORE, [(i, sum(s) / len(s)) for i, s in scores.items()])++ def connect(self, path=None):+ """+ Creates a new database connection.++ Args:+ path: path to database file++ Returns:+ connection+ """++ raise NotImplementedError++ def getcursor(self):+ """+ Opens a cursor for current connection.++ Returns:+ cursor+ """++ raise NotImplementedError++ def rows(self):+ """+ Returns current cursor row iterator for last executed query.++ Args:+ cursor: cursor++ Returns:+ iterable collection of rows+ """++ raise NotImplementedError++ def addfunctions(self):+ """+ Adds custom functions in current connection.+ """++ raise NotImplementedError++ def copy(self, path):+ """+ Copies the current database into path.++ Args:+ path: path to write database++ Returns:+ new connection with data copied over+ """++ raise NotImplementedErrordiff --git a/src/python/txtai/database/sqlite.py b/src/python/txtai/database/sqlite.pyindex 4fd7e497..438acf5c 100644--- a/src/python/txtai/database/sqlite.py+++ b/src/python/txtai/database/sqlite.py@@ -2,344 +2,27 @@SQLite module"""-import datetime-import jsonimport osimport sqlite3-from .base import Database+from .filedb import FileDB-class SQLite(Database):+class SQLite(FileDB):"""- Document database backed by SQLite.+ Database instance backed by SQLite."""- # Temporary table for working with id batches- CREATE_BATCH = """- CREATE TEMP TABLE IF NOT EXISTS batch (- indexid INTEGER,- id TEXT,- batch INTEGER- )- """-- DELETE_BATCH = "DELETE FROM batch"- INSERT_BATCH_INDEXID = "INSERT INTO batch (indexid, batch) VALUES (?, ?)"- INSERT_BATCH_ID = "INSERT INTO batch (id, batch) VALUES (?, ?)"-- # Temporary table for joining similarity scores- CREATE_SCORES = """- CREATE TEMP TABLE IF NOT EXISTS scores (- indexid INTEGER PRIMARY KEY,- score REAL- )- """-- DELETE_SCORES = "DELETE FROM scores"- INSERT_SCORE = "INSERT INTO scores VALUES (?, ?)"-- # Documents - stores full content- CREATE_DOCUMENTS = """- CREATE TABLE IF NOT EXISTS documents (- id TEXT PRIMARY KEY,- data JSON,- tags TEXT,- entry DATETIME- )- """-- INSERT_DOCUMENT = "REPLACE INTO documents VALUES (?, ?, ?, ?)"- DELETE_DOCUMENTS = "DELETE FROM documents WHERE id IN (SELECT id FROM batch)"-- # Objects - stores binary content- CREATE_OBJECTS = """- CREATE TABLE IF NOT EXISTS objects (- id TEXT PRIMARY KEY,- object BLOB,- tags TEXT,- entry DATETIME- )- """-- INSERT_OBJECT = "REPLACE INTO objects VALUES (?, ?, ?, ?)"- DELETE_OBJECTS = "DELETE FROM objects WHERE id IN (SELECT id FROM batch)"-- # Sections - stores section text- CREATE_SECTIONS = """- CREATE TABLE IF NOT EXISTS %s (- indexid INTEGER PRIMARY KEY,- id TEXT,- text TEXT,- tags TEXT,- entry DATETIME- )- """-- CREATE_SECTIONS_INDEX = "CREATE INDEX section_id ON sections(id)"- INSERT_SECTION = "INSERT INTO sections VALUES (?, ?, ?, ?, ?)"- DELETE_SECTIONS = "DELETE FROM sections WHERE id IN (SELECT id FROM batch)"- COPY_SECTIONS = (- "INSERT INTO %s SELECT (select count(*) - 1 from sections s1 where s.indexid >= s1.indexid) indexid, "- + "s.id, %s AS text, s.tags, s.entry FROM sections s LEFT JOIN documents d ON s.id = d.id ORDER BY indexid"- )- STREAM_SECTIONS = "SELECT s.id, s.text, object, s.tags FROM %s s LEFT JOIN objects o ON s.id = o.id ORDER BY indexid"- DROP_SECTIONS = "DROP TABLE sections"- RENAME_SECTIONS = "ALTER TABLE %s RENAME TO sections"-- # Queries- SELECT_IDS = "SELECT indexid, id FROM sections WHERE id in (SELECT id FROM batch)"-- # Partial sql clauses- TABLE_CLAUSE = (- "SELECT %s FROM sections s "- + "LEFT JOIN documents d ON s.id = d.id "- + "LEFT JOIN objects o ON s.id = o.id "- + "LEFT JOIN scores sc ON s.indexid = sc.indexid"- )- IDS_CLAUSE = "s.indexid in (SELECT indexid from batch WHERE batch=%s)"-- def __init__(self, config):- """- Creates a new Database.-- Args:- config: database configuration parameters- """-- super().__init__(config)-- # SQLite connection handle- self.connection = None- self.cursor = None- self.path = None-- def load(self, path):- # Load an existing database. Thread locking must be handled externally.- self.connection = sqlite3.connect(path, check_same_thread=False)- self.cursor = self.connection.cursor()- self.path = path-- # Register custom functions- self.addfunctions()-- def insert(self, documents, index=0):- # Initialize connection if not open- self.initialize()-- # Get entry date- entry = datetime.datetime.now()-- # Insert documents- for uid, document, tags in documents:- if isinstance(document, dict):- # Insert document and use return value for sections table- document = self.insertdocument(uid, document, tags, entry)-- if document is not None:- if isinstance(document, list):- # Join tokens to text- document = " ".join(document)- elif not isinstance(document, str):- # If object support is enabled, save object- self.insertobject(uid, document, tags, entry)-- # Clear section text for objects, even when objects aren't inserted- document = None-- # Save text section- self.insertsection(index, uid, document, tags, entry)- index += 1-- def delete(self, ids):- if self.connection:- # Batch ids- self.batch(ids=ids)-- # Delete all documents, objects and sections by id- self.cursor.execute(SQLite.DELETE_DOCUMENTS)- self.cursor.execute(SQLite.DELETE_OBJECTS)- self.cursor.execute(SQLite.DELETE_SECTIONS)-- def reindex(self, columns=None):- if self.connection:- # Working table name- name = "rebuild"-- # Resolve and build column strings if provided- select = "text"- if columns:- select = "|| ' ' ||".join([self.resolve(c) for c in columns])-- # Create new table to hold reordered sections- self.cursor.execute(SQLite.CREATE_SECTIONS % name)-- # Copy data over- self.cursor.execute(SQLite.COPY_SECTIONS % (name, select))-- # Stream new results- self.cursor.execute(SQLite.STREAM_SECTIONS % name)- for uid, text, obj, tags in self.cursor:- if not text and self.encoder and obj:- yield (uid, self.encoder.decode(obj), tags)- else:- yield (uid, text, tags)-- # Swap as new table- self.cursor.execute(SQLite.DROP_SECTIONS)- self.cursor.execute(SQLite.RENAME_SECTIONS % name)- self.cursor.execute(SQLite.CREATE_SECTIONS_INDEX)-- def save(self, path):- # Temporary database- if not self.path:- # Save temporary database- self.connection.commit()-- # Copy data from current to new- connection = self.copy(path)-- # Close temporary database- self.connection.close()-- # Point connection to new connection- self.connection = connection- self.cursor = self.connection.cursor()- self.path = path-- # Register custom functions- self.addfunctions()-- # Paths are equal, commit changes- elif self.path == path:- self.connection.commit()-- # New path is different from current path, copy data and continue using current connection- else:- self.copy(path).close()-- def close(self):- # Close connection- if self.connection:- self.connection.close()-- def ids(self, ids):- # Batch ids and run query- self.batch(ids=ids)- self.cursor.execute(SQLite.SELECT_IDS)-- # Format and return results- return self.cursor.fetchall()-- def resolve(self, name, alias=None):- # Standard column names- sections = ["indexid", "id", "tags", "entry"]- noprefix = ["data", "object", "score", "text"]-- # Alias expression- if alias:- # Skip if name matches alias or alias is a standard column name- if name == alias or alias in sections:- return name-- # Build alias clause- return f'{name} as "{alias}"'-- # Resolve expression- if self.expressions and name in self.expressions:- return self.expressions[name]-- # Name is already resolved, skip- if name.startswith("json_extract(data") or any(f"s.{s}" == name for s in sections):- return name-- # Standard columns - need prefixes- if name.lower() in sections:- return f"s.{name}"+ def connect(self, path=""):+ return sqlite3.connect(path, check_same_thread=False)- # Standard columns - no prefixes- if name.lower() in noprefix:- return name+ def getcursor(self):+ return self.connection.cursor()- # Other columns come from documents.data JSON- return f'json_extract(data, "$.{name}")'-- def embed(self, similarity, batch):- # Load similarity results id batch- self.batch(indexids=[i for i, _ in similarity[batch]], batch=batch)-- # Average and load all similarity scores with first batch- if not batch:- self.scores(similarity)-- # Return ids clause placeholder- return SQLite.IDS_CLAUSE % batch-- # pylint: disable=R0912- def query(self, query, limit):- # Extract query components- select = query.get("select", self.defaults())- where = query.get("where")- groupby, having = query.get("groupby"), query.get("having")- orderby, qlimit, offset = query.get("orderby"), query.get("limit"), query.get("offset")- similarity = query.get("similar")-- # Build query text- query = SQLite.TABLE_CLAUSE % select- if where is not None:- query += f" WHERE {where}"- if groupby is not None:- query += f" GROUP BY {groupby}"- if having is not None:- query += f" HAVING {having}"- if orderby is not None:- query += f" ORDER BY {orderby}"-- # Default ORDER BY if not provided and similarity scores are available- if similarity and orderby is None:- query += " ORDER BY score DESC"-- # Apply query limit- if qlimit is not None or limit:- query += f" LIMIT {qlimit if qlimit else limit}"-- # Apply offset- if offset is not None:- query += f" OFFSET {offset}"-- # Clear scores when no similar clauses present- if not similarity:- self.scores(None)-- # Runs a user query through execute method, which has common user query handling logic- self.execute(self.cursor.execute, query)-- # Retrieve column list from query- columns = [c[0] for c in self.cursor.description]-- # Map results and return- results = []- for row in self.cursor:- result = {}-- # Copy columns to result. In cases with duplicate column names, find one with a value- for x, column in enumerate(columns):- if column not in result or result[column] is None:- # Decode object- if self.encoder and column == "object":- result[column] = self.encoder.decode(row[x])- else:- result[column] = row[x]-- results.append(result)-- return results+ def rows(self):+ return self.cursordef addfunctions(self):- """- Adds custom functions in current connection.- """-if self.connection and self.functions:# Enable callback tracebacks to show user-defined function errorssqlite3.enable_callback_tracebacks(True)@@ -347,109 +30,13 @@ class SQLite(Database):for name, argcount, fn in self.functions:self.connection.create_function(name, argcount, fn)- def initialize(self):- """- Creates connection and initial database schema if no connection exists.- """-- if not self.connection:- name = "sections"-- # Create temporary database. Thread locking must be handled externally.- self.connection = sqlite3.connect("", check_same_thread=False)- self.cursor = self.connection.cursor()-- # Register custom functions- self.addfunctions()-- # Create initial schema and indices- self.cursor.execute(SQLite.CREATE_DOCUMENTS)- self.cursor.execute(SQLite.CREATE_OBJECTS)- self.cursor.execute(SQLite.CREATE_SECTIONS % name)- self.cursor.execute(SQLite.CREATE_SECTIONS_INDEX)-- def insertdocument(self, uid, document, tags, entry):- """- Inserts a document.-- Args:- uid: unique id- document: input document- tags: document tags- entry: generated entry date-- Returns:- section value- """-- # Make a copy of document before changing- document = document.copy()-- # Get and remove object field from document- obj = document.pop("object") if "object" in document else None-- # Insert document as JSON- if document:- self.cursor.execute(SQLite.INSERT_DOCUMENT, [uid, json.dumps(document, allow_nan=False), tags, entry])-- # Get value of text field- text = document.get("text")-- # If both text and object are set, insert object as it won't otherwise be used- if text and obj:- self.insertobject(uid, obj, tags, entry)-- # Return value to use for section - use text if available otherwise use object- return text if text else obj-- def insertobject(self, uid, obj, tags, entry):- """- Inserts an object.-- Args:- uid: unique id- obj: input object- tags: object tags- entry: generated entry date- """-- # If object support is enabled, save object- if self.encoder:- self.cursor.execute(SQLite.INSERT_OBJECT, [uid, self.encoder.encode(obj), tags, entry])-- def insertsection(self, index, uid, text, tags, entry):- """- Inserts a section.-- Args:- index: index id- uid: unique id- text: section text- tags: section tags- entry: generated entry date- """-- # Save text section- self.cursor.execute(SQLite.INSERT_SECTION, [index, uid, text, tags, entry])-def copy(self, path):- """- Copies the current database into path. This method will use the backup API if the current connection has no uncommitted changes.- Otherwise, iterdump is used, as the backup call will hang for an uncommitted connection.-- Args:- path: path to write database-- Returns:- new connection with data copied over- """-# Delete existing file, if necessaryif os.path.exists(path):os.remove(path)# Create database. Thread locking must be handled externally.- connection = sqlite3.connect(path, check_same_thread=False)+ connection = self.connect(path)if self.connection.in_transaction:# The backup call will hang if there are uncommitted changes, need to copy over@@ -461,61 +48,3 @@ class SQLite(Database):self.connection.backup(connection)return connection-- def defaults(self):- """- Returns a list of default columns when there is no select clause.-- Returns:- list of default columns- """-- return "s.id, text, score"-- def batch(self, indexids=None, ids=None, batch=None):- """- Loads ids to a temporary batch table for efficient query processing.-- Args:- indexids: list of indexids- ids: list of ids- batch: batch index, used when statement has multiple subselects- """-- # Create or Replace temporary batch table- self.cursor.execute(SQLite.CREATE_BATCH)-- # Delete batch when batch id is empty or for batch 0- if not batch:- self.cursor.execute(SQLite.DELETE_BATCH)-- if indexids:- self.cursor.executemany(SQLite.INSERT_BATCH_INDEXID, [(i, batch) for i in indexids])- if ids:- self.cursor.executemany(SQLite.INSERT_BATCH_ID, [(str(uid), batch) for uid in ids])-- def scores(self, similarity):- """- Loads a batch of similarity scores to a temporary table for efficient query processing.-- Args:- similarity: similarity results as [(indexid, score)]- """-- # Create or Replace temporary scores table- self.cursor.execute(SQLite.CREATE_SCORES)-- # Delete scores- self.cursor.execute(SQLite.DELETE_SCORES)-- if similarity:- # Average scores per id, needed for multiple similar() clauses- scores = {}- for s in similarity:- for i, score in s:- if i not in scores:- scores[i] = []- scores[i].append(score)-- # Average scores by id- self.cursor.executemany(SQLite.INSERT_SCORE, [(i, sum(s) / len(s)) for i, s in scores.items()])diff --git a/test/python/testdatabase/testduckdb.py b/test/python/testdatabase/testduckdb.pynew file mode 100644index 00000000..507dc01a--- /dev/null+++ b/test/python/testdatabase/testduckdb.py@@ -0,0 +1,44 @@+"""+Embeddings+database module tests+"""++from txtai.embeddings import Embeddings++from .testsqlite import TestSQLite+++# pylint: disable=R0904+class TestDuckDB(TestSQLite):+ """+ Embeddings with content stored in DuckDB.+ """++ @classmethod+ def setUpClass(cls):+ """+ Initialize test data.+ """++ cls.data = [+ "US tops 5 million confirmed virus cases",+ "Canada's last fully intact ice shelf has suddenly collapsed, forming a Manhattan-sized iceberg",+ "Beijing mobilises invasion craft along coast as Taiwan tensions escalate",+ "The National Park Service warns against sacrificing slower friends in a bear attack",+ "Maine man wins $1M from $25 lottery ticket",+ "Make huge profits without work, earn up to $100,000 a day",+ ]++ # Content backend+ cls.backend = "duckdb"++ # Create embeddings model, backed by sentence-transformers & transformers+ cls.embeddings = Embeddings({"path": "sentence-transformers/nli-mpnet-base-v2", "content": cls.backend})++ @classmethod+ def tearDownClass(cls):+ """+ Cleanup data.+ """++ if cls.embeddings:+ cls.embeddings.close()diff --git a/test/python/testdatabase/testfiledb.py b/test/python/testdatabase/testfiledb.pynew file mode 100644index 00000000..d01865a0--- /dev/null+++ b/test/python/testdatabase/testfiledb.py@@ -0,0 +1,26 @@+"""+FileDB module tests+"""++import unittest++from txtai.database import FileDB+++class TestFileDB(unittest.TestCase):+ """+ Test base file database methods+ """++ def testNotImplemented(self):+ """+ Test exceptions for non-implemented methods+ """++ db = FileDB({})++ self.assertRaises(NotImplementedError, db.connect, None)+ self.assertRaises(NotImplementedError, db.getcursor)+ self.assertRaises(NotImplementedError, db.rows)+ self.assertRaises(NotImplementedError, db.addfunctions)+ self.assertRaises(NotImplementedError, db.copy, None)diff --git a/test/python/testdatabase/testsql.py b/test/python/testdatabase/testsql.pyindex 2c396703..c46ae2a3 100644--- a/test/python/testdatabase/testsql.py+++ b/test/python/testdatabase/testsql.py@@ -29,20 +29,20 @@ class TestSQL(unittest.TestCase):Test alias clauses"""- self.assertSql("select", "select a as a1 from txtai", 'json_extract(data, "$.a") as a1')- self.assertSql("select", "select a 'a1' from txtai", "json_extract(data, \"$.a\") 'a1'")- self.assertSql("select", 'select a "a1" from txtai', 'json_extract(data, "$.a") "a1"')- self.assertSql("select", "select a a1 from txtai", 'json_extract(data, "$.a") a1')+ self.assertSql("select", "select a as a1 from txtai", "json_extract(data, '$.a') as a1")+ self.assertSql("select", "select a 'a1' from txtai", "json_extract(data, '$.a') 'a1'")+ self.assertSql("select", 'select a "a1" from txtai', "json_extract(data, '$.a') \"a1\"")+ self.assertSql("select", "select a a1 from txtai", "json_extract(data, '$.a') a1")self.assertSql("select","select a, b as b1, c, d + 1 as 'd1' from txtai",- 'json_extract(data, "$.a") as "a", json_extract(data, "$.b") as b1, '- + 'json_extract(data, "$.c") as "c", json_extract(data, "$.d") + 1 as \'d1\'',+ "json_extract(data, '$.a') as \"a\", json_extract(data, '$.b') as b1, "+ + "json_extract(data, '$.c') as \"c\", json_extract(data, '$.d') + 1 as 'd1'",)self.assertSql("select", "select id as myid from txtai", "s.id as myid")- self.assertSql("select", "select length(a) t from txtai", 'length(json_extract(data, "$.a")) t')+ self.assertSql("select", "select length(a) t from txtai", "length(json_extract(data, '$.a')) t")- self.assertSql("where", "select id as myid from txtai where myid != 3 and a != 1", 'myid != 3 and json_extract(data, "$.a") != 1')+ self.assertSql("where", "select id as myid from txtai where myid != 3 and a != 1", "myid != 3 and json_extract(data, '$.a') != 1")self.assertSql("where", "select txt T from txtai where t LIKE '%abc%'", "t LIKE '%abc%'")self.assertSql("where", "select txt 'T' from txtai where t LIKE '%abc%'", "t LIKE '%abc%'")self.assertSql("where", "select txt \"T\" from txtai where t LIKE '%abc%'", "t LIKE '%abc%'")@@ -50,8 +50,8 @@ class TestSQL(unittest.TestCase):self.assertSql("where", "select txt as 'T' from txtai where t LIKE '%abc%'", "t LIKE '%abc%'")self.assertSql("where", "select txt as \"T\" from txtai where t LIKE '%abc%'", "t LIKE '%abc%'")- self.assertSql("groupby", "select id as myid, count(*) from txtai group by myid, a", 'myid, json_extract(data, "$.a")')- self.assertSql("orderby", "select id as myid from txtai order by myid, a", 'myid, json_extract(data, "$.a")')+ self.assertSql("groupby", "select id as myid, count(*) from txtai group by myid, a", "myid, json_extract(data, '$.a')")+ self.assertSql("orderby", "select id as myid from txtai order by myid, a", "myid, json_extract(data, '$.a')")def testBadSQL(self):"""@@ -75,20 +75,20 @@ class TestSQL(unittest.TestCase):Test bracket expressions"""- self.assertSql("select", "select [a] from txtai", 'json_extract(data, "$.a") as "a"')- self.assertSql("select", "select [a] ab from txtai", 'json_extract(data, "$.a") ab')- self.assertSql("select", "select [abc] from txtai", 'json_extract(data, "$.abc") as "abc"')+ self.assertSql("select", "select [a] from txtai", "json_extract(data, '$.a') as \"a\"")+ self.assertSql("select", "select [a] ab from txtai", "json_extract(data, '$.a') ab")+ self.assertSql("select", "select [abc] from txtai", "json_extract(data, '$.abc') as \"abc\"")self.assertSql("select", "select [id], text, score from txtai", "s.id, text, score")- self.assertSql("select", "select [ab cd], text, score from txtai", 'json_extract(data, "$.ab cd") as "ab cd", text, score')- self.assertSql("select", "select [a[0]] from txtai", 'json_extract(data, "$.a[0]") as "a[0]"')- self.assertSql("select", "select [a[0].ab] from txtai", 'json_extract(data, "$.a[0].ab") as "a[0].ab"')- self.assertSql("select", "select [a[0].c[0]] from txtai", 'json_extract(data, "$.a[0].c[0]") as "a[0].c[0]"')- self.assertSql("select", "select avg([a]) from txtai", 'avg(json_extract(data, "$.a")) as "avg([a])"')+ self.assertSql("select", "select [ab cd], text, score from txtai", "json_extract(data, '$.ab cd') as \"ab cd\", text, score")+ self.assertSql("select", "select [a[0]] from txtai", "json_extract(data, '$.a[0]') as \"a[0]\"")+ self.assertSql("select", "select [a[0].ab] from txtai", "json_extract(data, '$.a[0].ab') as \"a[0].ab\"")+ self.assertSql("select", "select [a[0].c[0]] from txtai", "json_extract(data, '$.a[0].c[0]') as \"a[0].c[0]\"")+ self.assertSql("select", "select avg([a]) from txtai", "avg(json_extract(data, '$.a')) as \"avg([a])\"")- self.assertSql("where", "select * from txtai where [a b] < 1 or a > 1", 'json_extract(data, "$.a b") < 1 or json_extract(data, "$.a") > 1')+ self.assertSql("where", "select * from txtai where [a b] < 1 or a > 1", "json_extract(data, '$.a b') < 1 or json_extract(data, '$.a') > 1")self.assertSql("where", "select [a[0].c[0]] a from txtai where a < 1", "a < 1")- self.assertSql("groupby", "select * from txtai group by [a]", 'json_extract(data, "$.a")')- self.assertSql("orderby", "select * from txtai where order by [a]", 'json_extract(data, "$.a")')+ self.assertSql("groupby", "select * from txtai group by [a]", "json_extract(data, '$.a')")+ self.assertSql("orderby", "select * from txtai where order by [a]", "json_extract(data, '$.a')")def testDistinct(self):"""@@ -98,24 +98,24 @@ class TestSQL(unittest.TestCase):# Attributesself.assertSql("select", "select distinct id from txtai", "distinct s.id")self.assertSql("select", "select distinct id as myid from txtai", "distinct s.id as myid")- self.assertSql("select", "select distinct a from txtai", 'distinct json_extract(data, "$.a") as "a"')- self.assertSql("select", "select distinct a.b from txtai", 'distinct json_extract(data, "$.a.b") as "a.b"')+ self.assertSql("select", "select distinct a from txtai", "distinct json_extract(data, '$.a') as \"a\"")+ self.assertSql("select", "select distinct a.b from txtai", "distinct json_extract(data, '$.a.b') as \"a.b\"")# Bracket expression- self.assertSql("select", "select distinct [ab cd] from txtai", 'distinct json_extract(data, "$.ab cd") as "distinct[ab cd]"')+ self.assertSql("select", "select distinct [ab cd] from txtai", "distinct json_extract(data, '$.ab cd') as \"distinct[ab cd]\"")# Function expressionself.assertSql("select", "select distinct(id) from txtai", 'distinct(s.id) as "distinct(id)"')self.assertSql("select", "select count(distinct id) from txtai", 'count(distinct s.id) as "count(distinct id)"')- self.assertSql("select", "select count(distinct a) from txtai", 'count(distinct json_extract(data, "$.a")) as "count(distinct a)"')+ self.assertSql("select", "select count(distinct a) from txtai", "count(distinct json_extract(data, '$.a')) as \"count(distinct a)\"")self.assertSql("select", "select count(distinct avg(id)) from txtai", 'count(distinct avg(s.id)) as "count(distinct avg(id))"')self.assertSql(- "select", "select count(distinct avg(a)) from txtai", 'count(distinct avg(json_extract(data, "$.a"))) as "count(distinct avg(a))"'+ "select", "select count(distinct avg(a)) from txtai", "count(distinct avg(json_extract(data, '$.a'))) as \"count(distinct avg(a))\"")# Compound expression- self.assertSql("select", "select distinct a/1 from txtai", 'distinct json_extract(data, "$.a") / 1 as "a / 1"')- self.assertSql("select", "select distinct(a/1) from txtai", 'distinct(json_extract(data, "$.a") / 1) as "distinct(a / 1)"')+ self.assertSql("select", "select distinct a/1 from txtai", "distinct json_extract(data, '$.a') / 1 as \"a / 1\"")+ self.assertSql("select", "select distinct(a/1) from txtai", "distinct(json_extract(data, '$.a') / 1) as \"distinct(a / 1)\"")def testGroupby(self):"""@@ -125,7 +125,7 @@ class TestSQL(unittest.TestCase):prefix = "select count(*), flag from txtai "self.assertSql("groupby", prefix + "group by text", "text")- self.assertSql("groupby", prefix + "group by distinct(a)", 'distinct(json_extract(data, "$.a"))')+ self.assertSql("groupby", prefix + "group by distinct(a)", "distinct(json_extract(data, '$.a'))")self.assertSql("groupby", prefix + "where a > 1 group by text", "text")def testHaving(self):@@ -184,8 +184,8 @@ class TestSQL(unittest.TestCase):"""self.assertSql("select", "select id, indexid, tags from txtai", "s.id, s.indexid, s.tags")- self.assertSql("select", "select id, indexid, flag from txtai", 's.id, s.indexid, json_extract(data, "$.flag") as "flag"')- self.assertSql("select", "select id, indexid, a.b.c from txtai", 's.id, s.indexid, json_extract(data, "$.a.b.c") as "a.b.c"')+ self.assertSql("select", "select id, indexid, flag from txtai", "s.id, s.indexid, json_extract(data, '$.flag') as \"flag\"")+ self.assertSql("select", "select id, indexid, a.b.c from txtai", "s.id, s.indexid, json_extract(data, '$.a.b.c') as \"a.b.c\"")self.assertSql("select", "select 'id', [id], (id) from txtai", "'id', s.id, (s.id)")self.assertSql("select", "select * from txtai", "*")@@ -194,22 +194,22 @@ class TestSQL(unittest.TestCase):Test compound select clauses"""- self.assertSql("select", "select a + 1 from txtai", 'json_extract(data, "$.a") + 1 as "a + 1"')- self.assertSql("select", "select 1 * a from txtai", '1 * json_extract(data, "$.a") as "1 * a"')- self.assertSql("select", "select a/1 from txtai", 'json_extract(data, "$.a") / 1 as "a / 1"')- self.assertSql("select", "select avg(a-b) from txtai", 'avg(json_extract(data, "$.a") - json_extract(data, "$.b")) as "avg(a - b)"')+ self.assertSql("select", "select a + 1 from txtai", "json_extract(data, '$.a') + 1 as \"a + 1\"")+ self.assertSql("select", "select 1 * a from txtai", "1 * json_extract(data, '$.a') as \"1 * a\"")+ self.assertSql("select", "select a/1 from txtai", "json_extract(data, '$.a') / 1 as \"a / 1\"")+ self.assertSql("select", "select avg(a-b) from txtai", "avg(json_extract(data, '$.a') - json_extract(data, '$.b')) as \"avg(a - b)\"")self.assertSql("select", "select distinct(text) from txtai", "distinct(text)")- self.assertSql("select", "select id, score, (a/2)*3 from txtai", 's.id, score, (json_extract(data, "$.a") / 2) * 3 as "(a / 2) * 3"')- self.assertSql("select", "select id, score, (a/2*3) from txtai", 's.id, score, (json_extract(data, "$.a") / 2 * 3) as "(a / 2 * 3)"')+ self.assertSql("select", "select id, score, (a/2)*3 from txtai", "s.id, score, (json_extract(data, '$.a') / 2) * 3 as \"(a / 2) * 3\"")+ self.assertSql("select", "select id, score, (a/2*3) from txtai", "s.id, score, (json_extract(data, '$.a') / 2 * 3) as \"(a / 2 * 3)\"")self.assertSql("select","select func(func2(indexid + 1), a) from txtai",- 'func(func2(s.indexid + 1), json_extract(data, "$.a")) as "func(func2(indexid + 1), a)"',+ "func(func2(s.indexid + 1), json_extract(data, '$.a')) as \"func(func2(indexid + 1), a)\"",)- self.assertSql("select", "select func(func2(indexid + 1), a) a from txtai", 'func(func2(s.indexid + 1), json_extract(data, "$.a")) a')+ self.assertSql("select", "select func(func2(indexid + 1), a) a from txtai", "func(func2(s.indexid + 1), json_extract(data, '$.a')) a")self.assertSql("select", "select 'prefix' || id from txtai", "'prefix' || s.id as \"'prefix' || id\"")self.assertSql("select", "select 'prefix' || id id from txtai", "'prefix' || s.id id")- self.assertSql("select", "select 'prefix' || a a from txtai", "'prefix' || json_extract(data, \"$.a\") a")+ self.assertSql("select", "select 'prefix' || a a from txtai", "'prefix' || json_extract(data, '$.a') a")def testSimilar(self):"""@@ -251,15 +251,15 @@ class TestSQL(unittest.TestCase):prefix = "select * from txtai "- self.assertSql("where", prefix + "where a = b", 'json_extract(data, "$.a") = json_extract(data, "$.b")')- self.assertSql("where", prefix + "where abc = def", 'json_extract(data, "$.abc") = json_extract(data, "$.def")')- self.assertSql("where", prefix + "where a = b.value", 'json_extract(data, "$.a") = json_extract(data, "$.b.value")')- self.assertSql("where", prefix + "where a = 1", 'json_extract(data, "$.a") = 1')- self.assertSql("where", prefix + "WHERE 1 = a", '1 = json_extract(data, "$.a")')- self.assertSql("where", prefix + "WHERE a LIKE 'abc'", "json_extract(data, \"$.a\") LIKE 'abc'")- self.assertSql("where", prefix + "WHERE a NOT LIKE 'abc'", "json_extract(data, \"$.a\") NOT LIKE 'abc'")- self.assertSql("where", prefix + "WHERE a IN (1, 2, 3, b)", 'json_extract(data, "$.a") IN (1, 2, 3, json_extract(data, "$.b"))')- self.assertSql("where", prefix + "WHERE a is not null", 'json_extract(data, "$.a") is not null')+ self.assertSql("where", prefix + "where a = b", "json_extract(data, '$.a') = json_extract(data, '$.b')")+ self.assertSql("where", prefix + "where abc = def", "json_extract(data, '$.abc') = json_extract(data, '$.def')")+ self.assertSql("where", prefix + "where a = b.value", "json_extract(data, '$.a') = json_extract(data, '$.b.value')")+ self.assertSql("where", prefix + "where a = 1", "json_extract(data, '$.a') = 1")+ self.assertSql("where", prefix + "WHERE 1 = a", "1 = json_extract(data, '$.a')")+ self.assertSql("where", prefix + "WHERE a LIKE 'abc'", "json_extract(data, '$.a') LIKE 'abc'")+ self.assertSql("where", prefix + "WHERE a NOT LIKE 'abc'", "json_extract(data, '$.a') NOT LIKE 'abc'")+ self.assertSql("where", prefix + "WHERE a IN (1, 2, 3, b)", "json_extract(data, '$.a') IN (1, 2, 3, json_extract(data, '$.b'))")+ self.assertSql("where", prefix + "WHERE a is not null", "json_extract(data, '$.a') is not null")self.assertSql("where", prefix + "WHERE score >= 0.15", "score >= 0.15")def testWhereCompound(self):@@ -269,15 +269,15 @@ class TestSQL(unittest.TestCase):prefix = "select * from txtai "- self.assertSql("where", prefix + "where a > (b + 1)", 'json_extract(data, "$.a") > (json_extract(data, "$.b") + 1)')- self.assertSql("where", prefix + "where a > func('abc')", "json_extract(data, \"$.a\") > func('abc')")+ self.assertSql("where", prefix + "where a > (b + 1)", "json_extract(data, '$.a') > (json_extract(data, '$.b') + 1)")+ self.assertSql("where", prefix + "where a > func('abc')", "json_extract(data, '$.a') > func('abc')")self.assertSql(- "where", prefix + "where (id = 1 or id = 2) and a like 'abc'", "(s.id = 1 or s.id = 2) and json_extract(data, \"$.a\") like 'abc'"+ "where", prefix + "where (id = 1 or id = 2) and a like 'abc'", "(s.id = 1 or s.id = 2) and json_extract(data, '$.a') like 'abc'")self.assertSql("where",prefix + "where a > f(d(b, c, 1),1)",- 'json_extract(data, "$.a") > f(d(json_extract(data, "$.b"), json_extract(data, "$.c"), 1), 1)',+ "json_extract(data, '$.a') > f(d(json_extract(data, '$.b'), json_extract(data, '$.c'), 1), 1)",)self.assertSql("where", prefix + "where (id = 1 AND id = 2) OR indexid = 3", "(s.id = 1 AND s.id = 2) OR s.indexid = 3")self.assertSql("where", prefix + "where f(id) = b(id)", "f(s.id) = b(s.id)")diff --git a/test/python/testdatabase/testembeddings.py b/test/python/testdatabase/testsqlite.pysimilarity index 91%rename from test/python/testdatabase/testembeddings.pyrename to test/python/testdatabase/testsqlite.pyindex e7664d2e..bdc2cbbd 100644--- a/test/python/testdatabase/testembeddings.py+++ b/test/python/testdatabase/testsqlite.py@@ -13,9 +13,9 @@ from txtai.database import Database, SQLError# pylint: disable=R0904-class TestEmbeddings(unittest.TestCase):+class TestSQLite(unittest.TestCase):"""- Embeddings with a database tests.+ Embeddings with content stored in SQLite"""@classmethod@@ -33,8 +33,11 @@ class TestEmbeddings(unittest.TestCase):"Make huge profits without work, earn up to $100,000 a day",]+ # Content backend+ cls.backend = "sqlite"+# Create embeddings model, backed by sentence-transformers & transformers- cls.embeddings = Embeddings({"path": "sentence-transformers/nli-mpnet-base-v2", "content": True})+ cls.embeddings = Embeddings({"path": "sentence-transformers/nli-mpnet-base-v2", "content": cls.backend})@classmethoddef tearDownClass(cls):@@ -55,7 +58,7 @@ class TestEmbeddings(unittest.TestCase):self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])# Generate temp file path- index = os.path.join(tempfile.gettempdir(), f"embeddings.{extension}")+ index = os.path.join(tempfile.gettempdir(), f"embeddings.{self.backend}.{extension}")self.embeddings.save(index)self.embeddings.load(index)@@ -78,13 +81,13 @@ class TestEmbeddings(unittest.TestCase):# Create index twice to test open/close and ensure resources are freedfor _ in range(2):- embeddings = Embeddings({"path": "sentence-transformers/nli-mpnet-base-v2", "content": True})+ embeddings = Embeddings({"path": "sentence-transformers/nli-mpnet-base-v2", "content": self.backend})# Add record to indexembeddings.index([(0, "Close test", None)])# Save index- index = os.path.join(tempfile.gettempdir(), "embeddings.close")+ index = os.path.join(tempfile.gettempdir(), f"embeddings.{self.backend}.close")embeddings.save(index)# Close index@@ -131,7 +134,7 @@ class TestEmbeddings(unittest.TestCase):"""# Test search against empty index- embeddings = Embeddings({"path": "sentence-transformers/nli-mpnet-base-v2", "content": True})+ embeddings = Embeddings({"path": "sentence-transformers/nli-mpnet-base-v2", "content": self.backend})self.assertEqual(embeddings.search("test"), [])# Test index with no data@@ -180,8 +183,8 @@ class TestEmbeddings(unittest.TestCase):embeddings = Embeddings({"path": "sentence-transformers/nli-mpnet-base-v2",- "content": True,- "functions": [{"name": "length", "function": "testdatabase.testembeddings.length"}],+ "content": self.backend,+ "functions": [{"name": "length", "function": "testdatabase.testsqlite.length"}],})@@ -256,7 +259,7 @@ class TestEmbeddings(unittest.TestCase):"""embeddings = Embeddings(- {"path": "sentence-transformers/nli-mpnet-base-v2", "content": True, "instructions": {"query": "query: ", "data": "passage: "}}+ {"path": "sentence-transformers/nli-mpnet-base-v2", "content": self.backend, "instructions": {"query": "query: ", "data": "passage: "}})embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])@@ -284,14 +287,14 @@ class TestEmbeddings(unittest.TestCase):{"format": "json","path": "sentence-transformers/nli-mpnet-base-v2",- "content": True,+ "content": self.backend,})embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])# Generate temp file path- index = os.path.join(tempfile.gettempdir(), "embeddings.json")+ index = os.path.join(tempfile.gettempdir(), f"embeddings.{self.backend}.json")embeddings.save(index)@@ -307,7 +310,7 @@ class TestEmbeddings(unittest.TestCase):Test indexing with multiple data types (text, documents)"""- embeddings = Embeddings({"path": "sentence-transformers/nli-mpnet-base-v2", "content": True, "batch": len(self.data)})+ embeddings = Embeddings({"path": "sentence-transformers/nli-mpnet-base-v2", "content": self.backend, "batch": len(self.data)})# Create an index using mixed data (text and documents)data = []@@ -331,14 +334,14 @@ class TestEmbeddings(unittest.TestCase):self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])# Save original index- index = os.path.join(tempfile.gettempdir(), "embeddings.insert")+ index = os.path.join(tempfile.gettempdir(), f"embeddings.{self.backend}.insert")self.embeddings.save(index)# Modify indexself.embeddings.upsert([(0, "Looking out into the dreadful abyss", None)])# Save to a different location- indexupdate = os.path.join(tempfile.gettempdir(), "embeddings.update")+ indexupdate = os.path.join(tempfile.gettempdir(), f"embeddings.{self.backend}.update")self.embeddings.save(indexupdate)# Save to same location@@ -379,13 +382,15 @@ class TestEmbeddings(unittest.TestCase):Test index"""- embeddings = Embeddings({"path": "sentence-transformers/nli-mpnet-base-v2", "content": True, "query": {"path": "neuml/t5-small-txtsql"}})+ embeddings = Embeddings(+ {"path": "sentence-transformers/nli-mpnet-base-v2", "content": self.backend, "query": {"path": "neuml/t5-small-txtsql"}}+ )# Create an index for the list of textembeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])# Search for best match- result = embeddings.search("feel good story with maine in text", 1)[0]+ result = embeddings.search("feel good story with win in text", 1)[0]self.assertEqual(result["text"], self.data[4])@@ -417,7 +422,7 @@ class TestEmbeddings(unittest.TestCase):self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])# Generate temp file path- index = os.path.join(tempfile.gettempdir(), "embeddings")+ index = os.path.join(tempfile.gettempdir(), f"embeddings.{self.backend}")self.embeddings.save(index)self.embeddings.load(index)@@ -441,7 +446,7 @@ class TestEmbeddings(unittest.TestCase):# Test similarresult = self.embeddings.search(- "select * from txtai where similar('feel good story') group by text having count(*) > 0 order by score desc", 1+ "select text, score from txtai where similar('feel good story') group by text, score having count(*) > 0 order by score desc", 1)[0]self.assertEqual(result["text"], self.data[4])