Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Add DuckDB support, closes #462

davidmezzettid4dc811f181e
diff --git a/setup.py b/setup.py
index 946309ec..1744f765 100644
--- a/setup.py
+++ b/setup.py
@@ -24,7 +24,7 @@ extras["cloud"] = ["apache-libcloud>=3.3.1"]
extras["console"] = ["rich>=12.0.1"]
-extras["database"] = ["pillow>=7.1.2"]
+extras["database"] = ["duckdb>=0.7.1", "pillow>=7.1.2"]
extras["graph"] = ["networkx>=2.6.3", "python-louvain>=0.16"]
diff --git a/src/python/txtai/database/__init__.py b/src/python/txtai/database/__init__.py
index 6ac4f3a8..4655c4f4 100644
--- a/src/python/txtai/database/__init__.py
+++ b/src/python/txtai/database/__init__.py
@@ -3,7 +3,9 @@ Database imports
"""
from .base import Database
+from .duckdb import DuckDB
from .encoder import *
from .factory import DatabaseFactory
+from .filedb import FileDB
from .sqlite import SQLite
from .sql import *
diff --git a/src/python/txtai/database/duckdb.py b/src/python/txtai/database/duckdb.py
new file mode 100644
index 00000000..1ae614d5
--- /dev/null
+++ b/src/python/txtai/database/duckdb.py
@@ -0,0 +1,79 @@
+"""
+DuckDB module
+"""
+
+import os
+
+from tempfile import TemporaryDirectory
+
+import duckdb
+
+from .filedb import FileDB
+
+
+class DuckDB(FileDB):
+ """
+ Database instance backed by DuckDB.
+ """
+
+ def connect(self, path=":memory:"):
+ # Create connection and start a transaction
+ # pylint: disable=I1101
+ connection = duckdb.connect(path)
+ connection.begin()
+
+ return connection
+
+ def getcursor(self):
+ return self.connection
+
+ def rows(self):
+ # Yield rows from batches of query results
+ batch = 256
+ rows = self.cursor.fetchmany(batch)
+ while rows:
+ yield from rows
+ rows = self.cursor.fetchmany(batch)
+
+ def addfunctions(self):
+ # DuckDB doesn't currently support scalar functions
+ return
+
+ def copy(self, path):
+ # Delete existing file, if necessary
+ if os.path.exists(path):
+ os.remove(path)
+
+ # Create database connection
+ # pylint: disable=I1101
+ connection = duckdb.connect(path)
+
+ # List of tables
+ tables = ["documents", "objects", "sections"]
+
+ with TemporaryDirectory() as directory:
+ # Export existing tables
+ for table in tables:
+ self.connection.execute(f"COPY {table} TO '{directory}/{table}.parquet' (FORMAT parquet)")
+
+ # Create initial schema
+ for schema in [FileDB.CREATE_DOCUMENTS, FileDB.CREATE_OBJECTS, FileDB.CREATE_SECTIONS % "sections"]:
+ connection.execute(schema)
+
+ # Import tables into new schema
+ for table in tables:
+ connection.execute(f"COPY {table} FROM '{directory}/{table}.parquet' (FORMAT parquet)")
+
+ # Create indexes and sync data to database file
+ connection.execute(FileDB.CREATE_SECTIONS_INDEX)
+ connection.execute("CHECKPOINT")
+
+ # Delete empty WAL files
+ wal = f"{path}.wal"
+ if os.path.exists(wal) and os.path.getsize(wal) == 0:
+ os.remove(wal)
+
+ # Start transaction
+ connection.begin()
+
+ return connection
diff --git a/src/python/txtai/database/factory.py b/src/python/txtai/database/factory.py
index ee4db61a..7ac92d82 100644
--- a/src/python/txtai/database/factory.py
+++ b/src/python/txtai/database/factory.py
@@ -4,6 +4,7 @@ Factory module
from ..util import Resolver
+from .duckdb import DuckDB
from .sqlite import SQLite
@@ -37,6 +38,8 @@ class DatabaseFactory:
# Create document database instance
if content == "sqlite":
database = SQLite(config)
+ elif content == "duckdb":
+ database = DuckDB(config)
elif content:
database = DatabaseFactory.resolve(content, config)
diff --git a/src/python/txtai/database/filedb.py b/src/python/txtai/database/filedb.py
new file mode 100644
index 00000000..f48f55f2
--- /dev/null
+++ b/src/python/txtai/database/filedb.py
@@ -0,0 +1,532 @@
+"""
+FileDB module
+"""
+
+import datetime
+import json
+
+from .base import Database
+
+
+# pylint: disable=R0904
+class FileDB(Database):
+ """
+ Base file database class.
+ """
+
+ # Temporary table for working with id batches
+ CREATE_BATCH = """
+ CREATE TEMP TABLE IF NOT EXISTS batch (
+ indexid INTEGER,
+ id TEXT,
+ batch INTEGER
+ )
+ """
+
+ DELETE_BATCH = "DELETE FROM batch"
+ INSERT_BATCH_INDEXID = "INSERT INTO batch (indexid, batch) VALUES (?, ?)"
+ INSERT_BATCH_ID = "INSERT INTO batch (id, batch) VALUES (?, ?)"
+
+ # Temporary table for joining similarity scores
+ CREATE_SCORES = """
+ CREATE TEMP TABLE IF NOT EXISTS scores (
+ indexid INTEGER PRIMARY KEY,
+ score REAL
+ )
+ """
+
+ DELETE_SCORES = "DELETE FROM scores"
+ INSERT_SCORE = "INSERT INTO scores VALUES (?, ?)"
+
+ # Documents - stores full content
+ CREATE_DOCUMENTS = """
+ CREATE TABLE IF NOT EXISTS documents (
+ id TEXT PRIMARY KEY,
+ data JSON,
+ tags TEXT,
+ entry DATETIME
+ )
+ """
+
+ INSERT_DOCUMENT = "INSERT OR REPLACE INTO documents VALUES (?, ?, ?, ?)"
+ DELETE_DOCUMENTS = "DELETE FROM documents WHERE id IN (SELECT id FROM batch)"
+
+ # Objects - stores binary content
+ CREATE_OBJECTS = """
+ CREATE TABLE IF NOT EXISTS objects (
+ id TEXT PRIMARY KEY,
+ object BLOB,
+ tags TEXT,
+ entry DATETIME
+ )
+ """
+
+ INSERT_OBJECT = "INSERT OR REPLACE INTO objects VALUES (?, ?, ?, ?)"
+ DELETE_OBJECTS = "DELETE FROM objects WHERE id IN (SELECT id FROM batch)"
+
+ # Sections - stores section text
+ CREATE_SECTIONS = """
+ CREATE TABLE IF NOT EXISTS %s (
+ indexid INTEGER PRIMARY KEY,
+ id TEXT,
+ text TEXT,
+ tags TEXT,
+ entry DATETIME
+ )
+ """
+
+ CREATE_SECTIONS_INDEX = "CREATE INDEX section_id ON sections(id)"
+ INSERT_SECTION = "INSERT INTO sections VALUES (?, ?, ?, ?, ?)"
+ DELETE_SECTIONS = "DELETE FROM sections WHERE id IN (SELECT id FROM batch)"
+ COPY_SECTIONS = (
+ "INSERT INTO %s SELECT (select count(*) - 1 from sections s1 where s.indexid >= s1.indexid) indexid, "
+ + "s.id, %s AS text, s.tags, s.entry FROM sections s LEFT JOIN documents d ON s.id = d.id ORDER BY indexid"
+ )
+ STREAM_SECTIONS = "SELECT s.id, s.text, object, s.tags FROM %s s LEFT JOIN objects o ON s.id = o.id ORDER BY indexid"
+ DROP_SECTIONS = "DROP TABLE sections"
+ RENAME_SECTIONS = "ALTER TABLE %s RENAME TO sections"
+
+ # Queries
+ SELECT_IDS = "SELECT indexid, id FROM sections WHERE id in (SELECT id FROM batch)"
+
+ # Partial sql clauses
+ TABLE_CLAUSE = (
+ "SELECT %s FROM sections s "
+ + "LEFT JOIN documents d ON s.id = d.id "
+ + "LEFT JOIN objects o ON s.id = o.id "
+ + "LEFT JOIN scores sc ON s.indexid = sc.indexid"
+ )
+ IDS_CLAUSE = "s.indexid in (SELECT indexid from batch WHERE batch=%s)"
+
+ def __init__(self, config):
+ """
+ Creates a new Database.
+
+ Args:
+ config: database configuration parameters
+ """
+
+ super().__init__(config)
+
+ # Database connection handle
+ self.connection = None
+ self.cursor = None
+ self.path = None
+
+ def load(self, path):
+ # Load an existing database. Thread locking must be handled externally.
+ self.connection = self.connect(path)
+ self.cursor = self.getcursor()
+ self.path = path
+
+ # Register custom functions
+ self.addfunctions()
+
+ def insert(self, documents, index=0):
+ # Initialize connection if not open
+ self.initialize()
+
+ # Get entry date
+ entry = datetime.datetime.now()
+
+ # Insert documents
+ for uid, document, tags in documents:
+ if isinstance(document, dict):
+ # Insert document and use return value for sections table
+ document = self.insertdocument(uid, document, tags, entry)
+
+ if document is not None:
+ if isinstance(document, list):
+ # Join tokens to text
+ document = " ".join(document)
+ elif not isinstance(document, str):
+ # If object support is enabled, save object
+ self.insertobject(uid, document, tags, entry)
+
+ # Clear section text for objects, even when objects aren't inserted
+ document = None
+
+ # Save text section
+ self.insertsection(index, uid, document, tags, entry)
+ index += 1
+
+ def delete(self, ids):
+ if self.connection:
+ # Batch ids
+ self.batch(ids=ids)
+
+ # Delete all documents, objects and sections by id
+ self.cursor.execute(FileDB.DELETE_DOCUMENTS)
+ self.cursor.execute(FileDB.DELETE_OBJECTS)
+ self.cursor.execute(FileDB.DELETE_SECTIONS)
+
+ def reindex(self, columns=None):
+ if self.connection:
+ # Working table name
+ name = "rebuild"
+
+ # Resolve and build column strings if provided
+ select = "text"
+ if columns:
+ select = "|| ' ' ||".join([self.resolve(c) for c in columns])
+
+ # Create new table to hold reordered sections
+ self.cursor.execute(FileDB.CREATE_SECTIONS % name)
+
+ # Copy data over
+ self.cursor.execute(FileDB.COPY_SECTIONS % (name, select))
+
+ # Stream new results
+ self.cursor.execute(FileDB.STREAM_SECTIONS % name)
+ for uid, text, obj, tags in self.rows():
+ if not text and self.encoder and obj:
+ yield (uid, self.encoder.decode(obj), tags)
+ else:
+ yield (uid, text, tags)
+
+ # Swap as new table
+ self.cursor.execute(FileDB.DROP_SECTIONS)
+ self.cursor.execute(FileDB.RENAME_SECTIONS % name)
+ self.cursor.execute(FileDB.CREATE_SECTIONS_INDEX)
+
+ def save(self, path):
+ # Temporary database
+ if not self.path:
+ # Save temporary database
+ self.connection.commit()
+
+ # Copy data from current to new
+ connection = self.copy(path)
+
+ # Close temporary database
+ self.connection.close()
+
+ # Point connection to new connection
+ self.connection = connection
+ self.cursor = self.getcursor()
+ self.path = path
+
+ # Register custom functions
+ self.addfunctions()
+
+ # Paths are equal, commit changes
+ elif self.path == path:
+ self.connection.commit()
+
+ # New path is different from current path, copy data and continue using current connection
+ else:
+ self.copy(path).close()
+
+ def close(self):
+ # Close connection
+ if self.connection:
+ self.connection.close()
+
+ def ids(self, ids):
+ # Batch ids and run query
+ self.batch(ids=ids)
+ self.cursor.execute(FileDB.SELECT_IDS)
+
+ # Format and return results
+ return self.cursor.fetchall()
+
+ def resolve(self, name, alias=None):
+ # Standard column names
+ sections = ["indexid", "id", "tags", "entry"]
+ noprefix = ["data", "object", "score", "text"]
+
+ # Alias expression
+ if alias:
+ # Skip if name matches alias or alias is a standard column name
+ if name == alias or alias in sections:
+ return name
+
+ # Build alias clause
+ return f'{name} as "{alias}"'
+
+ # Resolve expression
+ if self.expressions and name in self.expressions:
+ return self.expressions[name]
+
+ # Name is already resolved, skip
+ if name.startswith("json_extract(data") or any(f"s.{s}" == name for s in sections):
+ return name
+
+ # Standard columns - need prefixes
+ if name.lower() in sections:
+ return f"s.{name}"
+
+ # Standard columns - no prefixes
+ if name.lower() in noprefix:
+ return name
+
+ # Other columns come from documents.data JSON
+ return f"json_extract(data, '$.{name}')"
+
+ def embed(self, similarity, batch):
+ # Load similarity results id batch
+ self.batch(indexids=[i for i, _ in similarity[batch]], batch=batch)
+
+ # Average and load all similarity scores with first batch
+ if not batch:
+ self.scores(similarity)
+
+ # Return ids clause placeholder
+ return FileDB.IDS_CLAUSE % batch
+
+ # pylint: disable=R0912
+ def query(self, query, limit):
+ # Extract query components
+ select = query.get("select", self.defaults())
+ where = query.get("where")
+ groupby, having = query.get("groupby"), query.get("having")
+ orderby, qlimit, offset = query.get("orderby"), query.get("limit"), query.get("offset")
+ similarity = query.get("similar")
+
+ # Build query text
+ query = FileDB.TABLE_CLAUSE % select
+ if where is not None:
+ query += f" WHERE {where}"
+ if groupby is not None:
+ query += f" GROUP BY {groupby}"
+ if having is not None:
+ query += f" HAVING {having}"
+ if orderby is not None:
+ query += f" ORDER BY {orderby}"
+
+ # Default ORDER BY if not provided and similarity scores are available
+ if similarity and orderby is None:
+ query += " ORDER BY score DESC"
+
+ # Apply query limit
+ if qlimit is not None or limit:
+ query += f" LIMIT {qlimit if qlimit else limit}"
+
+ # Apply offset
+ if offset is not None:
+ query += f" OFFSET {offset}"
+
+ # Clear scores when no similar clauses present
+ if not similarity:
+ self.scores(None)
+
+ # Runs a user query through execute method, which has common user query handling logic
+ self.execute(self.cursor.execute, query)
+
+ # Retrieve column list from query
+ columns = [c[0] for c in self.cursor.description]
+
+ # Map results and return
+ results = []
+ for row in self.rows():
+ result = {}
+
+ # Copy columns to result. In cases with duplicate column names, find one with a value
+ for x, column in enumerate(columns):
+ if column not in result or result[column] is None:
+ # Decode object
+ if self.encoder and column == "object":
+ result[column] = self.encoder.decode(row[x])
+ else:
+ result[column] = row[x]
+
+ results.append(result)
+
+ return results
+
+ def initialize(self):
+ """
+ Creates connection and initial database schema if no connection exists.
+ """
+
+ if not self.connection:
+ # Create temporary database. Thread locking must be handled externally.
+ self.connection = self.connect()
+ self.cursor = self.getcursor()
+
+ # Register custom functions
+ self.addfunctions()
+
+ # Create initial schema and indices
+ self.cursor.execute(FileDB.CREATE_DOCUMENTS)
+ self.cursor.execute(FileDB.CREATE_OBJECTS)
+ self.cursor.execute(FileDB.CREATE_SECTIONS % "sections")
+ self.cursor.execute(FileDB.CREATE_SECTIONS_INDEX)
+
+ def insertdocument(self, uid, document, tags, entry):
+ """
+ Inserts a document.
+
+ Args:
+ uid: unique id
+ document: input document
+ tags: document tags
+ entry: generated entry date
+
+ Returns:
+ section value
+ """
+
+ # Make a copy of document before changing
+ document = document.copy()
+
+ # Get and remove object field from document
+ obj = document.pop("object") if "object" in document else None
+
+ # Insert document as JSON
+ if document:
+ self.cursor.execute(FileDB.INSERT_DOCUMENT, [uid, json.dumps(document, allow_nan=False), tags, entry])
+
+ # Get value of text field
+ text = document.get("text")
+
+ # If both text and object are set, insert object as it won't otherwise be used
+ if text and obj:
+ self.insertobject(uid, obj, tags, entry)
+
+ # Return value to use for section - use text if available otherwise use object
+ return text if text else obj
+
+ def insertobject(self, uid, obj, tags, entry):
+ """
+ Inserts an object.
+
+ Args:
+ uid: unique id
+ obj: input object
+ tags: object tags
+ entry: generated entry date
+ """
+
+ # If object support is enabled, save object
+ if self.encoder:
+ self.cursor.execute(FileDB.INSERT_OBJECT, [uid, self.encoder.encode(obj), tags, entry])
+
+ def insertsection(self, index, uid, text, tags, entry):
+ """
+ Inserts a section.
+
+ Args:
+ index: index id
+ uid: unique id
+ text: section text
+ tags: section tags
+ entry: generated entry date
+ """
+
+ # Save text section
+ self.cursor.execute(FileDB.INSERT_SECTION, [index, uid, text, tags, entry])
+
+ def defaults(self):
+ """
+ Returns a list of default columns when there is no select clause.
+
+ Returns:
+ list of default columns
+ """
+
+ return "s.id, text, score"
+
+ def batch(self, indexids=None, ids=None, batch=None):
+ """
+ Loads ids to a temporary batch table for efficient query processing.
+
+ Args:
+ indexids: list of indexids
+ ids: list of ids
+ batch: batch index, used when statement has multiple subselects
+ """
+
+ # Create or Replace temporary batch table
+ self.cursor.execute(FileDB.CREATE_BATCH)
+
+ # Delete batch when batch id is empty or for batch 0
+ if not batch:
+ self.cursor.execute(FileDB.DELETE_BATCH)
+
+ if indexids:
+ self.cursor.executemany(FileDB.INSERT_BATCH_INDEXID, [(i, batch) for i in indexids])
+ if ids:
+ self.cursor.executemany(FileDB.INSERT_BATCH_ID, [(str(uid), batch) for uid in ids])
+
+ def scores(self, similarity):
+ """
+ Loads a batch of similarity scores to a temporary table for efficient query processing.
+
+ Args:
+ similarity: similarity results as [(indexid, score)]
+ """
+
+ # Create or Replace temporary scores table
+ self.cursor.execute(FileDB.CREATE_SCORES)
+
+ # Delete scores
+ self.cursor.execute(FileDB.DELETE_SCORES)
+
+ if similarity:
+ # Average scores per id, needed for multiple similar() clauses
+ scores = {}
+ for s in similarity:
+ for i, score in s:
+ if i not in scores:
+ scores[i] = []
+ scores[i].append(score)
+
+ # Average scores by id
+ self.cursor.executemany(FileDB.INSERT_SCORE, [(i, sum(s) / len(s)) for i, s in scores.items()])
+
+ def connect(self, path=None):
+ """
+ Creates a new database connection.
+
+ Args:
+ path: path to database file
+
+ Returns:
+ connection
+ """
+
+ raise NotImplementedError
+
+ def getcursor(self):
+ """
+ Opens a cursor for current connection.
+
+ Returns:
+ cursor
+ """
+
+ raise NotImplementedError
+
+ def rows(self):
+ """
+ Returns current cursor row iterator for last executed query.
+
+ Args:
+ cursor: cursor
+
+ Returns:
+ iterable collection of rows
+ """
+
+ raise NotImplementedError
+
+ def addfunctions(self):
+ """
+ Adds custom functions in current connection.
+ """
+
+ raise NotImplementedError
+
+ def copy(self, path):
+ """
+ Copies the current database into path.
+
+ Args:
+ path: path to write database
+
+ Returns:
+ new connection with data copied over
+ """
+
+ raise NotImplementedError
diff --git a/src/python/txtai/database/sqlite.py b/src/python/txtai/database/sqlite.py
index 4fd7e497..438acf5c 100644
--- a/src/python/txtai/database/sqlite.py
+++ b/src/python/txtai/database/sqlite.py
@@ -2,344 +2,27 @@
SQLite module
"""
-import datetime
-import json
import os
import sqlite3
-from .base import Database
+from .filedb import FileDB
-class SQLite(Database):
+class SQLite(FileDB):
"""
- Document database backed by SQLite.
+ Database instance backed by SQLite.
"""
- # Temporary table for working with id batches
- CREATE_BATCH = """
- CREATE TEMP TABLE IF NOT EXISTS batch (
- indexid INTEGER,
- id TEXT,
- batch INTEGER
- )
- """
-
- DELETE_BATCH = "DELETE FROM batch"
- INSERT_BATCH_INDEXID = "INSERT INTO batch (indexid, batch) VALUES (?, ?)"
- INSERT_BATCH_ID = "INSERT INTO batch (id, batch) VALUES (?, ?)"
-
- # Temporary table for joining similarity scores
- CREATE_SCORES = """
- CREATE TEMP TABLE IF NOT EXISTS scores (
- indexid INTEGER PRIMARY KEY,
- score REAL
- )
- """
-
- DELETE_SCORES = "DELETE FROM scores"
- INSERT_SCORE = "INSERT INTO scores VALUES (?, ?)"
-
- # Documents - stores full content
- CREATE_DOCUMENTS = """
- CREATE TABLE IF NOT EXISTS documents (
- id TEXT PRIMARY KEY,
- data JSON,
- tags TEXT,
- entry DATETIME
- )
- """
-
- INSERT_DOCUMENT = "REPLACE INTO documents VALUES (?, ?, ?, ?)"
- DELETE_DOCUMENTS = "DELETE FROM documents WHERE id IN (SELECT id FROM batch)"
-
- # Objects - stores binary content
- CREATE_OBJECTS = """
- CREATE TABLE IF NOT EXISTS objects (
- id TEXT PRIMARY KEY,
- object BLOB,
- tags TEXT,
- entry DATETIME
- )
- """
-
- INSERT_OBJECT = "REPLACE INTO objects VALUES (?, ?, ?, ?)"
- DELETE_OBJECTS = "DELETE FROM objects WHERE id IN (SELECT id FROM batch)"
-
- # Sections - stores section text
- CREATE_SECTIONS = """
- CREATE TABLE IF NOT EXISTS %s (
- indexid INTEGER PRIMARY KEY,
- id TEXT,
- text TEXT,
- tags TEXT,
- entry DATETIME
- )
- """
-
- CREATE_SECTIONS_INDEX = "CREATE INDEX section_id ON sections(id)"
- INSERT_SECTION = "INSERT INTO sections VALUES (?, ?, ?, ?, ?)"
- DELETE_SECTIONS = "DELETE FROM sections WHERE id IN (SELECT id FROM batch)"
- COPY_SECTIONS = (
- "INSERT INTO %s SELECT (select count(*) - 1 from sections s1 where s.indexid >= s1.indexid) indexid, "
- + "s.id, %s AS text, s.tags, s.entry FROM sections s LEFT JOIN documents d ON s.id = d.id ORDER BY indexid"
- )
- STREAM_SECTIONS = "SELECT s.id, s.text, object, s.tags FROM %s s LEFT JOIN objects o ON s.id = o.id ORDER BY indexid"
- DROP_SECTIONS = "DROP TABLE sections"
- RENAME_SECTIONS = "ALTER TABLE %s RENAME TO sections"
-
- # Queries
- SELECT_IDS = "SELECT indexid, id FROM sections WHERE id in (SELECT id FROM batch)"
-
- # Partial sql clauses
- TABLE_CLAUSE = (
- "SELECT %s FROM sections s "
- + "LEFT JOIN documents d ON s.id = d.id "
- + "LEFT JOIN objects o ON s.id = o.id "
- + "LEFT JOIN scores sc ON s.indexid = sc.indexid"
- )
- IDS_CLAUSE = "s.indexid in (SELECT indexid from batch WHERE batch=%s)"
-
- def __init__(self, config):
- """
- Creates a new Database.
-
- Args:
- config: database configuration parameters
- """
-
- super().__init__(config)
-
- # SQLite connection handle
- self.connection = None
- self.cursor = None
- self.path = None
-
- def load(self, path):
- # Load an existing database. Thread locking must be handled externally.
- self.connection = sqlite3.connect(path, check_same_thread=False)
- self.cursor = self.connection.cursor()
- self.path = path
-
- # Register custom functions
- self.addfunctions()
-
- def insert(self, documents, index=0):
- # Initialize connection if not open
- self.initialize()
-
- # Get entry date
- entry = datetime.datetime.now()
-
- # Insert documents
- for uid, document, tags in documents:
- if isinstance(document, dict):
- # Insert document and use return value for sections table
- document = self.insertdocument(uid, document, tags, entry)
-
- if document is not None:
- if isinstance(document, list):
- # Join tokens to text
- document = " ".join(document)
- elif not isinstance(document, str):
- # If object support is enabled, save object
- self.insertobject(uid, document, tags, entry)
-
- # Clear section text for objects, even when objects aren't inserted
- document = None
-
- # Save text section
- self.insertsection(index, uid, document, tags, entry)
- index += 1
-
- def delete(self, ids):
- if self.connection:
- # Batch ids
- self.batch(ids=ids)
-
- # Delete all documents, objects and sections by id
- self.cursor.execute(SQLite.DELETE_DOCUMENTS)
- self.cursor.execute(SQLite.DELETE_OBJECTS)
- self.cursor.execute(SQLite.DELETE_SECTIONS)
-
- def reindex(self, columns=None):
- if self.connection:
- # Working table name
- name = "rebuild"
-
- # Resolve and build column strings if provided
- select = "text"
- if columns:
- select = "|| ' ' ||".join([self.resolve(c) for c in columns])
-
- # Create new table to hold reordered sections
- self.cursor.execute(SQLite.CREATE_SECTIONS % name)
-
- # Copy data over
- self.cursor.execute(SQLite.COPY_SECTIONS % (name, select))
-
- # Stream new results
- self.cursor.execute(SQLite.STREAM_SECTIONS % name)
- for uid, text, obj, tags in self.cursor:
- if not text and self.encoder and obj:
- yield (uid, self.encoder.decode(obj), tags)
- else:
- yield (uid, text, tags)
-
- # Swap as new table
- self.cursor.execute(SQLite.DROP_SECTIONS)
- self.cursor.execute(SQLite.RENAME_SECTIONS % name)
- self.cursor.execute(SQLite.CREATE_SECTIONS_INDEX)
-
- def save(self, path):
- # Temporary database
- if not self.path:
- # Save temporary database
- self.connection.commit()
-
- # Copy data from current to new
- connection = self.copy(path)
-
- # Close temporary database
- self.connection.close()
-
- # Point connection to new connection
- self.connection = connection
- self.cursor = self.connection.cursor()
- self.path = path
-
- # Register custom functions
- self.addfunctions()
-
- # Paths are equal, commit changes
- elif self.path == path:
- self.connection.commit()
-
- # New path is different from current path, copy data and continue using current connection
- else:
- self.copy(path).close()
-
- def close(self):
- # Close connection
- if self.connection:
- self.connection.close()
-
- def ids(self, ids):
- # Batch ids and run query
- self.batch(ids=ids)
- self.cursor.execute(SQLite.SELECT_IDS)
-
- # Format and return results
- return self.cursor.fetchall()
-
- def resolve(self, name, alias=None):
- # Standard column names
- sections = ["indexid", "id", "tags", "entry"]
- noprefix = ["data", "object", "score", "text"]
-
- # Alias expression
- if alias:
- # Skip if name matches alias or alias is a standard column name
- if name == alias or alias in sections:
- return name
-
- # Build alias clause
- return f'{name} as "{alias}"'
-
- # Resolve expression
- if self.expressions and name in self.expressions:
- return self.expressions[name]
-
- # Name is already resolved, skip
- if name.startswith("json_extract(data") or any(f"s.{s}" == name for s in sections):
- return name
-
- # Standard columns - need prefixes
- if name.lower() in sections:
- return f"s.{name}"
+ def connect(self, path=""):
+ return sqlite3.connect(path, check_same_thread=False)
- # Standard columns - no prefixes
- if name.lower() in noprefix:
- return name
+ def getcursor(self):
+ return self.connection.cursor()
- # Other columns come from documents.data JSON
- return f'json_extract(data, "$.{name}")'
-
- def embed(self, similarity, batch):
- # Load similarity results id batch
- self.batch(indexids=[i for i, _ in similarity[batch]], batch=batch)
-
- # Average and load all similarity scores with first batch
- if not batch:
- self.scores(similarity)
-
- # Return ids clause placeholder
- return SQLite.IDS_CLAUSE % batch
-
- # pylint: disable=R0912
- def query(self, query, limit):
- # Extract query components
- select = query.get("select", self.defaults())
- where = query.get("where")
- groupby, having = query.get("groupby"), query.get("having")
- orderby, qlimit, offset = query.get("orderby"), query.get("limit"), query.get("offset")
- similarity = query.get("similar")
-
- # Build query text
- query = SQLite.TABLE_CLAUSE % select
- if where is not None:
- query += f" WHERE {where}"
- if groupby is not None:
- query += f" GROUP BY {groupby}"
- if having is not None:
- query += f" HAVING {having}"
- if orderby is not None:
- query += f" ORDER BY {orderby}"
-
- # Default ORDER BY if not provided and similarity scores are available
- if similarity and orderby is None:
- query += " ORDER BY score DESC"
-
- # Apply query limit
- if qlimit is not None or limit:
- query += f" LIMIT {qlimit if qlimit else limit}"
-
- # Apply offset
- if offset is not None:
- query += f" OFFSET {offset}"
-
- # Clear scores when no similar clauses present
- if not similarity:
- self.scores(None)
-
- # Runs a user query through execute method, which has common user query handling logic
- self.execute(self.cursor.execute, query)
-
- # Retrieve column list from query
- columns = [c[0] for c in self.cursor.description]
-
- # Map results and return
- results = []
- for row in self.cursor:
- result = {}
-
- # Copy columns to result. In cases with duplicate column names, find one with a value
- for x, column in enumerate(columns):
- if column not in result or result[column] is None:
- # Decode object
- if self.encoder and column == "object":
- result[column] = self.encoder.decode(row[x])
- else:
- result[column] = row[x]
-
- results.append(result)
-
- return results
+ def rows(self):
+ return self.cursor
def addfunctions(self):
- """
- Adds custom functions in current connection.
- """
-
if self.connection and self.functions:
# Enable callback tracebacks to show user-defined function errors
sqlite3.enable_callback_tracebacks(True)
@@ -347,109 +30,13 @@ class SQLite(Database):
for name, argcount, fn in self.functions:
self.connection.create_function(name, argcount, fn)
- def initialize(self):
- """
- Creates connection and initial database schema if no connection exists.
- """
-
- if not self.connection:
- name = "sections"
-
- # Create temporary database. Thread locking must be handled externally.
- self.connection = sqlite3.connect("", check_same_thread=False)
- self.cursor = self.connection.cursor()
-
- # Register custom functions
- self.addfunctions()
-
- # Create initial schema and indices
- self.cursor.execute(SQLite.CREATE_DOCUMENTS)
- self.cursor.execute(SQLite.CREATE_OBJECTS)
- self.cursor.execute(SQLite.CREATE_SECTIONS % name)
- self.cursor.execute(SQLite.CREATE_SECTIONS_INDEX)
-
- def insertdocument(self, uid, document, tags, entry):
- """
- Inserts a document.
-
- Args:
- uid: unique id
- document: input document
- tags: document tags
- entry: generated entry date
-
- Returns:
- section value
- """
-
- # Make a copy of document before changing
- document = document.copy()
-
- # Get and remove object field from document
- obj = document.pop("object") if "object" in document else None
-
- # Insert document as JSON
- if document:
- self.cursor.execute(SQLite.INSERT_DOCUMENT, [uid, json.dumps(document, allow_nan=False), tags, entry])
-
- # Get value of text field
- text = document.get("text")
-
- # If both text and object are set, insert object as it won't otherwise be used
- if text and obj:
- self.insertobject(uid, obj, tags, entry)
-
- # Return value to use for section - use text if available otherwise use object
- return text if text else obj
-
- def insertobject(self, uid, obj, tags, entry):
- """
- Inserts an object.
-
- Args:
- uid: unique id
- obj: input object
- tags: object tags
- entry: generated entry date
- """
-
- # If object support is enabled, save object
- if self.encoder:
- self.cursor.execute(SQLite.INSERT_OBJECT, [uid, self.encoder.encode(obj), tags, entry])
-
- def insertsection(self, index, uid, text, tags, entry):
- """
- Inserts a section.
-
- Args:
- index: index id
- uid: unique id
- text: section text
- tags: section tags
- entry: generated entry date
- """
-
- # Save text section
- self.cursor.execute(SQLite.INSERT_SECTION, [index, uid, text, tags, entry])
-
def copy(self, path):
- """
- Copies the current database into path. This method will use the backup API if the current connection has no uncommitted changes.
- Otherwise, iterdump is used, as the backup call will hang for an uncommitted connection.
-
- Args:
- path: path to write database
-
- Returns:
- new connection with data copied over
- """
-
# Delete existing file, if necessary
if os.path.exists(path):
os.remove(path)
# Create database. Thread locking must be handled externally.
- connection = sqlite3.connect(path, check_same_thread=False)
+ connection = self.connect(path)
if self.connection.in_transaction:
# The backup call will hang if there are uncommitted changes, need to copy over
@@ -461,61 +48,3 @@ class SQLite(Database):
self.connection.backup(connection)
return connection
-
- def defaults(self):
- """
- Returns a list of default columns when there is no select clause.
-
- Returns:
- list of default columns
- """
-
- return "s.id, text, score"
-
- def batch(self, indexids=None, ids=None, batch=None):
- """
- Loads ids to a temporary batch table for efficient query processing.
-
- Args:
- indexids: list of indexids
- ids: list of ids
- batch: batch index, used when statement has multiple subselects
- """
-
- # Create or Replace temporary batch table
- self.cursor.execute(SQLite.CREATE_BATCH)
-
- # Delete batch when batch id is empty or for batch 0
- if not batch:
- self.cursor.execute(SQLite.DELETE_BATCH)
-
- if indexids:
- self.cursor.executemany(SQLite.INSERT_BATCH_INDEXID, [(i, batch) for i in indexids])
- if ids:
- self.cursor.executemany(SQLite.INSERT_BATCH_ID, [(str(uid), batch) for uid in ids])
-
- def scores(self, similarity):
- """
- Loads a batch of similarity scores to a temporary table for efficient query processing.
-
- Args:
- similarity: similarity results as [(indexid, score)]
- """
-
- # Create or Replace temporary scores table
- self.cursor.execute(SQLite.CREATE_SCORES)
-
- # Delete scores
- self.cursor.execute(SQLite.DELETE_SCORES)
-
- if similarity:
- # Average scores per id, needed for multiple similar() clauses
- scores = {}
- for s in similarity:
- for i, score in s:
- if i not in scores:
- scores[i] = []
- scores[i].append(score)
-
- # Average scores by id
- self.cursor.executemany(SQLite.INSERT_SCORE, [(i, sum(s) / len(s)) for i, s in scores.items()])
diff --git a/test/python/testdatabase/testduckdb.py b/test/python/testdatabase/testduckdb.py
new file mode 100644
index 00000000..507dc01a
--- /dev/null
+++ b/test/python/testdatabase/testduckdb.py
@@ -0,0 +1,44 @@
+"""
+Embeddings+database module tests
+"""
+
+from txtai.embeddings import Embeddings
+
+from .testsqlite import TestSQLite
+
+
+# pylint: disable=R0904
+class TestDuckDB(TestSQLite):
+ """
+ Embeddings with content stored in DuckDB.
+ """
+
+ @classmethod
+ def setUpClass(cls):
+ """
+ Initialize test data.
+ """
+
+ cls.data = [
+ "US tops 5 million confirmed virus cases",
+ "Canada's last fully intact ice shelf has suddenly collapsed, forming a Manhattan-sized iceberg",
+ "Beijing mobilises invasion craft along coast as Taiwan tensions escalate",
+ "The National Park Service warns against sacrificing slower friends in a bear attack",
+ "Maine man wins $1M from $25 lottery ticket",
+ "Make huge profits without work, earn up to $100,000 a day",
+ ]
+
+ # Content backend
+ cls.backend = "duckdb"
+
+ # Create embeddings model, backed by sentence-transformers & transformers
+ cls.embeddings = Embeddings({"path": "sentence-transformers/nli-mpnet-base-v2", "content": cls.backend})
+
+ @classmethod
+ def tearDownClass(cls):
+ """
+ Cleanup data.
+ """
+
+ if cls.embeddings:
+ cls.embeddings.close()
diff --git a/test/python/testdatabase/testfiledb.py b/test/python/testdatabase/testfiledb.py
new file mode 100644
index 00000000..d01865a0
--- /dev/null
+++ b/test/python/testdatabase/testfiledb.py
@@ -0,0 +1,26 @@
+"""
+FileDB module tests
+"""
+
+import unittest
+
+from txtai.database import FileDB
+
+
+class TestFileDB(unittest.TestCase):
+ """
+ Test base file database methods
+ """
+
+ def testNotImplemented(self):
+ """
+ Test exceptions for non-implemented methods
+ """
+
+ db = FileDB({})
+
+ self.assertRaises(NotImplementedError, db.connect, None)
+ self.assertRaises(NotImplementedError, db.getcursor)
+ self.assertRaises(NotImplementedError, db.rows)
+ self.assertRaises(NotImplementedError, db.addfunctions)
+ self.assertRaises(NotImplementedError, db.copy, None)
diff --git a/test/python/testdatabase/testsql.py b/test/python/testdatabase/testsql.py
index 2c396703..c46ae2a3 100644
--- a/test/python/testdatabase/testsql.py
+++ b/test/python/testdatabase/testsql.py
@@ -29,20 +29,20 @@ class TestSQL(unittest.TestCase):
Test alias clauses
"""
- self.assertSql("select", "select a as a1 from txtai", 'json_extract(data, "$.a") as a1')
- self.assertSql("select", "select a 'a1' from txtai", "json_extract(data, \"$.a\") 'a1'")
- self.assertSql("select", 'select a "a1" from txtai', 'json_extract(data, "$.a") "a1"')
- self.assertSql("select", "select a a1 from txtai", 'json_extract(data, "$.a") a1')
+ self.assertSql("select", "select a as a1 from txtai", "json_extract(data, '$.a') as a1")
+ self.assertSql("select", "select a 'a1' from txtai", "json_extract(data, '$.a') 'a1'")
+ self.assertSql("select", 'select a "a1" from txtai', "json_extract(data, '$.a') \"a1\"")
+ self.assertSql("select", "select a a1 from txtai", "json_extract(data, '$.a') a1")
self.assertSql(
"select",
"select a, b as b1, c, d + 1 as 'd1' from txtai",
- 'json_extract(data, "$.a") as "a", json_extract(data, "$.b") as b1, '
- + 'json_extract(data, "$.c") as "c", json_extract(data, "$.d") + 1 as \'d1\'',
+ "json_extract(data, '$.a') as \"a\", json_extract(data, '$.b') as b1, "
+ + "json_extract(data, '$.c') as \"c\", json_extract(data, '$.d') + 1 as 'd1'",
)
self.assertSql("select", "select id as myid from txtai", "s.id as myid")
- self.assertSql("select", "select length(a) t from txtai", 'length(json_extract(data, "$.a")) t')
+ self.assertSql("select", "select length(a) t from txtai", "length(json_extract(data, '$.a')) t")
- self.assertSql("where", "select id as myid from txtai where myid != 3 and a != 1", 'myid != 3 and json_extract(data, "$.a") != 1')
+ self.assertSql("where", "select id as myid from txtai where myid != 3 and a != 1", "myid != 3 and json_extract(data, '$.a') != 1")
self.assertSql("where", "select txt T from txtai where t LIKE '%abc%'", "t LIKE '%abc%'")
self.assertSql("where", "select txt 'T' from txtai where t LIKE '%abc%'", "t LIKE '%abc%'")
self.assertSql("where", "select txt \"T\" from txtai where t LIKE '%abc%'", "t LIKE '%abc%'")
@@ -50,8 +50,8 @@ class TestSQL(unittest.TestCase):
self.assertSql("where", "select txt as 'T' from txtai where t LIKE '%abc%'", "t LIKE '%abc%'")
self.assertSql("where", "select txt as \"T\" from txtai where t LIKE '%abc%'", "t LIKE '%abc%'")
- self.assertSql("groupby", "select id as myid, count(*) from txtai group by myid, a", 'myid, json_extract(data, "$.a")')
- self.assertSql("orderby", "select id as myid from txtai order by myid, a", 'myid, json_extract(data, "$.a")')
+ self.assertSql("groupby", "select id as myid, count(*) from txtai group by myid, a", "myid, json_extract(data, '$.a')")
+ self.assertSql("orderby", "select id as myid from txtai order by myid, a", "myid, json_extract(data, '$.a')")
def testBadSQL(self):
"""
@@ -75,20 +75,20 @@ class TestSQL(unittest.TestCase):
Test bracket expressions
"""
- self.assertSql("select", "select [a] from txtai", 'json_extract(data, "$.a") as "a"')
- self.assertSql("select", "select [a] ab from txtai", 'json_extract(data, "$.a") ab')
- self.assertSql("select", "select [abc] from txtai", 'json_extract(data, "$.abc") as "abc"')
+ self.assertSql("select", "select [a] from txtai", "json_extract(data, '$.a') as \"a\"")
+ self.assertSql("select", "select [a] ab from txtai", "json_extract(data, '$.a') ab")
+ self.assertSql("select", "select [abc] from txtai", "json_extract(data, '$.abc') as \"abc\"")
self.assertSql("select", "select [id], text, score from txtai", "s.id, text, score")
- self.assertSql("select", "select [ab cd], text, score from txtai", 'json_extract(data, "$.ab cd") as "ab cd", text, score')
- self.assertSql("select", "select [a[0]] from txtai", 'json_extract(data, "$.a[0]") as "a[0]"')
- self.assertSql("select", "select [a[0].ab] from txtai", 'json_extract(data, "$.a[0].ab") as "a[0].ab"')
- self.assertSql("select", "select [a[0].c[0]] from txtai", 'json_extract(data, "$.a[0].c[0]") as "a[0].c[0]"')
- self.assertSql("select", "select avg([a]) from txtai", 'avg(json_extract(data, "$.a")) as "avg([a])"')
+ self.assertSql("select", "select [ab cd], text, score from txtai", "json_extract(data, '$.ab cd') as \"ab cd\", text, score")
+ self.assertSql("select", "select [a[0]] from txtai", "json_extract(data, '$.a[0]') as \"a[0]\"")
+ self.assertSql("select", "select [a[0].ab] from txtai", "json_extract(data, '$.a[0].ab') as \"a[0].ab\"")
+ self.assertSql("select", "select [a[0].c[0]] from txtai", "json_extract(data, '$.a[0].c[0]') as \"a[0].c[0]\"")
+ self.assertSql("select", "select avg([a]) from txtai", "avg(json_extract(data, '$.a')) as \"avg([a])\"")
- self.assertSql("where", "select * from txtai where [a b] < 1 or a > 1", 'json_extract(data, "$.a b") < 1 or json_extract(data, "$.a") > 1')
+ self.assertSql("where", "select * from txtai where [a b] < 1 or a > 1", "json_extract(data, '$.a b') < 1 or json_extract(data, '$.a') > 1")
self.assertSql("where", "select [a[0].c[0]] a from txtai where a < 1", "a < 1")
- self.assertSql("groupby", "select * from txtai group by [a]", 'json_extract(data, "$.a")')
- self.assertSql("orderby", "select * from txtai where order by [a]", 'json_extract(data, "$.a")')
+ self.assertSql("groupby", "select * from txtai group by [a]", "json_extract(data, '$.a')")
+ self.assertSql("orderby", "select * from txtai where order by [a]", "json_extract(data, '$.a')")
def testDistinct(self):
"""
@@ -98,24 +98,24 @@ class TestSQL(unittest.TestCase):
# Attributes
self.assertSql("select", "select distinct id from txtai", "distinct s.id")
self.assertSql("select", "select distinct id as myid from txtai", "distinct s.id as myid")
- self.assertSql("select", "select distinct a from txtai", 'distinct json_extract(data, "$.a") as "a"')
- self.assertSql("select", "select distinct a.b from txtai", 'distinct json_extract(data, "$.a.b") as "a.b"')
+ self.assertSql("select", "select distinct a from txtai", "distinct json_extract(data, '$.a') as \"a\"")
+ self.assertSql("select", "select distinct a.b from txtai", "distinct json_extract(data, '$.a.b') as \"a.b\"")
# Bracket expression
- self.assertSql("select", "select distinct [ab cd] from txtai", 'distinct json_extract(data, "$.ab cd") as "distinct[ab cd]"')
+ self.assertSql("select", "select distinct [ab cd] from txtai", "distinct json_extract(data, '$.ab cd') as \"distinct[ab cd]\"")
# Function expression
self.assertSql("select", "select distinct(id) from txtai", 'distinct(s.id) as "distinct(id)"')
self.assertSql("select", "select count(distinct id) from txtai", 'count(distinct s.id) as "count(distinct id)"')
- self.assertSql("select", "select count(distinct a) from txtai", 'count(distinct json_extract(data, "$.a")) as "count(distinct a)"')
+ self.assertSql("select", "select count(distinct a) from txtai", "count(distinct json_extract(data, '$.a')) as \"count(distinct a)\"")
self.assertSql("select", "select count(distinct avg(id)) from txtai", 'count(distinct avg(s.id)) as "count(distinct avg(id))"')
self.assertSql(
- "select", "select count(distinct avg(a)) from txtai", 'count(distinct avg(json_extract(data, "$.a"))) as "count(distinct avg(a))"'
+ "select", "select count(distinct avg(a)) from txtai", "count(distinct avg(json_extract(data, '$.a'))) as \"count(distinct avg(a))\""
)
# Compound expression
- self.assertSql("select", "select distinct a/1 from txtai", 'distinct json_extract(data, "$.a") / 1 as "a / 1"')
- self.assertSql("select", "select distinct(a/1) from txtai", 'distinct(json_extract(data, "$.a") / 1) as "distinct(a / 1)"')
+ self.assertSql("select", "select distinct a/1 from txtai", "distinct json_extract(data, '$.a') / 1 as \"a / 1\"")
+ self.assertSql("select", "select distinct(a/1) from txtai", "distinct(json_extract(data, '$.a') / 1) as \"distinct(a / 1)\"")
def testGroupby(self):
"""
@@ -125,7 +125,7 @@ class TestSQL(unittest.TestCase):
prefix = "select count(*), flag from txtai "
self.assertSql("groupby", prefix + "group by text", "text")
- self.assertSql("groupby", prefix + "group by distinct(a)", 'distinct(json_extract(data, "$.a"))')
+ self.assertSql("groupby", prefix + "group by distinct(a)", "distinct(json_extract(data, '$.a'))")
self.assertSql("groupby", prefix + "where a > 1 group by text", "text")
def testHaving(self):
@@ -184,8 +184,8 @@ class TestSQL(unittest.TestCase):
"""
self.assertSql("select", "select id, indexid, tags from txtai", "s.id, s.indexid, s.tags")
- self.assertSql("select", "select id, indexid, flag from txtai", 's.id, s.indexid, json_extract(data, "$.flag") as "flag"')
- self.assertSql("select", "select id, indexid, a.b.c from txtai", 's.id, s.indexid, json_extract(data, "$.a.b.c") as "a.b.c"')
+ self.assertSql("select", "select id, indexid, flag from txtai", "s.id, s.indexid, json_extract(data, '$.flag') as \"flag\"")
+ self.assertSql("select", "select id, indexid, a.b.c from txtai", "s.id, s.indexid, json_extract(data, '$.a.b.c') as \"a.b.c\"")
self.assertSql("select", "select 'id', [id], (id) from txtai", "'id', s.id, (s.id)")
self.assertSql("select", "select * from txtai", "*")
@@ -194,22 +194,22 @@ class TestSQL(unittest.TestCase):
Test compound select clauses
"""
- self.assertSql("select", "select a + 1 from txtai", 'json_extract(data, "$.a") + 1 as "a + 1"')
- self.assertSql("select", "select 1 * a from txtai", '1 * json_extract(data, "$.a") as "1 * a"')
- self.assertSql("select", "select a/1 from txtai", 'json_extract(data, "$.a") / 1 as "a / 1"')
- self.assertSql("select", "select avg(a-b) from txtai", 'avg(json_extract(data, "$.a") - json_extract(data, "$.b")) as "avg(a - b)"')
+ self.assertSql("select", "select a + 1 from txtai", "json_extract(data, '$.a') + 1 as \"a + 1\"")
+ self.assertSql("select", "select 1 * a from txtai", "1 * json_extract(data, '$.a') as \"1 * a\"")
+ self.assertSql("select", "select a/1 from txtai", "json_extract(data, '$.a') / 1 as \"a / 1\"")
+ self.assertSql("select", "select avg(a-b) from txtai", "avg(json_extract(data, '$.a') - json_extract(data, '$.b')) as \"avg(a - b)\"")
self.assertSql("select", "select distinct(text) from txtai", "distinct(text)")
- self.assertSql("select", "select id, score, (a/2)*3 from txtai", 's.id, score, (json_extract(data, "$.a") / 2) * 3 as "(a / 2) * 3"')
- self.assertSql("select", "select id, score, (a/2*3) from txtai", 's.id, score, (json_extract(data, "$.a") / 2 * 3) as "(a / 2 * 3)"')
+ self.assertSql("select", "select id, score, (a/2)*3 from txtai", "s.id, score, (json_extract(data, '$.a') / 2) * 3 as \"(a / 2) * 3\"")
+ self.assertSql("select", "select id, score, (a/2*3) from txtai", "s.id, score, (json_extract(data, '$.a') / 2 * 3) as \"(a / 2 * 3)\"")
self.assertSql(
"select",
"select func(func2(indexid + 1), a) from txtai",
- 'func(func2(s.indexid + 1), json_extract(data, "$.a")) as "func(func2(indexid + 1), a)"',
+ "func(func2(s.indexid + 1), json_extract(data, '$.a')) as \"func(func2(indexid + 1), a)\"",
)
- self.assertSql("select", "select func(func2(indexid + 1), a) a from txtai", 'func(func2(s.indexid + 1), json_extract(data, "$.a")) a')
+ self.assertSql("select", "select func(func2(indexid + 1), a) a from txtai", "func(func2(s.indexid + 1), json_extract(data, '$.a')) a")
self.assertSql("select", "select 'prefix' || id from txtai", "'prefix' || s.id as \"'prefix' || id\"")
self.assertSql("select", "select 'prefix' || id id from txtai", "'prefix' || s.id id")
- self.assertSql("select", "select 'prefix' || a a from txtai", "'prefix' || json_extract(data, \"$.a\") a")
+ self.assertSql("select", "select 'prefix' || a a from txtai", "'prefix' || json_extract(data, '$.a') a")
def testSimilar(self):
"""
@@ -251,15 +251,15 @@ class TestSQL(unittest.TestCase):
prefix = "select * from txtai "
- self.assertSql("where", prefix + "where a = b", 'json_extract(data, "$.a") = json_extract(data, "$.b")')
- self.assertSql("where", prefix + "where abc = def", 'json_extract(data, "$.abc") = json_extract(data, "$.def")')
- self.assertSql("where", prefix + "where a = b.value", 'json_extract(data, "$.a") = json_extract(data, "$.b.value")')
- self.assertSql("where", prefix + "where a = 1", 'json_extract(data, "$.a") = 1')
- self.assertSql("where", prefix + "WHERE 1 = a", '1 = json_extract(data, "$.a")')
- self.assertSql("where", prefix + "WHERE a LIKE 'abc'", "json_extract(data, \"$.a\") LIKE 'abc'")
- self.assertSql("where", prefix + "WHERE a NOT LIKE 'abc'", "json_extract(data, \"$.a\") NOT LIKE 'abc'")
- self.assertSql("where", prefix + "WHERE a IN (1, 2, 3, b)", 'json_extract(data, "$.a") IN (1, 2, 3, json_extract(data, "$.b"))')
- self.assertSql("where", prefix + "WHERE a is not null", 'json_extract(data, "$.a") is not null')
+ self.assertSql("where", prefix + "where a = b", "json_extract(data, '$.a') = json_extract(data, '$.b')")
+ self.assertSql("where", prefix + "where abc = def", "json_extract(data, '$.abc') = json_extract(data, '$.def')")
+ self.assertSql("where", prefix + "where a = b.value", "json_extract(data, '$.a') = json_extract(data, '$.b.value')")
+ self.assertSql("where", prefix + "where a = 1", "json_extract(data, '$.a') = 1")
+ self.assertSql("where", prefix + "WHERE 1 = a", "1 = json_extract(data, '$.a')")
+ self.assertSql("where", prefix + "WHERE a LIKE 'abc'", "json_extract(data, '$.a') LIKE 'abc'")
+ self.assertSql("where", prefix + "WHERE a NOT LIKE 'abc'", "json_extract(data, '$.a') NOT LIKE 'abc'")
+ self.assertSql("where", prefix + "WHERE a IN (1, 2, 3, b)", "json_extract(data, '$.a') IN (1, 2, 3, json_extract(data, '$.b'))")
+ self.assertSql("where", prefix + "WHERE a is not null", "json_extract(data, '$.a') is not null")
self.assertSql("where", prefix + "WHERE score >= 0.15", "score >= 0.15")
def testWhereCompound(self):
@@ -269,15 +269,15 @@ class TestSQL(unittest.TestCase):
prefix = "select * from txtai "
- self.assertSql("where", prefix + "where a > (b + 1)", 'json_extract(data, "$.a") > (json_extract(data, "$.b") + 1)')
- self.assertSql("where", prefix + "where a > func('abc')", "json_extract(data, \"$.a\") > func('abc')")
+ self.assertSql("where", prefix + "where a > (b + 1)", "json_extract(data, '$.a') > (json_extract(data, '$.b') + 1)")
+ self.assertSql("where", prefix + "where a > func('abc')", "json_extract(data, '$.a') > func('abc')")
self.assertSql(
- "where", prefix + "where (id = 1 or id = 2) and a like 'abc'", "(s.id = 1 or s.id = 2) and json_extract(data, \"$.a\") like 'abc'"
+ "where", prefix + "where (id = 1 or id = 2) and a like 'abc'", "(s.id = 1 or s.id = 2) and json_extract(data, '$.a') like 'abc'"
)
self.assertSql(
"where",
prefix + "where a > f(d(b, c, 1),1)",
- 'json_extract(data, "$.a") > f(d(json_extract(data, "$.b"), json_extract(data, "$.c"), 1), 1)',
+ "json_extract(data, '$.a') > f(d(json_extract(data, '$.b'), json_extract(data, '$.c'), 1), 1)",
)
self.assertSql("where", prefix + "where (id = 1 AND id = 2) OR indexid = 3", "(s.id = 1 AND s.id = 2) OR s.indexid = 3")
self.assertSql("where", prefix + "where f(id) = b(id)", "f(s.id) = b(s.id)")
diff --git a/test/python/testdatabase/testembeddings.py b/test/python/testdatabase/testsqlite.py
similarity index 91%
rename from test/python/testdatabase/testembeddings.py
rename to test/python/testdatabase/testsqlite.py
index e7664d2e..bdc2cbbd 100644
--- a/test/python/testdatabase/testembeddings.py
+++ b/test/python/testdatabase/testsqlite.py
@@ -13,9 +13,9 @@ from txtai.database import Database, SQLError
# pylint: disable=R0904
-class TestEmbeddings(unittest.TestCase):
+class TestSQLite(unittest.TestCase):
"""
- Embeddings with a database tests.
+ Embeddings with content stored in SQLite
"""
@classmethod
@@ -33,8 +33,11 @@ class TestEmbeddings(unittest.TestCase):
"Make huge profits without work, earn up to $100,000 a day",
]
+ # Content backend
+ cls.backend = "sqlite"
+
# Create embeddings model, backed by sentence-transformers & transformers
- cls.embeddings = Embeddings({"path": "sentence-transformers/nli-mpnet-base-v2", "content": True})
+ cls.embeddings = Embeddings({"path": "sentence-transformers/nli-mpnet-base-v2", "content": cls.backend})
@classmethod
def tearDownClass(cls):
@@ -55,7 +58,7 @@ class TestEmbeddings(unittest.TestCase):
self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])
# Generate temp file path
- index = os.path.join(tempfile.gettempdir(), f"embeddings.{extension}")
+ index = os.path.join(tempfile.gettempdir(), f"embeddings.{self.backend}.{extension}")
self.embeddings.save(index)
self.embeddings.load(index)
@@ -78,13 +81,13 @@ class TestEmbeddings(unittest.TestCase):
# Create index twice to test open/close and ensure resources are freed
for _ in range(2):
- embeddings = Embeddings({"path": "sentence-transformers/nli-mpnet-base-v2", "content": True})
+ embeddings = Embeddings({"path": "sentence-transformers/nli-mpnet-base-v2", "content": self.backend})
# Add record to index
embeddings.index([(0, "Close test", None)])
# Save index
- index = os.path.join(tempfile.gettempdir(), "embeddings.close")
+ index = os.path.join(tempfile.gettempdir(), f"embeddings.{self.backend}.close")
embeddings.save(index)
# Close index
@@ -131,7 +134,7 @@ class TestEmbeddings(unittest.TestCase):
"""
# Test search against empty index
- embeddings = Embeddings({"path": "sentence-transformers/nli-mpnet-base-v2", "content": True})
+ embeddings = Embeddings({"path": "sentence-transformers/nli-mpnet-base-v2", "content": self.backend})
self.assertEqual(embeddings.search("test"), [])
# Test index with no data
@@ -180,8 +183,8 @@ class TestEmbeddings(unittest.TestCase):
embeddings = Embeddings(
{
"path": "sentence-transformers/nli-mpnet-base-v2",
- "content": True,
- "functions": [{"name": "length", "function": "testdatabase.testembeddings.length"}],
+ "content": self.backend,
+ "functions": [{"name": "length", "function": "testdatabase.testsqlite.length"}],
}
)
@@ -256,7 +259,7 @@ class TestEmbeddings(unittest.TestCase):
"""
embeddings = Embeddings(
- {"path": "sentence-transformers/nli-mpnet-base-v2", "content": True, "instructions": {"query": "query: ", "data": "passage: "}}
+ {"path": "sentence-transformers/nli-mpnet-base-v2", "content": self.backend, "instructions": {"query": "query: ", "data": "passage: "}}
)
embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])
@@ -284,14 +287,14 @@ class TestEmbeddings(unittest.TestCase):
{
"format": "json",
"path": "sentence-transformers/nli-mpnet-base-v2",
- "content": True,
+ "content": self.backend,
}
)
embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])
# Generate temp file path
- index = os.path.join(tempfile.gettempdir(), "embeddings.json")
+ index = os.path.join(tempfile.gettempdir(), f"embeddings.{self.backend}.json")
embeddings.save(index)
@@ -307,7 +310,7 @@ class TestEmbeddings(unittest.TestCase):
Test indexing with multiple data types (text, documents)
"""
- embeddings = Embeddings({"path": "sentence-transformers/nli-mpnet-base-v2", "content": True, "batch": len(self.data)})
+ embeddings = Embeddings({"path": "sentence-transformers/nli-mpnet-base-v2", "content": self.backend, "batch": len(self.data)})
# Create an index using mixed data (text and documents)
data = []
@@ -331,14 +334,14 @@ class TestEmbeddings(unittest.TestCase):
self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])
# Save original index
- index = os.path.join(tempfile.gettempdir(), "embeddings.insert")
+ index = os.path.join(tempfile.gettempdir(), f"embeddings.{self.backend}.insert")
self.embeddings.save(index)
# Modify index
self.embeddings.upsert([(0, "Looking out into the dreadful abyss", None)])
# Save to a different location
- indexupdate = os.path.join(tempfile.gettempdir(), "embeddings.update")
+ indexupdate = os.path.join(tempfile.gettempdir(), f"embeddings.{self.backend}.update")
self.embeddings.save(indexupdate)
# Save to same location
@@ -379,13 +382,15 @@ class TestEmbeddings(unittest.TestCase):
Test index
"""
- embeddings = Embeddings({"path": "sentence-transformers/nli-mpnet-base-v2", "content": True, "query": {"path": "neuml/t5-small-txtsql"}})
+ embeddings = Embeddings(
+ {"path": "sentence-transformers/nli-mpnet-base-v2", "content": self.backend, "query": {"path": "neuml/t5-small-txtsql"}}
+ )
# Create an index for the list of text
embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])
# Search for best match
- result = embeddings.search("feel good story with maine in text", 1)[0]
+ result = embeddings.search("feel good story with win in text", 1)[0]
self.assertEqual(result["text"], self.data[4])
@@ -417,7 +422,7 @@ class TestEmbeddings(unittest.TestCase):
self.embeddings.index([(uid, text, None) for uid, text in enumerate(self.data)])
# Generate temp file path
- index = os.path.join(tempfile.gettempdir(), "embeddings")
+ index = os.path.join(tempfile.gettempdir(), f"embeddings.{self.backend}")
self.embeddings.save(index)
self.embeddings.load(index)
@@ -441,7 +446,7 @@ class TestEmbeddings(unittest.TestCase):
# Test similar
result = self.embeddings.search(
- "select * from txtai where similar('feel good story') group by text having count(*) > 0 order by score desc", 1
+ "select text, score from txtai where similar('feel good story') group by text, score having count(*) > 0 order by score desc", 1
)[0]
self.assertEqual(result["text"], self.data[4])