💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Refactor of ann and vectors packages for sparse vectors. Closes #930. Closes #934. Closes #935. Closes #936. Closes #937.
davidmezzettie765be3f6c77
diff --git a/docs/embeddings/configuration/scoring.md b/docs/embeddings/configuration/scoring.mdindex 7e9c7618..2c579d12 100644--- a/docs/embeddings/configuration/scoring.md+++ b/docs/embeddings/configuration/scoring.md@@ -28,17 +28,30 @@ Additional settings for Postgres full-text keyword indexes.### sparse```yamlpath: sparse vector model path+vectormethod: vector embeddings methodgpu: boolean|int|string|device-encodebatch: int-modelargs: additional model init args+batch: Sets the transform batch size+encodebatch: Sets the encode batch size+vectors: additional model init argsencodeargs: additional encode() args-ivf: #IVFFlat index configuration+backend: ivfsparse|pgsparse+```++Sparse vector scoring options. The sparse scoring instance combines a sparse vector model with a sparse approximate nearest neighbor index (ANN).++#### ivfsparse+```yaml+ivfsparse:sample: percent of data to use for model training (0.0 - 1.0)- nclusters: desired number of clusters (int)+ nlist: desired number of clusters (int)nprobe: search probe setting (int)```-Sparse vector scoring options.+Inverted file (IVF) index with flat vector file storage and sparse array support.++#### pgsparse++Sparse ANN backed by Postgres. Supports same options as the [pgvector](../ann/#pgvector) ANN.## terms```yamldiff --git a/setup.py b/setup.pyindex c6c960d1..ddf07f00 100644--- a/setup.py+++ b/setup.py@@ -31,8 +31,10 @@ extras["agent"] = ["mcpadapt>=0.1.0", "smolagents>=1.17"]extras["ann"] = ["annoy>=1.16.3",+ "scikit-learn>=0.23.1","hnswlib>=0.5.0",- "pgvector>=0.3.0",+ "pgvector>=0.4.1",+ "scipy>=1.4.1","sqlalchemy>=2.0.20","sqlite-vec>=0.1.1",]@@ -94,13 +96,14 @@ extras["pipeline"] = (+ extras["pipeline-train"])-extras["scoring"] = ["sentence-transformers>=5.0.0", "sqlalchemy>=2.0.20"]+extras["scoring"] = ["sqlalchemy>=2.0.20"]extras["vectors"] = ["litellm>=1.37.16","llama-cpp-python>=0.2.75","model2vec>=0.3.0","scikit-learn>=0.23.1",+ "scipy>=1.4.1","sentence-transformers>=5.0.0","skops>=0.9.0","staticvectors>=0.2.0",diff --git a/src/python/txtai/ann/__init__.py b/src/python/txtai/ann/__init__.pyindex ad25b6f3..8a54db19 100644--- a/src/python/txtai/ann/__init__.py+++ b/src/python/txtai/ann/__init__.py@@ -2,11 +2,6 @@ANN imports"""-from .annoy import Annoyfrom .base import ANN-from .factory import ANNFactory-from .faiss import Faiss-from .hnsw import HNSW-from .numpy import NumPy-from .pgvector import PGVector-from .torch import Torch+from .dense import *+from .sparse import *diff --git a/src/python/txtai/ann/dense/__init__.py b/src/python/txtai/ann/dense/__init__.pynew file mode 100644index 00000000..804b2481--- /dev/null+++ b/src/python/txtai/ann/dense/__init__.py@@ -0,0 +1,11 @@+"""+Dense ANN imports+"""++from .annoy import Annoy+from .factory import ANNFactory+from .faiss import Faiss+from .hnsw import HNSW+from .numpy import NumPy+from .pgvector import PGVector+from .torch import Torchdiff --git a/src/python/txtai/ann/annoy.py b/src/python/txtai/ann/dense/annoy.pysimilarity index 98%rename from src/python/txtai/ann/annoy.pyrename to src/python/txtai/ann/dense/annoy.pyindex 14551317..eca8e865 100644--- a/src/python/txtai/ann/annoy.py+++ b/src/python/txtai/ann/dense/annoy.py@@ -10,7 +10,7 @@ try:except ImportError:ANNOY = False-from .base import ANN+from ..base import ANN# pylint: disable=W0223diff --git a/src/python/txtai/ann/factory.py b/src/python/txtai/ann/dense/factory.pysimilarity index 98%rename from src/python/txtai/ann/factory.pyrename to src/python/txtai/ann/dense/factory.pyindex cb0a4892..4beeccd3 100644--- a/src/python/txtai/ann/factory.py+++ b/src/python/txtai/ann/dense/factory.py@@ -2,7 +2,7 @@Factory module"""-from ..util import Resolver+from ...util import Resolverfrom .annoy import Annoyfrom .faiss import Faissdiff --git a/src/python/txtai/ann/faiss.py b/src/python/txtai/ann/dense/faiss.pysimilarity index 99%rename from src/python/txtai/ann/faiss.pyrename to src/python/txtai/ann/dense/faiss.pyindex 6144794e..cbef53bf 100644--- a/src/python/txtai/ann/faiss.py+++ b/src/python/txtai/ann/dense/faiss.py@@ -11,7 +11,7 @@ from faiss import omp_set_num_threadsfrom faiss import index_factory, IO_FLAG_MMAP, METRIC_INNER_PRODUCT, read_index, write_indexfrom faiss import index_binary_factory, read_index_binary, write_index_binary, IndexBinaryIDMap-from .base import ANN+from ..base import ANNif platform.system() == "Darwin":# Workaround for a Faiss issue causing segmentation faults on macOS. See txtai FAQ for more.diff --git a/src/python/txtai/ann/hnsw.py b/src/python/txtai/ann/dense/hnsw.pysimilarity index 99%rename from src/python/txtai/ann/hnsw.pyrename to src/python/txtai/ann/dense/hnsw.pyindex 4460238c..f101ade1 100644--- a/src/python/txtai/ann/hnsw.py+++ b/src/python/txtai/ann/dense/hnsw.py@@ -13,7 +13,7 @@ try:except ImportError:HNSWLIB = False-from .base import ANN+from ..base import ANNclass HNSW(ANN):diff --git a/src/python/txtai/ann/numpy.py b/src/python/txtai/ann/dense/numpy.pysimilarity index 98%rename from src/python/txtai/ann/numpy.pyrename to src/python/txtai/ann/dense/numpy.pyindex 0ab0a4ae..e8fb433e 100644--- a/src/python/txtai/ann/numpy.py+++ b/src/python/txtai/ann/dense/numpy.py@@ -4,9 +4,9 @@ NumPy moduleimport numpy as np-from ..serialize import SerializeFactory+from ...serialize import SerializeFactory-from .base import ANN+from ..base import ANNclass NumPy(ANN):diff --git a/src/python/txtai/ann/pgvector.py b/src/python/txtai/ann/dense/pgvector.pysimilarity index 93%rename from src/python/txtai/ann/pgvector.pyrename to src/python/txtai/ann/dense/pgvector.pyindex b5068e94..5b78cc1b 100644--- a/src/python/txtai/ann/pgvector.py+++ b/src/python/txtai/ann/dense/pgvector.py@@ -18,7 +18,7 @@ try:except ImportError:PGVECTOR = False-from .base import ANN+from ..base import ANNclass PGVector(ANN):@@ -111,7 +111,7 @@ class PGVector(ANN):self.schema()# Table name- table = self.setting("table", "vectors")+ table = self.setting("table", self.defaulttable())# Get embedding column and index settingscolumn, index = self.column()@@ -147,7 +147,7 @@ class PGVector(ANN):self.close()# Create engine- self.engine = create_engine(self.setting("url", os.environ.get("ANN_URL")), poolclass=StaticPool, echo=False)+ self.engine = create_engine(self.url(), poolclass=StaticPool, echo=False)self.connection = self.engine.connect()# Start database session@@ -191,6 +191,26 @@ class PGVector(ANN):args = (sql, parameters) if self.engine.dialect.name == "postgresql" else (text("SELECT 1"),)self.database.execute(*args)+ def defaulttable(self):+ """+ Returns the default table name.++ Returns:+ default table name+ """++ return "vectors"++ def url(self):+ """+ Reads the database url parameter.++ Returns:+ database url+ """++ return self.setting("url", os.environ.get("ANN_URL"))+def column(self):"""Gets embedding column and index definitions for the current settings.diff --git a/src/python/txtai/ann/sqlite.py b/src/python/txtai/ann/dense/sqlite.pysimilarity index 99%rename from src/python/txtai/ann/sqlite.pyrename to src/python/txtai/ann/dense/sqlite.pyindex 7c00c85c..0dc185e9 100644--- a/src/python/txtai/ann/sqlite.py+++ b/src/python/txtai/ann/dense/sqlite.py@@ -13,7 +13,7 @@ try:except ImportError:SQLITEVEC = False-from .base import ANN+from ..base import ANNclass SQLite(ANN):diff --git a/src/python/txtai/ann/torch.py b/src/python/txtai/ann/dense/torch.pysimilarity index 100%rename from src/python/txtai/ann/torch.pyrename to src/python/txtai/ann/dense/torch.pydiff --git a/src/python/txtai/ann/sparse/__init__.py b/src/python/txtai/ann/sparse/__init__.pynew file mode 100644index 00000000..616cdd4e--- /dev/null+++ b/src/python/txtai/ann/sparse/__init__.py@@ -0,0 +1,7 @@+"""+Sparse ANN imports+"""++from .factory import SparseANNFactory+from .ivfsparse import IVFSparse+from .pgsparse import PGSparsediff --git a/src/python/txtai/ann/sparse/factory.py b/src/python/txtai/ann/sparse/factory.pynew file mode 100644index 00000000..d7b42b21--- /dev/null+++ b/src/python/txtai/ann/sparse/factory.py@@ -0,0 +1,61 @@+"""+Factory module+"""++from ...util import Resolver++from .ivfsparse import IVFSparse+from .pgsparse import PGSparse+++class SparseANNFactory:+ """+ Methods to create Sparse ANN indexes.+ """++ @staticmethod+ def create(config):+ """+ Create an Sparse ANN.++ Args:+ config: index configuration parameters++ Returns:+ Sparse ANN+ """++ # ANN instance+ ann = None+ backend = config.get("backend", "ivfsparse")++ # Create ANN instance+ if backend == "ivfsparse":+ ann = IVFSparse(config)+ elif backend == "pgsparse":+ ann = PGSparse(config)+ else:+ ann = SparseANNFactory.resolve(backend, config)++ # Store config back+ config["backend"] = backend++ return ann++ @staticmethod+ def resolve(backend, config):+ """+ Attempt to resolve a custom backend.++ Args:+ backend: backend class+ config: index configuration parameters++ Returns:+ ANN+ """++ try:+ return Resolver()(backend)(config)+ except Exception as e:+ raise ImportError(f"Unable to resolve sparse ann backend: '{backend}'") from ediff --git a/src/python/txtai/scoring/ivf.py b/src/python/txtai/ann/sparse/ivfsparse.pysimilarity index 58%rename from src/python/txtai/scoring/ivf.pyrename to src/python/txtai/ann/sparse/ivfsparse.pyindex 41c9f051..c14e98fd 100644--- a/src/python/txtai/scoring/ivf.py+++ b/src/python/txtai/ann/sparse/ivfsparse.py@@ -1,5 +1,5 @@"""-IVF module+IVFSparse module"""import math@@ -8,23 +8,25 @@ import numpy as np# Conditional importtry:- from scipy.sparse import csr_matrix, vstack+ from scipy.sparse import vstackfrom sklearn.cluster import MiniBatchKMeansfrom sklearn.metrics import pairwise_distances_argmin_minfrom sklearn.utils.extmath import safe_sparse_dot- SKLEARN = True+ IVFSPARSE = Trueexcept ImportError:- SKLEARN = False+ IVFSPARSE = False-from ..serialize import SerializeFactory+from ...serialize import SerializeFactory+from ...util import SparseArray+from ..base import ANN-class IVFFlat:+class IVFSparse(ANN):"""- Inverted file (IVF) index with flat vector storage and sparse array support.+ Inverted file (IVF) index with flat vector file storage and sparse array support.- IVFFlat builds an IVF index and enables approximate nearest neighbor (ANN) search.+ IVFSparse builds an IVF index and enables approximate nearest neighbor (ANN) search.This index is modeled after Faiss and supports many of the same parameters.@@ -32,18 +34,10 @@ class IVFFlat:"""def __init__(self, config):- """- Create a new IVFFlat instance.-- Args:- config: index configuration- """+ super().__init__(config)- if not SKLEARN:- raise ImportError('IVFFlat is not available - install "scoring" extra to enable')-- # Index configuration- self.config = config if isinstance(config, dict) else {}+ if not IVFSPARSE:+ raise ImportError('IVFSparse is not available - install "ann" extra to enable')# Cluster centroids, if computedself.centroids = None@@ -57,17 +51,9 @@ class IVFFlat:# Deleted idsself.deletes = None- def index(self, data):- """- Builds a new IVFFlat index.-- Args:- data: input data- config: index configuration- """-+ def index(self, embeddings):# Compute model training size- train, sample = data, self.config.get("sample")+ train, sample = embeddings, self.setting("sample")if sample:# Get sample for trainingrng = np.random.default_rng(0)@@ -75,7 +61,7 @@ class IVFFlat:train = train[indices]# Get number of clusters. Note that final number of clusters could be lower due to filtering duplicate centroids.- clusters = self.nclusters(data.shape[0], train.shape[0])+ clusters = self.nlist(embeddings.shape[0], train.shape[0])# A single cluster is an exact searchif clusters > 1:@@ -85,103 +71,75 @@ class IVFFlat:# Find closest points to each cluster center and use those as centroids# Filter out duplicate centroidsindices, _ = pairwise_distances_argmin_min(kmeans.cluster_centers_, train, metric="cosine")- self.centroids = data[np.unique(indices)]+ self.centroids = embeddings[np.unique(indices)]# Sort into clusters- ids = self.aggregate(data)+ ids = self.aggregate(embeddings)# Sort clusters by idself.ids = dict(sorted(ids.items(), key=lambda x: x[0]))# Create cluster data blocks- self.blocks = {k: data[v] for k, v in self.ids.items()}+ self.blocks = {k: embeddings[v] for k, v in self.ids.items()}# Initialize deletesself.deletes = []- def append(self, data):- """- Appends elements to an existing index.-- Args:- data: input data- """+ # Add id offset and index build metadata+ self.config["offset"] = embeddings.shape[0]+ self.metadata({"clusters": clusters})+ def append(self, embeddings):# Get offsetoffset = self.size()# Sort into clusters and merge- for cluster, ids in self.aggregate(data).items():+ for cluster, ids in self.aggregate(embeddings).items():# Add new idsself.ids[cluster].extend([x + offset for x in ids])# Add new data- self.blocks[cluster] = vstack([self.blocks[cluster], data[ids]])-- def delete(self, ids):- """- Mark ids as deleted. This prevents deleted results from showing up in search results.- The data is not removed from the underlying data structures.+ self.blocks[cluster] = vstack([self.blocks[cluster], embeddings[ids]])- Args:- ids: ids to delete- """+ # Update id offset and index metadata+ self.config["offset"] += embeddings.shape[0]+ self.metadata()+ def delete(self, ids):# Set index ids as deletedself.deletes.extend(ids)- def search(self, query, limit):- """- Searches IVFFlat index for query. Returns topn results.-- Args:- query: query array- limit: maximum results-- Returns:- query results- """+ def search(self, queries, limit):+ results = []+ for query in queries:+ if self.centroids is not None:+ # Approximate search+ indices, _ = self.topn(query, self.centroids, self.nprobe())- if self.centroids is not None:- # Approximate search- indices, _ = self.topn(query, self.centroids, self.nprobe())+ # Stack into single ids list+ ids = np.concatenate([self.ids[x] for x in indices if x in self.ids])- # Stack into single ids list- ids = np.concatenate([self.ids[x] for x in indices if x in self.ids])+ # Stack data rows+ data = vstack([self.blocks[x] for x in indices if x in self.blocks])+ else:+ # Exact search+ ids, data = np.array(self.ids[0]), self.blocks[0]- # Stack data rows- data = vstack([self.blocks[x] for x in indices if x in self.blocks])- else:- # Exact search- ids, data = np.array(self.ids[0]), self.blocks[0]+ # Get deletes+ deletes = np.argwhere(np.isin(ids, self.deletes)).ravel()- # Get deletes- deletes = np.argwhere(np.isin(ids, self.deletes)).ravel()+ # Calculate similarity+ indices, scores = self.topn(query, data, limit, deletes)- # Calculate similarity- indices, scores = self.topn(query, data, limit, deletes)+ # Map data ids and return+ results.append(list(zip(ids[indices].tolist(), scores.tolist())))- # Map data ids and return- return list(zip(ids[indices].tolist(), scores.tolist()))+ return resultsdef count(self):- """- Number of elements in the IVFFlat index.-- Returns:- count- """-return self.size() - len(self.deletes)def load(self, path):- """- Loads an IVFFlat index from path.-- Args:- path: file path- """-# Create streaming serializer and limit read size to a byte at a time to ensure# only msgpack data is consumedserializer = SerializeFactory.create("msgpack", streaming=True, read_size=1)@@ -192,7 +150,7 @@ class IVFFlat:header = next(unpacker)# Read cluster centroids, if available- self.centroids = SparseArray().read(f) if header["centroids"] else None+ self.centroids = SparseArray().load(f) if header["centroids"] else None# Read cluster idsself.ids = dict(next(unpacker))@@ -200,23 +158,18 @@ class IVFFlat:# Read cluster data blocksself.blocks = {}for key in self.ids:- self.blocks[key] = SparseArray().read(f)+ self.blocks[key] = SparseArray().load(f)# Read deletesself.deletes = next(unpacker)def save(self, path):- """- Saves an IVFFlat index to path. This format uses a combination of msgpack and NumPy serialization.- Sparse arrays are saved as groups of NumPy arrays.-- IVFFlat storage format:- - header msgpack- - centroids sparse array (optional based on header parameters)- - cluster ids msgpack- - cluster data blocks list of sparse arrays- - deletes msgpack- """+ # IVFSparse storage format:+ # - header msgpack+ # - centroids sparse array (optional based on header parameters)+ # - cluster ids msgpack+ # - cluster data blocks list of sparse arrays+ # - deletes msgpack# Create message pack serializerserializer = SerializeFactory.create("msgpack")@@ -227,14 +180,14 @@ class IVFFlat:# Write cluster centroids, if availableif self.centroids is not None:- SparseArray().write(self.centroids, f)+ SparseArray().save(f, self.centroids)# Write cluster id mappingserializer.savestream(list(self.ids.items()), f)# Write cluster data blocksfor block in self.blocks.values():- SparseArray().write(block, f)+ SparseArray().save(f, block)# Write deletesserializer.savestream(self.deletes, f)@@ -296,10 +249,10 @@ class IVFFlat:return indices[0], scores[0]- def nclusters(self, count, train):+ def nlist(self, count, train):"""- Calculates the number of clusters for this IVFFlat index. Note that the final number of clusters could be- lower as duplicate cluster centroids are filtered out.+ Calculates the number of clusters for this IVFSparse index. Note that the final number of clusters+ could be lower as duplicate cluster centroids are filtered out.Args:count: initial dataset size@@ -313,7 +266,7 @@ class IVFFlat:default = 1 if count <= 5000 else self.cells(train)# Number of clusters to create- return self.config.get("nclusters", default)+ return self.setting("nlist", default)def nprobe(self):"""@@ -327,11 +280,11 @@ class IVFFlat:size = self.size()default = 6 if size <= 5000 else self.cells(size) // 64- return self.config.get("nprobe", default)+ return self.setting("nprobe", default)def cells(self, count):"""- Calculates the number of IVF cells for an IVFFlat index.+ Calculates the number of IVF cells for an IVFSparse index.Args:count: number of rows@@ -353,38 +306,3 @@ class IVFFlat:"""return sum(len(x) for x in self.ids.values())---class SparseArray:- """- Methods to read and write sparse arrays to file.- """-- def read(self, f):- """- Reads a sparse array from input file.-- Args:- f: input file handle-- Returns:- sparse array- """-- data, indices, indptr, shape = np.load(f), np.load(f), np.load(f), np.load(f)-- # Read sparse array- return csr_matrix((data, indices, indptr), shape=shape)-- def write(self, array, f):- """- Writes a sparse array to file.-- Args:- array: sparse array- f: output file handle- """-- # Write sparse array to file- for x in [array.data, array.indices, array.indptr, array.shape]:- np.save(f, x, allow_pickle=False)diff --git a/src/python/txtai/ann/sparse/pgsparse.py b/src/python/txtai/ann/sparse/pgsparse.pynew file mode 100644index 00000000..1a971f21--- /dev/null+++ b/src/python/txtai/ann/sparse/pgsparse.py@@ -0,0 +1,44 @@+"""+PGSparse module+"""++import os++# Conditional import+try:+ from pgvector import SparseVector+ from pgvector.sqlalchemy import SPARSEVEC++ PGSPARSE = True+except ImportError:+ PGSPARSE = False++from ..dense import PGVector+++class PGSparse(PGVector):+ """+ Builds a Sparse ANN index backed by a Postgres database.+ """++ def __init__(self, config):+ if not PGSPARSE:+ raise ImportError('PGSparse is not available - install "ann" extra to enable')++ super().__init__(config)++ # Quantization not supported+ self.qbits = None++ def defaulttable(self):+ return "svectors"++ def url(self):+ return self.setting("url", os.environ.get("SCORING_URL", os.environ.get("ANN_URL")))++ def column(self):+ return SPARSEVEC(self.config["dimensions"]), "sparsevec_ip_ops"++ def prepare(self, data):+ # Wrap as sparse vector+ return SparseVector(data)diff --git a/src/python/txtai/embeddings/base.py b/src/python/txtai/embeddings/base.pyindex 8868503f..3fdbfac7 100644--- a/src/python/txtai/embeddings/base.py+++ b/src/python/txtai/embeddings/base.py@@ -6,8 +6,6 @@ import jsonimport osimport tempfile-import numpy as np-from ..ann import ANNFactoryfrom ..archive import ArchiveFactoryfrom ..cloud import CloudFactory@@ -323,7 +321,7 @@ class Embeddings:self.defaults()# Get vector model- model = self.indexes.model(index) if index and self.indexes else self.model if self.model else self.indexes.model()+ model = self.findmodel(index)# Convert documents into embeddingsembeddings = model.batchtransform(Stream(self)(documents), category)@@ -437,8 +435,11 @@ class Embeddings:queries = self.batchtransform(((None, query, None) for query in queries), "query")data = self.batchtransform(((None, row, None) for row in data), "data")+ # Get vector model+ model = self.findmodel()+# Dot product on normalized vectors is equal to cosine similarity- scores = np.dot(queries, data.T).tolist()+ scores = model.dot(queries, data)# Add index and sort desc based on scorereturn [sorted(enumerate(score), key=lambda x: x[1], reverse=True) for score in scores]@@ -711,13 +712,13 @@ class Embeddings:def issparse(self):"""- Checks if this instance has an associated scoring instance with term indexing enabled.+ Checks if this instance has an associated sparse keyword or sparse vectors scoring index.Returns:- True if term index is enabled, False otherwise+ True if scoring has an associated sparse keyword/vector index, False otherwise"""- return self.scoring and self.scoring.hasterms()+ return self.scoring and self.scoring.issparse()def isdense(self):"""@@ -737,7 +738,25 @@ class Embeddings:True if term weighting is enabled, False otherwise"""- return self.scoring and not self.scoring.hasterms()+ return self.scoring and self.scoring.isweighted()++ def findmodel(self, index=None):+ """+ Finds the primary vector model used by this instance.++ Returns:+ Vectors+ """++ return (+ self.indexes.findmodel(index)+ if index and self.indexes+ else (+ self.model+ if self.model+ else self.scoring.findmodel() if self.scoring and self.scoring.findmodel() else self.indexes.findmodel() if self.indexes else None+ )+ )def configure(self, config):"""@@ -755,7 +774,7 @@ class Embeddings:# Create scoring instance for word vectors term weightingscoring = self.config.get("scoring") if self.config else None- self.scoring = self.createscoring() if scoring and (not isinstance(scoring, dict) or not scoring.get("terms")) else None+ self.scoring = self.createscoring() if scoring and not self.hassparse() else None# Dense vectors - transforms data to embeddings vectorsself.model = self.loadvectors() if self.config else None@@ -791,9 +810,8 @@ class Embeddings:# Initialize ANN, will be created after index transformations completeself.ann = None- # Create scoring only if term indexing is enabled- scoring = self.config.get("scoring")- if scoring and isinstance(scoring, dict) and self.config["scoring"].get("terms"):+ # Create scoring only if the scoring config is for a sparse index+ if self.hassparse():self.scoring = self.createscoring()# Create subindexes, if necessary@@ -832,24 +850,29 @@ class Embeddings:Logic to derive default sparse index configuration."""- # Get sparse index configuration- index, method = None, None- for x in ["keyword", "sparse", "hybrid"]:+ # Check for keyword and hybrid parameters+ method = None+ for x in ["keyword", "hybrid"]:value = self.config.get(x)if value:- index, method = (x, value if isinstance(value, str) else "bm25")+ method = value if isinstance(value, str) else "bm25"- # Default scoring configuration- self.config["scoring"] = {"method": method, "terms": True, "normalize": True}+ # Enable dense index when hybrid enabled+ if x == "hybrid":+ self.config["dense"] = True- # Sparse vector models- if index == "sparse" or "/" in method:- # Derive sparse model path- path = self.config["sparse"] if index == "sparse" else method- path = path if isinstance(path, str) else "prithivida/Splade_PP_en_v2"+ sparse = self.config.get("sparse", {})+ if sparse or method == "sparse":+ # Sparse vector configuration+ sparse = {"path": self.config.get("sparse")} if isinstance(sparse, str) else {} if isinstance(sparse, bool) else sparse+ sparse["path"] = sparse.get("path", "prithivida/Splade_PP_en_v2")# Merge in sparse parameters- self.config["scoring"] = {**self.config["scoring"], **{"method": "sparse", "path": path}}+ self.config["scoring"] = {**{"method": "sparse"}, **sparse}++ elif method:+ # Sparse keyword configuration+ self.config["scoring"] = {"method": method, "terms": True, "normalize": True}def defaultallowed(self):"""@@ -1051,6 +1074,17 @@ class Embeddings:return None+ def hassparse(self):+ """+ Checks is this embeddings database has an associated sparse index.++ Returns:+ True if this embeddings has an associated scoring index+ """++ # Create scoring only if scoring is a sparse keyword/vector index+ return ScoringFactory.issparse(self.config.get("scoring"))+def columns(self, config):"""Adds custom text/object column information if it's provided.diff --git a/src/python/txtai/embeddings/index/indexes.py b/src/python/txtai/embeddings/index/indexes.pyindex f9e5f12d..721e945f 100644--- a/src/python/txtai/embeddings/index/indexes.py+++ b/src/python/txtai/embeddings/index/indexes.py@@ -25,6 +25,7 @@ class Indexes:self.indexes = indexesself.documents = None+ self.checkpoint = None# Transform columnscolumns = embeddings.config.get("columns", {})@@ -83,29 +84,34 @@ class Indexes:return list(self.indexes.keys())[0]- def model(self, index=None):+ def findmodel(self, index=None):"""- Lookups a vector model. If index is empty, the first vector model is returned.+ Finds a vector model. If index is empty, the first vector model is returned.++ Args:+ index: index name to matchReturns:Vectors"""# Find vector model- matches = [self.indexes[index]] if index else [index.model for index in self.indexes.values() if index.model]+ matches = [self.indexes[index].findmodel()] if index else [index.findmodel() for index in self.indexes.values() if index.findmodel()]return matches[0] if matches else None- def insert(self, documents, index=None):+ def insert(self, documents, index=None, checkpoint=None):"""Inserts a batch of documents into each subindex.Args:documents: list of (id, data, tags)index: indexid offset+ checkpoint: optional checkpoint directory, enables indexing restart"""if not self.documents:self.documents = Documents()+ self.checkpoint = checkpoint# Create batch containing documents added to parent indexbatch = []@@ -139,12 +145,13 @@ class Indexes:Builds each subindex."""- for index in self.indexes.values():- index.index(self.documents)+ for name, index in self.indexes.items():+ index.index(self.documents, checkpoint=f"{self.checkpoint}/{name}" if self.checkpoint else None)# Reset document streamself.documents.close()self.documents = None+ self.checkpoint = Nonedef upsert(self):"""diff --git a/src/python/txtai/embeddings/index/transform.py b/src/python/txtai/embeddings/index/transform.pyindex ce7e27d0..3d7a4d0e 100644--- a/src/python/txtai/embeddings/index/transform.py+++ b/src/python/txtai/embeddings/index/transform.py@@ -2,8 +2,6 @@Transform module"""-import os-import numpy as npfrom .action import Action@@ -92,29 +90,11 @@ class Transform:(document ids, dimensions, embeddings)"""- # Consume stream and transform documents to vectors- ids, dimensions, batches, stream = self.model.index(self.stream(documents), self.batch, self.checkpoint)-- # Check that embeddings are available and load as a memmap- embeddings = None- if ids:- # Determine dtype- dtype = np.uint8 if self.qbits else np.float32-- # Write batches- embeddings = np.memmap(buffer, dtype=dtype, shape=(len(ids), dimensions), mode="w+")- with open(stream, "rb") as queue:- x = 0- for _ in range(batches):- batch = np.load(queue)- embeddings[x : x + batch.shape[0]] = batch- x += batch.shape[0]+ # Determine dtype+ dtype = np.uint8 if self.qbits else np.float32- # Remove temporary file (if checkpointing is disabled)- if not self.checkpoint:- os.remove(stream)-- return (ids, dimensions, embeddings)+ # Transform documents into vectors+ return self.model.vectors(self.stream(documents), self.batch, self.checkpoint, buffer, dtype)def ids(self, documents):"""@@ -211,11 +191,11 @@ class Transform:# Load batch into scoringif self.scoring:- self.scoring.insert(batch, self.offset)+ self.scoring.insert(batch, self.offset, self.checkpoint)# Load batch into subindex documents streamif self.indexes:- self.indexes.insert(batch, self.offset)+ self.indexes.insert(batch, self.offset, self.checkpoint)# Load batch into graphif self.graph:diff --git a/src/python/txtai/pipeline/audio/texttospeech.py b/src/python/txtai/pipeline/audio/texttospeech.pyindex af23a388..82b1ee7b 100644--- a/src/python/txtai/pipeline/audio/texttospeech.py+++ b/src/python/txtai/pipeline/audio/texttospeech.py@@ -407,7 +407,7 @@ class SpeechT5(SpeechPipeline):self.vocoder = ort.InferenceSession(cached_file(path_or_repo_id=path, filename="decoder_postnet_and_vocoder.onnx"), providers=providers)self.processor = SpeechT5Processor.from_pretrained(path)- self.defaultspeaker = np.load(cached_file(path_or_repo_id=path, filename="speaker.npy"))+ self.defaultspeaker = np.load(cached_file(path_or_repo_id=path, filename="speaker.npy"), allow_pickle=False)# Max number of input tokens model can handleself.maxtokens = maxtokensdiff --git a/src/python/txtai/scoring/__init__.py b/src/python/txtai/scoring/__init__.pyindex e1816992..cb7cdc3d 100644--- a/src/python/txtai/scoring/__init__.py+++ b/src/python/txtai/scoring/__init__.py@@ -5,7 +5,6 @@ Scoring importsfrom .base import Scoringfrom .bm25 import BM25from .factory import ScoringFactory-from .ivf import IVFFlatfrom .pgtext import PGTextfrom .sif import SIFfrom .sparse import Sparsediff --git a/src/python/txtai/scoring/base.py b/src/python/txtai/scoring/base.pyindex 63503ad9..ddd4e51b 100644--- a/src/python/txtai/scoring/base.py+++ b/src/python/txtai/scoring/base.py@@ -24,13 +24,17 @@ class Scoring:self.text = columns.get("text", "text")self.object = columns.get("object", "object")- def insert(self, documents, index=None):+ # Vector model, if available+ self.model = None++ def insert(self, documents, index=None, checkpoint=None):"""Inserts documents into the scoring index.Args:documents: list of (id, dict|text|tokens, tags)index: indexid offset+ checkpoint: optional checkpoint directory, enables indexing restart"""raise NotImplementedError@@ -143,16 +147,36 @@ class Scoring:raise NotImplementedError- def hasterms(self):+ def findmodel(self):+ """+ Returns the associated vector model used by this scoring instance, if any.++ Returns:+ associated vector model+ """++ return self.model++ def issparse(self):"""- Check if this scoring instance has an associated terms index.+ Check if this scoring instance has an associated sparse keyword or sparse vector index.Returns:- True if this scoring instance has an associated terms index.+ True if this index has an associated sparse index"""raise NotImplementedError+ def isweighted(self):+ """+ Check if this scoring instance is for term weighting (i.e.) it has no associated sparse index.++ Returns:+ True if this index is for term weighting+ """++ return not self.issparse()+def isnormalized(self):"""Check if this scoring instance returns normalized scores.diff --git a/src/python/txtai/scoring/factory.py b/src/python/txtai/scoring/factory.pyindex 868db626..70cccf7d 100644--- a/src/python/txtai/scoring/factory.py+++ b/src/python/txtai/scoring/factory.py@@ -22,7 +22,7 @@ class ScoringFactory:Factory method to construct a Scoring instance.Args:- config: scoring configuration parameters - supports bm25, sif, tfidf+ config: scoring configuration parametersmodels: models cacheReturns:@@ -58,6 +58,24 @@ class ScoringFactory:return scoring+ @staticmethod+ def issparse(config):+ """+ Checks if this scoring configuration builds a sparse index.++ Args:+ config: scoring configuration++ Returns:+ True if this config is for a sparse index+ """++ # Types that are always a sparse index+ indexes = ["pgtext", "sparse"]++ # True if this config is for a sparse index+ return config and isinstance(config, dict) and (config.get("method") in indexes or config.get("terms"))+@staticmethoddef resolve(backend, config):"""diff --git a/src/python/txtai/scoring/pgtext.py b/src/python/txtai/scoring/pgtext.pyindex 4569151a..a32cc630 100644--- a/src/python/txtai/scoring/pgtext.py+++ b/src/python/txtai/scoring/pgtext.py@@ -36,7 +36,7 @@ class PGText(Scoring):# Languageself.language = self.config.get("language", "english")- def insert(self, documents, index=None):+ def insert(self, documents, index=None, checkpoint=None):# Initialize tablesself.initialize(recreate=True)@@ -108,7 +108,7 @@ class PGText(Scoring):self.database.close()self.engine.dispose()- def hasterms(self):+ def issparse(self):return Truedef isnormalized(self):diff --git a/src/python/txtai/scoring/sparse.py b/src/python/txtai/scoring/sparse.pyindex 864186f0..717a0909 100644--- a/src/python/txtai/scoring/sparse.py+++ b/src/python/txtai/scoring/sparse.py@@ -2,20 +2,13 @@Sparse module"""-# Conditional import-try:- from scipy.sparse import csr_matrix, vstack- from sentence_transformers import SparseEncoder- from sklearn.preprocessing import normalize+from queue import Queue+from threading import Thread- SPARSE = True-except ImportError:- SPARSE = False--from ..models import Models+from ..ann import SparseANNFactory+from ..vectors import SparseVectorsFactoryfrom .base import Scoring-from .ivf import IVFFlatclass Sparse(Scoring):@@ -23,185 +16,166 @@ class Sparse(Scoring):Sparse vector scoring."""+ # End of stream message+ COMPLETE = 1+def __init__(self, config=None, models=None):super().__init__(config)- if not SPARSE:- raise ImportError('Sparse encoder is not available - install "scoring" extra to enable')-- # Models cache- self.models = models+ # Vector configuration+ config = {k: v for k, v in config.items() if k != "method"}+ if "vectormethod" in config:+ config["method"] = config["vectormethod"]- # Pool parameter- self.pool = None+ # Load the SparseVectors model+ self.model = SparseVectorsFactory.create(config, models)- # Sparse encoder- self.model = self.loadmodel()+ # Sparse ANN+ self.ann = None- # Encode batch size - controls underlying model batch size when encoding vectors- self.encodebatch = config.get("encodebatch", 32)+ # Encoding processing parameters+ self.batch = self.config.get("batch", 1024)+ self.thread, self.queue, self.data = None, None, None- # Index backend- self.backend = None+ def insert(self, documents, index=None, checkpoint=None):+ # Start processing thread, if necessary+ self.start(checkpoint)- # Input queue - data to be indexed- self.queue = None-- def insert(self, documents, index=None):data = []- for _, document, _ in documents:+ for uid, document, tags in documents:# Extract text, if necessaryif isinstance(document, dict):document = document.get(self.text, document.get(self.object))if document is not None:# Add data- data.append(" ".join(document) if isinstance(document, list) else document)-- # Encode and normalize data- data = self.encode(data)+ data.append((uid, " ".join(document) if isinstance(document, list) else document, tags))- # Create or add to existing sparse vectors- self.queue = vstack((self.queue, data)) if self.queue is not None else data+ # Add batch of data+ self.queue.put(data)def delete(self, ids):- self.backend.delete(ids)+ self.ann.delete(ids)def index(self, documents=None):# Insert documents, if providedif documents:self.insert(documents)- # Create index- if self.queue is not None:- # Create a new index instance- self.backend = IVFFlat(self.config.get("ivf"))- self.backend.index(self.queue)-- # Clear queue- self.queue = None+ # Create ANN, if there is pending data+ embeddings = self.stop()+ if embeddings is not None:+ self.ann = SparseANNFactory.create(self.config)+ self.ann.index(embeddings)def upsert(self, documents=None):# Insert documents, if providedif documents:self.insert(documents)- # Upsert index- if self.backend and self.queue is not None:- self.backend.append(self.queue)+ # Check for existing index and pending data+ if self.ann:+ embeddings = self.stop()+ if embeddings is not None:+ self.ann.append(embeddings)else:self.index()- # Clear queue- self.queue = None-def weights(self, tokens):# Not supportedreturn Nonedef search(self, query, limit=3):- return self.backend.search(self.encode([query]), limit)+ return self.batchsearch([query], limit)[0]def batchsearch(self, queries, limit=3, threads=True):- return [self.search(query, limit) for query in queries]+ # Convert queries to embedding vectors+ embeddings = self.model.batchtransform((None, query, None) for query in queries)++ # Run ANN search+ return self.ann.search(embeddings, limit)def count(self):- return self.backend.count()+ return self.ann.count()def load(self, path):- # Read IVFFlat index- self.backend = IVFFlat(self.config.get("ivf"))- self.backend.load(path)+ self.ann = SparseANNFactory.create(self.config)+ self.ann.load(path)def save(self, path):- # Write IVFFlat index- if self.backend is not None:- self.backend.save(path)+ # Save Sparse ANN+ if self.ann:+ self.ann.save(path)def close(self):- # Close pool before model is closed- if self.pool:- self.model.stop_multi_process_pool(self.pool)- self.pool = None+ # Close Sparse ANN+ if self.ann:+ self.ann.close()- self.model, self.backend, self.queue = None, None, None+ # Clear parameters+ self.model, self.ann, self.thread, self.queue = None, None, None, None- def hasterms(self):+ def issparse(self):return Truedef isnormalized(self):return True- def loadmodel(self):+ def start(self, checkpoint):"""- Loads the sparse encoder model.+ Starts an encoding processing thread.- Returns:- SparseEncoder+ Args:+ checkpoint: checkpoint directory"""- # Model path- path = self.config.get("path")-- # Check if model is cached- if self.models and path in self.models:- return self.models[path]+ if not self.thread:+ self.queue = Queue(5)+ self.thread = Thread(target=self.encode, args=(checkpoint,))+ self.thread.start()- # Get target device- gpu, pool = self.config.get("gpu", True), False-- # Default mode uses a single GPU. Setting to all spawns a process per GPU.- if isinstance(gpu, str) and gpu == "all":- # Get number of accelerator devices available- devices = Models.acceleratorcount()-- # Enable multiprocessing pooling only when multiple devices are available- gpu, pool = devices <= 1, devices > 1-- # Tensor device id- deviceid = Models.deviceid(gpu)+ def stop(self):+ """+ Stops an encoding processing thread. Return processed results.- # Additional model arguments- modelargs = self.config.get("modelargs", {})+ Returns:+ results+ """- # Build embeddings with sentence-transformers- model = SparseEncoder(path, device=Models.device(deviceid), **modelargs)+ results = None+ if self.thread:+ # Send EOS message+ self.queue.put(Sparse.COMPLETE)- # Start process pool for multiple GPUs- if pool:- self.pool = model.start_multi_process_pool()+ self.thread.join()+ self.thread, self.queue = None, None- # Store model in cache- if self.models is not None and path:- self.models[path] = model+ # Get return value+ results = self.data+ self.data = None- # Return model- return model+ return results- def encode(self, data):+ def encode(self, checkpoint):"""- Encodes a batch of data using the Sparse Encoder model.+ Encodes streaming data.Args:- data: input data-- Returns:- encoded data as a SciPy CSR Matrix+ checkpoint: checkpoint directory"""- # Additional encoding arguments- encodeargs = self.config.get("encodeargs", {})-- # Encode data- data = self.model.encode(data, pool=self.pool, batch_size=self.encodebatch, **encodeargs)+ # Streaming encoding of data+ _, dimensions, self.data = self.model.vectors(self.stream(), self.batch, checkpoint)- # Get data attributes- data = data.cpu().coalesce()- indices = data.indices().numpy()- values = data.values().numpy()+ # Save number of dimensions+ self.config["dimensions"] = dimensions- # Convert to CSR Matrix- matrix = csr_matrix((values, indices), shape=data.size())+ def stream(self):+ """+ Streams data from an input queue until end of stream message received.+ """- # Normalize and return- return normalize(matrix)+ batch = self.queue.get()+ while batch != Sparse.COMPLETE:+ yield from batch+ batch = self.queue.get()diff --git a/src/python/txtai/scoring/tfidf.py b/src/python/txtai/scoring/tfidf.pyindex 83ab2163..e1c5f29c 100644--- a/src/python/txtai/scoring/tfidf.py+++ b/src/python/txtai/scoring/tfidf.py@@ -55,7 +55,7 @@ class TFIDF(Scoring):self.normalize = self.config.get("normalize")self.avgscore = None- def insert(self, documents, index=None):+ def insert(self, documents, index=None, checkpoint=None):# Insert documents, calculate word frequency, total tokens and total documentsfor uid, document, tags in documents:# Extract text, if necessary@@ -232,7 +232,7 @@ class TFIDF(Scoring):if self.terms:self.terms.close()- def hasterms(self):+ def issparse(self):return self.terms is not Nonedef isnormalized(self):@@ -349,6 +349,9 @@ class TFIDF(Scoring):resolved results"""+ # Convert to Python values+ scores = [(x, float(score)) for x, score in scores]+if self.documents:return [{"id": x, "text": self.documents[x], "score": score} for x, score in scores]diff --git a/src/python/txtai/util/__init__.py b/src/python/txtai/util/__init__.pyindex ef78ad70..4cbc088d 100644--- a/src/python/txtai/util/__init__.py+++ b/src/python/txtai/util/__init__.py@@ -3,4 +3,5 @@ Utility imports"""from .resolver import Resolver+from .sparsearray import SparseArrayfrom .template import TemplateFormatterdiff --git a/src/python/txtai/util/sparsearray.py b/src/python/txtai/util/sparsearray.pynew file mode 100644index 00000000..dbd66a05--- /dev/null+++ b/src/python/txtai/util/sparsearray.py@@ -0,0 +1,62 @@+"""+SparseArray module+"""++import numpy as np++# Conditional import+try:+ from scipy.sparse import csr_matrix++ SCIPY = True+except ImportError:+ SCIPY = False+++class SparseArray:+ """+ Methods to load and save sparse arrays to file.+ """++ def __init__(self):+ """+ Creates a SparseArray instance.+ """++ if not SCIPY:+ raise ImportError("SciPy is not available - install scipy to enable")++ def load(self, f):+ """+ Loads a sparse array from file.++ Args:+ f: input file handle++ Returns:+ sparse array+ """++ # Load raw data+ data, indices, indptr, shape = (+ np.load(f, allow_pickle=False),+ np.load(f, allow_pickle=False),+ np.load(f, allow_pickle=False),+ np.load(f, allow_pickle=False),+ )++ # Load data into sparse array+ return csr_matrix((data, indices, indptr), shape=shape)++ def save(self, f, array):+ """+ Saves a sparse array to file.++ Args:+ f: output file handle+ array: sparse array+ """++ # Save sparse array to file+ for x in [array.data, array.indices, array.indptr, array.shape]:+ np.save(f, x, allow_pickle=False)diff --git a/src/python/txtai/vectors/__init__.py b/src/python/txtai/vectors/__init__.pyindex fba44cb3..13c64781 100644--- a/src/python/txtai/vectors/__init__.py+++ b/src/python/txtai/vectors/__init__.py@@ -1,14 +1,8 @@"""-Vectors import+Vectors imports"""from .base import Vectors-from .external import External-from .factory import VectorsFactory-from .huggingface import HFVectors-from .litellm import LiteLLM-from .llama import LlamaCpp-from .m2v import Model2Vec+from .dense import *from .recovery import Recovery-from .sbert import STVectors-from .words import WordVectors+from .sparse import *diff --git a/src/python/txtai/vectors/base.py b/src/python/txtai/vectors/base.pyindex 989ea0d6..733d6055 100644--- a/src/python/txtai/vectors/base.py+++ b/src/python/txtai/vectors/base.py@@ -70,12 +70,13 @@ class Vectors:raise NotImplementedError- def encode(self, data):+ def encode(self, data, category=None):"""Encodes a batch of data using vector model.Args:data: batch of data+ category: optional category for instruction-based embeddingsReturn:transformed data@@ -116,14 +117,14 @@ class Vectors:checkpoint: optional checkpoint directory, enables indexing restartReturns:- (ids, dimensions, stream)+ (ids, dimensions, batches, stream)"""ids, dimensions, batches, stream = [], None, 0, None# Generate recovery config if checkpoint is setvectorsid = self.vectorsid() if checkpoint else None- recovery = Recovery(checkpoint, vectorsid) if checkpoint else None+ recovery = Recovery(checkpoint, vectorsid, self.loadembeddings) if checkpoint else None# Convert all documents to embedding arrays, stream embeddings to disk to control memory usagewith self.spool(checkpoint, vectorsid) as output:@@ -148,6 +149,42 @@ class Vectors:return (ids, dimensions, batches, stream)+ def vectors(self, documents, batchsize=500, checkpoint=None, buffer=None, dtype=None):+ """+ Bulk encodes documents into vectors using index(). Return the data as a mmap-ed array.++ Args:+ documents: list of (id, data, tags)+ batchsize: index batch size+ checkpoint: optional checkpoint directory, enables indexing restart+ buffer: file path used for memmap buffer+ dtype: dtype for buffer++ Returns:+ (ids, dimensions, embeddings)+ """++ # Consume stream and transform documents to vectors+ ids, dimensions, batches, stream = self.index(documents, batchsize, checkpoint)++ # Check that embeddings are available and load as a memmap+ embeddings = None+ if ids:+ # Write batches+ embeddings = np.memmap(buffer, dtype=dtype, shape=(len(ids), dimensions), mode="w+")+ with open(stream, "rb") as queue:+ x = 0+ for _ in range(batches):+ batch = self.loadembeddings(queue)+ embeddings[x : x + batch.shape[0]] = batch+ x += batch.shape[0]++ # Remove temporary file (if checkpointing is disabled)+ if not checkpoint:+ os.remove(stream)++ return (ids, dimensions, embeddings)+def close(self):"""Closes this vectors instance.@@ -188,7 +225,22 @@ class Vectors:if documents and isinstance(documents[0], np.ndarray):return np.array(documents, dtype=np.float32)- return self.vectorize(documents)+ return self.vectorize(documents, category)++ def dot(self, queries, data):+ """+ Calculates the dot product similarity between queries and documents. This method+ assumes each of the inputs are normalized.++ Args:+ queries: queries+ data: search data++ Returns:+ dot product scores+ """++ return np.dot(queries, data.T).tolist()def vectorsid(self):"""@@ -246,10 +298,10 @@ class Vectors:# Attempt to read embeddings from a recovery fileembeddings = recovery() if recovery else None- embeddings = self.vectorize(documents) if embeddings is None else embeddings+ embeddings = self.vectorize(documents, "data") if embeddings is None else embeddingsif embeddings is not None:dimensions = embeddings.shape[1]- np.save(output, embeddings)+ self.saveembeddings(output, embeddings)return (ids, dimensions)@@ -299,7 +351,7 @@ class Vectors:return data- def vectorize(self, data):+ def vectorize(self, data, category=None):"""Runs data vectorization, which consists of the following steps.@@ -310,13 +362,17 @@ class Vectors:Args:data: input data+ category: category for instruction-based embeddingsReturns:embeddings vectors"""+ # Default instruction category+ category = category if category else "query"+# Transform data into vectors- embeddings = self.encode(data)+ embeddings = self.encode(data, category)if embeddings is not None:# Truncate embeddings, if necessary@@ -324,7 +380,7 @@ class Vectors:embeddings = self.truncate(embeddings)# Normalize data- self.normalize(embeddings)+ embeddings = self.normalize(embeddings)# Apply quantization, if necessaryif self.qbits:@@ -332,6 +388,30 @@ class Vectors:return embeddings+ def loadembeddings(self, f):+ """+ Loads embeddings from file.++ Args:+ f: file to load from++ Returns:+ embeddings+ """++ return np.load(f, allow_pickle=False)++ def saveembeddings(self, f, embeddings):+ """+ Saves embeddings to output.++ Args:+ f: output file+ embeddings: embeddings to save+ """++ np.save(f, embeddings, allow_pickle=False)+def truncate(self, embeddings):"""Truncates embeddings to the configured dimensionality.@@ -354,6 +434,9 @@ class Vectors:Args:embeddings: input embeddings++ Returns:+ embeddings"""# Calculation is different for matrices vs vectors@@ -362,6 +445,8 @@ class Vectors:else:embeddings /= np.linalg.norm(embeddings)+ return embeddings+def quantize(self, embeddings):"""Quantizes embeddings using scalar quantization.diff --git a/src/python/txtai/vectors/dense/__init__.py b/src/python/txtai/vectors/dense/__init__.pynew file mode 100644index 00000000..c7831436--- /dev/null+++ b/src/python/txtai/vectors/dense/__init__.py@@ -0,0 +1,12 @@+"""+Dense vectors imports+"""++from .external import External+from .factory import VectorsFactory+from .huggingface import HFVectors+from .litellm import LiteLLM+from .llama import LlamaCpp+from .m2v import Model2Vec+from .sbert import STVectors+from .words import WordVectorsdiff --git a/src/python/txtai/vectors/external.py b/src/python/txtai/vectors/dense/external.pysimilarity index 93%rename from src/python/txtai/vectors/external.pyrename to src/python/txtai/vectors/dense/external.pyindex bb2d7571..f8b10b74 100644--- a/src/python/txtai/vectors/external.py+++ b/src/python/txtai/vectors/dense/external.py@@ -6,9 +6,9 @@ import typesimport numpy as np-from ..util import Resolver+from ...util import Resolver-from .base import Vectors+from ..base import Vectorsclass External(Vectors):@@ -25,7 +25,7 @@ class External(Vectors):def loadmodel(self, path):return None- def encode(self, data):+ def encode(self, data, category=None):# Call external transform function, if available and data not already an array# Batching is handed by the external transform functionif self.transform and data and not isinstance(data[0], np.ndarray):diff --git a/src/python/txtai/vectors/factory.py b/src/python/txtai/vectors/dense/factory.pysimilarity index 97%rename from src/python/txtai/vectors/factory.pyrename to src/python/txtai/vectors/dense/factory.pyindex 5ef28fe3..ec6d88b4 100644--- a/src/python/txtai/vectors/factory.py+++ b/src/python/txtai/vectors/dense/factory.py@@ -2,7 +2,7 @@Factory module"""-from ..util import Resolver+from ...util import Resolverfrom .external import Externalfrom .huggingface import HFVectors@@ -15,7 +15,7 @@ from .words import WordVectorsclass VectorsFactory:"""- Methods to create Vectors models.+ Methods to create dense vector models."""@staticmethoddiff --git a/src/python/txtai/vectors/huggingface.py b/src/python/txtai/vectors/dense/huggingface.pysimilarity index 90%rename from src/python/txtai/vectors/huggingface.pyrename to src/python/txtai/vectors/dense/huggingface.pyindex 3db9520c..bbb563d9 100644--- a/src/python/txtai/vectors/huggingface.py+++ b/src/python/txtai/vectors/dense/huggingface.py@@ -2,9 +2,9 @@Hugging Face module"""-from ..models import Models, PoolingFactory+from ...models import Models, PoolingFactory-from .base import Vectors+from ..base import Vectorsclass HFVectors(Vectors):@@ -39,6 +39,6 @@ class HFVectors(Vectors):})- def encode(self, data):+ def encode(self, data, category=None):# Encode data using vectors modelreturn self.model.encode(data, batch=self.encodebatch)diff --git a/src/python/txtai/vectors/litellm.py b/src/python/txtai/vectors/dense/litellm.pysimilarity index 95%rename from src/python/txtai/vectors/litellm.pyrename to src/python/txtai/vectors/dense/litellm.pyindex 20d812c9..ee6292ea 100644--- a/src/python/txtai/vectors/litellm.py+++ b/src/python/txtai/vectors/dense/litellm.py@@ -12,7 +12,7 @@ try:except ImportError:LITELLM = False-from .base import Vectors+from ..base import Vectorsclass LiteLLM(Vectors):@@ -57,7 +57,7 @@ class LiteLLM(Vectors):def loadmodel(self, path):return None- def encode(self, data):+ def encode(self, data, category=None):# Call external embeddings API using LiteLLM# Batching is handled server-sideresponse = api.embedding(model=self.config.get("path"), input=data, **self.config.get("vectors", {}))diff --git a/src/python/txtai/vectors/llama.py b/src/python/txtai/vectors/dense/llama.pysimilarity index 96%rename from src/python/txtai/vectors/llama.pyrename to src/python/txtai/vectors/dense/llama.pyindex ac6d3fd5..9e5c7470 100644--- a/src/python/txtai/vectors/llama.py+++ b/src/python/txtai/vectors/dense/llama.py@@ -16,7 +16,7 @@ try:except ImportError:LLAMA_CPP = False-from .base import Vectors+from ..base import Vectorsclass LlamaCpp(Vectors):@@ -58,7 +58,7 @@ class LlamaCpp(Vectors):# Create llama.cpp instancereturn Llama(path, n_ctx=0, verbose=modelargs.pop("verbose", False), embedding=True, **modelargs)- def encode(self, data):+ def encode(self, data, category=None):# Generate embeddings and return as a NumPy array# llama-cpp-python has it's own batching built-in using n_batch parameterreturn np.array(self.model.embed(data), dtype=np.float32)diff --git a/src/python/txtai/vectors/m2v.py b/src/python/txtai/vectors/dense/m2v.pysimilarity index 95%rename from src/python/txtai/vectors/m2v.pyrename to src/python/txtai/vectors/dense/m2v.pyindex 46c2f18d..fa44e4a7 100644--- a/src/python/txtai/vectors/m2v.py+++ b/src/python/txtai/vectors/dense/m2v.py@@ -15,7 +15,7 @@ try:except ImportError:MODEL2VEC = False-from .base import Vectors+from ..base import Vectorsclass Model2Vec(Vectors):@@ -59,7 +59,7 @@ class Model2Vec(Vectors):def loadmodel(self, path):return StaticModel.from_pretrained(path)- def encode(self, data):+ def encode(self, data, category=None):# Additional model argumentsmodelargs = self.config.get("vectors", {})diff --git a/src/python/txtai/vectors/sbert.py b/src/python/txtai/vectors/dense/sbert.pysimilarity index 62%rename from src/python/txtai/vectors/sbert.pyrename to src/python/txtai/vectors/dense/sbert.pyindex e98bc909..93906c8f 100644--- a/src/python/txtai/vectors/sbert.py+++ b/src/python/txtai/vectors/dense/sbert.py@@ -1,5 +1,5 @@"""-SentenceTransformers module+Sentence Transformers module"""# Conditional import@@ -10,9 +10,9 @@ try:except ImportError:SENTENCE_TRANSFORMERS = False-from ..models import Models+from ...models import Models-from .base import Vectors+from ..base import Vectorsclass STVectors(Vectors):@@ -48,8 +48,8 @@ class STVectors(Vectors):# Additional model argumentsmodelargs = self.config.get("vectors", {})- # Build embeddings with sentence-transformers- model = SentenceTransformer(path, device=Models.device(deviceid), **modelargs)+ # Load sentence-transformers encoder+ model = self.loadencoder(path, device=Models.device(deviceid), **modelargs)# Start process pool for multiple GPUsif pool:@@ -58,9 +58,15 @@ class STVectors(Vectors):# Return modelreturn model- def encode(self, data):- # Encode with sentence transformers- return self.model.encode(data, pool=self.pool, batch_size=self.encodebatch)+ def encode(self, data, category=None):+ # Get encode method based on input category+ encode = self.model.encode_query if category == "query" else self.model.encode_document if category == "data" else self.model.encode++ # Additional encoding arguments+ encodeargs = self.config.get("encodeargs", {})++ # Encode with sentence transformers encoder+ return encode(data, pool=self.pool, batch_size=self.encodebatch, **encodeargs)def close(self):# Close pool before model is closed in parent method@@ -69,3 +75,18 @@ class STVectors(Vectors):self.pool = Nonesuper().close()++ def loadencoder(self, path, device, **kwargs):+ """+ Loads the embeddings encoder model from path.++ Args:+ path: model path+ device: tensor device+ kwargs: additional keyword args++ Returns:+ embeddings encoder+ """++ return SentenceTransformer(path, device=device, **kwargs)diff --git a/src/python/txtai/vectors/words.py b/src/python/txtai/vectors/dense/words.pysimilarity index 97%rename from src/python/txtai/vectors/words.pyrename to src/python/txtai/vectors/dense/words.pyindex 81e6700a..b748e990 100644--- a/src/python/txtai/vectors/words.py+++ b/src/python/txtai/vectors/dense/words.py@@ -22,9 +22,9 @@ try:except ImportError:STATICVECTORS = False-from ..pipeline import Tokenizer+from ...pipeline import Tokenizer-from .base import Vectors+from ..base import Vectors# Logging configurationlogger = logging.getLogger(__name__)@@ -128,7 +128,7 @@ class WordVectors(Vectors):def loadmodel(self, path):return StaticVectors(path)- def encode(self, data):+ def encode(self, data, category=None):# Iterate over each data element, tokenize (if necessary) and build an aggregated embeddings vectorembeddings = []for tokens in data:@@ -181,14 +181,14 @@ class WordVectors(Vectors):embeddings.append(embedding)if len(embeddings) == batchsize:- np.save(output, np.array(embeddings, dtype=np.float32))+ np.save(output, np.array(embeddings, dtype=np.float32), allow_pickle=False)batches += 1embeddings = []# Final embeddings batchif embeddings:- np.save(output, np.array(embeddings, dtype=np.float32))+ np.save(output, np.array(embeddings, dtype=np.float32), allow_pickle=False)batches += 1return (ids, dimensions, batches, stream)diff --git a/src/python/txtai/vectors/recovery.py b/src/python/txtai/vectors/recovery.pyindex 74930404..f0dfb7c8 100644--- a/src/python/txtai/vectors/recovery.py+++ b/src/python/txtai/vectors/recovery.py@@ -5,24 +5,23 @@ Recovery moduleimport osimport shutil-import numpy as np-class Recovery:"""Vector embeddings recovery. This class handles streaming embeddings from a vector checkpoint file."""- def __init__(self, checkpoint, vectorsid):+ def __init__(self, checkpoint, vectorsid, load):"""Creates a Recovery instance.Args:checkpoint: checkpoint directoryvectorsid: vectors uid for current configuration+ load: load embeddings method"""- self.spool, self.path = None, None+ self.spool, self.path, self.load = None, None, load# Get unique file idpath = f"{checkpoint}/{vectorsid}"@@ -46,7 +45,7 @@ class Recovery:"""try:- return np.load(self.spool) if self.spool else None+ return self.load(self.spool) if self.spool else Noneexcept EOFError:# End of spool file, cleanupself.spool.close()diff --git a/src/python/txtai/vectors/sparse/__init__.py b/src/python/txtai/vectors/sparse/__init__.pynew file mode 100644index 00000000..dfce5a8a--- /dev/null+++ b/src/python/txtai/vectors/sparse/__init__.py@@ -0,0 +1,7 @@+"""+Sparse vectors imports+"""++from .base import SparseVectors+from .factory import SparseVectorsFactory+from .sbert import SparseSTVectorsdiff --git a/src/python/txtai/vectors/sparse/base.py b/src/python/txtai/vectors/sparse/base.pynew file mode 100644index 00000000..11736919--- /dev/null+++ b/src/python/txtai/vectors/sparse/base.py@@ -0,0 +1,76 @@+"""+SparseVectors module+"""++# Conditional import+try:+ from scipy.sparse import csr_matrix, vstack+ from sklearn.preprocessing import normalize+ from sklearn.utils.extmath import safe_sparse_dot++ SPARSE = True+except ImportError:+ SPARSE = False++from ...util import SparseArray+from ..base import Vectors+++# pylint: disable=W0223+class SparseVectors(Vectors):+ """+ Base class for sparse vector models. Vector models transform input content into sparse arrays.+ """++ def __init__(self, config, scoring, models):+ # Check before parent constructor since it calls loadmodel+ if not SPARSE:+ raise ImportError('SparseVectors is not available - install "vectors" extra to enable')++ super().__init__(config, scoring, models)++ def encode(self, data, category=None):+ # Encode data to embeddings+ embeddings = super().encode(data, category)++ # Get sparse torch vector attributes+ embeddings = embeddings.cpu().coalesce()+ indices = embeddings.indices().numpy()+ values = embeddings.values().numpy()++ # Return as SciPy CSR Matrix+ return csr_matrix((values, indices), shape=embeddings.size())++ def vectors(self, documents, batchsize=500, checkpoint=None, buffer=None, dtype=None):+ # Run indexing+ ids, dimensions, batches, stream = self.index(documents, batchsize, checkpoint)++ # Rebuild sparse array+ embeddings = None+ with open(stream, "rb") as queue:+ for _ in range(batches):+ # Read in array batch+ data = self.loadembeddings(queue)+ embeddings = vstack((embeddings, data)) if embeddings is not None else data++ # Return sparse array+ return (ids, dimensions, embeddings)++ def dot(self, queries, data):+ return safe_sparse_dot(queries, data.T, dense_output=True).tolist()++ def loadembeddings(self, f):+ return SparseArray().load(f)++ def saveembeddings(self, f, embeddings):+ SparseArray().save(f, embeddings)++ def truncate(self, embeddings):+ raise ValueError("Truncate is not supported for sparse vectors")++ def normalize(self, embeddings):+ # Normalize in place using method that supports sparse vectors+ return normalize(embeddings, copy=False)++ def quantize(self, embeddings):+ raise ValueError("Quantize is not supported for sparse vectors")diff --git a/src/python/txtai/vectors/sparse/factory.py b/src/python/txtai/vectors/sparse/factory.pynew file mode 100644index 00000000..e9240716--- /dev/null+++ b/src/python/txtai/vectors/sparse/factory.py@@ -0,0 +1,55 @@+"""+Factory module+"""++from ...util import Resolver++from .sbert import SparseSTVectors+++class SparseVectorsFactory:+ """+ Methods to create sparse vector models.+ """++ @staticmethod+ def create(config, models=None):+ """+ Create a Vectors model instance.++ Args:+ config: vector configuration+ models: models cache++ Returns:+ Vectors+ """++ # Get vector method+ method = config.get("method", "sentence-transformers")++ # Sentence Transformers vectors+ if method == "sentence-transformers":+ return SparseSTVectors(config, None, models) if config and config.get("path") else None++ # Resolve custom method+ return SparseVectorsFactory.resolve(method, config, models) if method else None++ @staticmethod+ def resolve(backend, config, models):+ """+ Attempt to resolve a custom backend.++ Args:+ backend: backend class+ config: vector configuration+ models: models cache++ Returns:+ Vectors+ """++ try:+ return Resolver()(backend)(config, None, models)+ except Exception as e:+ raise ImportError(f"Unable to resolve sparse vectors backend: '{backend}'") from ediff --git a/src/python/txtai/vectors/sparse/sbert.py b/src/python/txtai/vectors/sparse/sbert.pynew file mode 100644index 00000000..4c76e0a6--- /dev/null+++ b/src/python/txtai/vectors/sparse/sbert.py@@ -0,0 +1,30 @@+"""+Sparse Sentence Transformers module+"""++# Conditional import+try:+ from sentence_transformers import SparseEncoder++ SENTENCE_TRANSFORMERS = True+except ImportError:+ SENTENCE_TRANSFORMERS = False++from ..dense.sbert import STVectors+from .base import SparseVectors+++class SparseSTVectors(SparseVectors, STVectors):+ """+ Builds sparse vectors using sentence-transformers (aka SBERT).+ """++ def __init__(self, config, scoring, models):+ # Check before parent constructor since it calls loadmodel+ if not SENTENCE_TRANSFORMERS:+ raise ImportError('sentence-transformers is not available - install "vectors" extra to enable')++ super().__init__(config, scoring, models)++ def loadencoder(self, path, device, **kwargs):+ return SparseEncoder(path, device=device, **kwargs)diff --git a/test/python/testann/__init__.py b/test/python/testann/__init__.pynew file mode 100644index 00000000..e69de29bdiff --git a/test/python/testann.py b/test/python/testann/testdense.pysimilarity index 98%rename from test/python/testann.pyrename to test/python/testann/testdense.pyindex de31ebe9..5f689a11 100644--- a/test/python/testann.py+++ b/test/python/testann/testdense.py@@ -1,5 +1,5 @@"""-ANN module tests+Dense ANN module tests"""import os@@ -17,9 +17,9 @@ from txtai.serialize import SerializeFactory# pylint: disable=R0904-class TestANN(unittest.TestCase):+class TestDense(unittest.TestCase):"""- ANN tests.+ Dense ANN tests."""def testAnnoy(self):@@ -79,10 +79,10 @@ class TestANN(unittest.TestCase):# pylint: disable=C0415, W0611# Force reload of class- name = "txtai.ann.faiss"+ name = "txtai.ann.dense.faiss"module = sys.modules[name]del sys.modules[name]- import txtai.ann.faiss+ import txtai.ann.dense.faiss# Run testsself.runTests("faiss")diff --git a/test/python/testann/testsparse.py b/test/python/testann/testsparse.pynew file mode 100644index 00000000..403f6f39--- /dev/null+++ b/test/python/testann/testsparse.py@@ -0,0 +1,131 @@+"""+Sparse ANN module tests+"""++import os+import tempfile+import unittest++from unittest.mock import patch++from scipy.sparse import random+from sklearn.preprocessing import normalize++from txtai.ann import SparseANNFactory+++class TestSparse(unittest.TestCase):+ """+ Sparse ANN tests.+ """++ def testCustomBackend(self):+ """+ Test resolving a custom backend+ """++ self.assertIsNotNone(SparseANNFactory.create({"backend": "txtai.ann.IVFSparse"}))++ def testCustomBackendNotFound(self):+ """+ Test resolving an unresolvable backend+ """++ with self.assertRaises(ImportError):+ SparseANNFactory.create({"backend": "notfound.ann"})++ def testIVFSparse(self):+ """+ Test IVFSparse backend+ """++ # Generate test record+ insert = self.generate(500, 30522)+ append = self.generate(500, 30522)++ # Count of records+ count = insert.shape[0] + append.shape[0]++ # Create ANN+ path = os.path.join(tempfile.gettempdir(), "ivfsparse")+ ann = SparseANNFactory.create({"backend": "ivfsparse", "ivfsparse": {"nlist": 2, "nprobe": 2, "sample": 1.0}})++ # Test indexing+ ann.index(insert)+ ann.append(append)++ # Validate search results+ results = [x[0] for x in ann.search(insert[5], 10)[0]]+ self.assertIn(5, results)++ # Validate save/load/delete+ ann.save(path)+ ann.load(path)++ # Validate count+ self.assertEqual(ann.count(), count)++ # Test delete+ ann.delete([0])+ self.assertEqual(ann.count(), count - 1)++ # Re-validate search results+ results = [x[0] for x in ann.search(append[0], 10)[0]]+ self.assertIn(insert.shape[0], results)++ # Close ANN+ ann.close()++ @patch("sqlalchemy.orm.Query.limit")+ def testPGSparse(self, query):+ """+ Test Sparse Postgres backend+ """++ # Generate test record+ data = self.generate(1, 240)++ # Mock database query+ query.return_value = [(x, -1.0) for x in range(data.shape[0])]++ # Create ANN+ path = os.path.join(tempfile.gettempdir(), "pgsparse.sqlite")+ ann = SparseANNFactory.create({"backend": "pgsparse", "dimensions": 240, "pgsparse": {"url": f"sqlite:///{path}", "schema": "txtai"}})++ # Test indexing+ ann.index(data)+ ann.append(data)++ # Validate search results+ self.assertEqual(ann.search(data, 1), [[(0, 1.0)]])++ # Validate save/load/delete+ ann.save(None)+ ann.load(None)++ # Validate count+ self.assertEqual(ann.count(), 2)++ # Test delete+ ann.delete([0])+ self.assertEqual(ann.count(), 1)++ # Close ANN+ ann.close()++ def generate(self, m, n):+ """+ Generates random normalized sparse data.++ Args:+ m, n: shape of the matrix++ Returns:+ csr matrix+ """++ # Generate random csr matrix+ data = random(m, n, format="csr")++ # Normalize and return+ return normalize(data)diff --git a/test/python/testembeddings.py b/test/python/testembeddings.pyindex 3e157d92..27816d91 100644--- a/test/python/testembeddings.py+++ b/test/python/testembeddings.py@@ -486,6 +486,10 @@ class TestEmbeddings(unittest.TestCase):uid = embeddings.search("lottery ticket", 1)[0][0]self.assertEqual(uid, 4)+ # Test similarity+ uid = embeddings.similarity("lottery ticket", self.data)[0][0]+ self.assertEqual(uid, 4)+# Update datadata[0] = (0, "Feel good story: baby panda born", None)embeddings.upsert([data[0]])diff --git a/test/python/testoptional.py b/test/python/testoptional.pyindex 54f42511..f1a88b2e 100644--- a/test/python/testoptional.py+++ b/test/python/testoptional.py@@ -107,8 +107,9 @@ class TestOptional(unittest.TestCase):Test missing ANN dependencies"""- from txtai.ann import ANNFactory+ from txtai.ann import ANNFactory, SparseANNFactory+ # Test dense methodswith self.assertRaises(ImportError):ANNFactory.create({"backend": "annoy"})@@ -121,6 +122,13 @@ class TestOptional(unittest.TestCase):with self.assertRaises(ImportError):ANNFactory.create({"backend": "sqlite"})+ # Test sparse methods+ with self.assertRaises(ImportError):+ SparseANNFactory.create({"backend": "ivfsparse"})++ with self.assertRaises(ImportError):+ SparseANNFactory.create({"backend": "pgsparse"})+def testApi(self):"""Test missing api dependencies@@ -294,24 +302,20 @@ class TestOptional(unittest.TestCase):Test missing scoring dependencies"""- from txtai.scoring import IVFFlat, ScoringFactory+ from txtai.scoring import ScoringFactorywith self.assertRaises(ImportError):ScoringFactory.create({"method": "pgtext"})- with self.assertRaises(ImportError):- ScoringFactory.create({"method": "sparse"})-- with self.assertRaises(ImportError):- IVFFlat({})-def testVectors(self):"""Test missing vector dependencies"""- from txtai.vectors import VectorsFactory+ from txtai.vectors import SparseVectors, VectorsFactory, SparseVectorsFactory+ from txtai.util import SparseArray+ # Test dense vectorswith self.assertRaises(ImportError):VectorsFactory.create({"method": "litellm", "path": "huggingface/sentence-transformers/all-MiniLM-L6-v2"}, None)@@ -331,6 +335,16 @@ class TestOptional(unittest.TestCase):model = VectorsFactory.create({"path": "sentence-transformers/all-MiniLM-L6-v2"}, None)self.assertIsNotNone(model)+ # Test sparse vectors+ with self.assertRaises(ImportError):+ SparseVectors(None, None, None)++ with self.assertRaises(ImportError):+ SparseVectorsFactory.create({"method": "sentence-transformers", "path": "sparse-encoder-testing/splade-bert-tiny-nq"}, None)++ with self.assertRaises(ImportError):+ SparseArray()+def testWorkflow(self):"""Test missing workflow dependenciesdiff --git a/test/python/testscoring/__init__.py b/test/python/testscoring/__init__.pynew file mode 100644index 00000000..e69de29bdiff --git a/test/python/testscoring.py b/test/python/testscoring/testkeyword.pysimilarity index 78%rename from test/python/testscoring.pyrename to test/python/testscoring/testkeyword.pyindex aa5b16b3..6f0b8446 100644--- a/test/python/testscoring.py+++ b/test/python/testscoring/testkeyword.py@@ -1,5 +1,5 @@"""-Scoring module tests+Keyword scoring tests"""import os@@ -12,9 +12,9 @@ from txtai.scoring import ScoringFactory, Scoring# pylint: disable=R0904-class TestScoring(unittest.TestCase):+class TestKeyword(unittest.TestCase):"""- Scoring tests.+ Sparse keyword scoring tests."""@classmethod@@ -73,7 +73,7 @@ class TestScoring(unittest.TestCase):self.assertRaises(NotImplementedError, scoring.load, None)self.assertRaises(NotImplementedError, scoring.save, None)self.assertRaises(NotImplementedError, scoring.close)- self.assertRaises(NotImplementedError, scoring.hasterms)+ self.assertRaises(NotImplementedError, scoring.issparse)self.assertRaises(NotImplementedError, scoring.isnormalized)@patch("sqlalchemy.orm.Query.params")@@ -109,8 +109,8 @@ class TestScoring(unittest.TestCase):scoring.delete([0])self.assertEqual(scoring.count(), len(self.data) - 1)- # PGText is a normalized terms index- self.assertTrue(scoring.hasterms() and scoring.isnormalized())+ # PGText is a normalized sparse index+ self.assertTrue(scoring.issparse() and scoring.isnormalized())self.assertIsNone(scoring.weights("This is a test".split()))# Close scoring@@ -123,93 +123,6 @@ class TestScoring(unittest.TestCase):self.runTests("sif")- def testSparse(self):- """- Test sparse vectors- """-- # Models cache- models = {}-- # Test sparse scoring- scoring = ScoringFactory.create({"method": "sparse", "path": "sparse-encoder-testing/splade-bert-tiny-nq"}, models=models)- scoring.index((uid, {"text": text}, tags) for uid, text, tags in self.data)-- # Run search and validate correct result returned- index, _ = scoring.search("lottery ticket", 1)[0]- self.assertEqual(index, 4)-- # Run batch search- index, _ = scoring.batchsearch(["lottery ticket"], 1)[0][0]- self.assertEqual(index, 4)-- # Validate count- self.assertEqual(scoring.count(), len(self.data))-- # Test delete- scoring.delete([4])- self.assertEqual(scoring.count(), len(self.data) - 1)-- # Run search after delete- index, _ = scoring.search("lottery ticket", 1)[0]- self.assertEqual(index, 5)-- # Sparse vectors is a normalized terms index- self.assertTrue(scoring.hasterms() and scoring.isnormalized())- self.assertIsNone(scoring.weights("This is a test".split()))-- # Close scoring- scoring.close()-- # Test model caching- scoring = ScoringFactory.create({"method": "sparse", "path": "sparse-encoder-testing/splade-bert-tiny-nq"}, models=models)- self.assertIsNotNone(scoring.model)- scoring.close()-- def testSparseEmpty(self):- """- Test empty sparse vectors- """-- scoring = ScoringFactory.create({"method": "sparse", "path": "sparse-encoder-testing/splade-bert-tiny-nq"})- scoring.upsert((uid, {"text": text}, tags) for uid, text, tags in self.data)- self.assertEqual(scoring.count(), len(self.data))-- @patch("torch.cuda.device_count")- def testSparseGPU(self, count):- """- Test sparse vectors with GPU encoding- """-- # Mock accelerator count- count.return_value = 2-- # Test multiple gpus- scoring = ScoringFactory.create({"method": "sparse", "path": "sparse-encoder-testing/splade-bert-tiny-nq", "gpu": "all"})- self.assertIsNotNone(scoring)- scoring.close()-- def testSparseIVFFlat(self):- """- Test sparse vectors with IVFFlat clustering- """-- # Expand dataset- data = self.data * 1000-- # Test higher volume IVFFlat index with clustering- config = {"method": "sparse", "path": "sparse-encoder-testing/splade-bert-tiny-nq", "ivf": {"sample": 1.0}}- scoring = ScoringFactory.create(config)- scoring.index((uid, {"text": text}, tags) for uid, text, tags in data)-- # Recreate index- self.save(scoring, config, "scoring.sparse.index")-- # Run search and validate correct result returned- results = scoring.search("lottery ticket", 1)- self.assertGreater(len(results), 0)- scoring.close()-def testTFIDF(self):"""Test tfidfdiff --git a/test/python/testscoring/testsparse.py b/test/python/testscoring/testsparse.pynew file mode 100644index 00000000..7471b212--- /dev/null+++ b/test/python/testscoring/testsparse.py@@ -0,0 +1,136 @@+"""+Sparse module tests+"""++import os+import tempfile+import unittest++from unittest.mock import patch++from txtai.scoring import ScoringFactory+++# pylint: disable=R0904+class TestSparse(unittest.TestCase):+ """+ Sparse vector scoring tests.+ """++ @classmethod+ def setUpClass(cls):+ """+ Initialize test data.+ """++ cls.data = [+ "US tops 5 million confirmed virus cases",+ "Canada's last fully intact ice shelf has suddenly collapsed, forming a Manhattan-sized iceberg",+ "Beijing mobilises invasion craft along coast as Taiwan tensions escalate",+ "The National Park Service warns against sacrificing slower friends in a bear attack",+ "Maine man wins $1M from $25 lottery ticket",+ "wins wins wins",+ "Make huge profits without work, earn up to $100,000 a day",+ ]++ cls.data = [(uid, x, None) for uid, x in enumerate(cls.data)]++ def testGeneral(self):+ """+ Test general sparse vector operations+ """++ # Models cache+ models = {}++ # Test sparse scoring+ scoring = ScoringFactory.create({"method": "sparse", "path": "sparse-encoder-testing/splade-bert-tiny-nq"}, models=models)+ scoring.index((uid, {"text": text}, tags) for uid, text, tags in self.data)++ # Run search and validate correct result returned+ index, _ = scoring.search("lottery ticket", 1)[0]+ self.assertEqual(index, 4)++ # Run batch search+ index, _ = scoring.batchsearch(["lottery ticket"], 1)[0][0]+ self.assertEqual(index, 4)++ # Validate count+ self.assertEqual(scoring.count(), len(self.data))++ # Test delete+ scoring.delete([4])+ self.assertEqual(scoring.count(), len(self.data) - 1)++ # Run search after delete+ index, _ = scoring.search("lottery ticket", 1)[0]+ self.assertEqual(index, 5)++ # Sparse vectors is a normalized sparse index+ self.assertTrue(scoring.issparse() and scoring.isnormalized())+ self.assertIsNone(scoring.weights("This is a test".split()))++ # Close scoring+ scoring.close()++ # Test model caching+ scoring = ScoringFactory.create({"method": "sparse", "path": "sparse-encoder-testing/splade-bert-tiny-nq"}, models=models)+ self.assertIsNotNone(scoring.model)+ scoring.close()++ def testEmpty(self):+ """+ Test empty sparse vectors+ """++ scoring = ScoringFactory.create({"method": "sparse", "path": "sparse-encoder-testing/splade-bert-tiny-nq"})+ scoring.upsert((uid, {"text": text}, tags) for uid, text, tags in self.data)+ self.assertEqual(scoring.count(), len(self.data))++ @patch("torch.cuda.device_count")+ def testGPU(self, count):+ """+ Test sparse vectors with GPU encoding+ """++ # Mock accelerator count+ count.return_value = 2++ # Test multiple gpus+ scoring = ScoringFactory.create({"method": "sparse", "path": "sparse-encoder-testing/splade-bert-tiny-nq", "gpu": "all"})+ self.assertIsNotNone(scoring)+ scoring.close()++ def testIVFFlat(self):+ """+ Test sparse vectors with IVFFlat clustering+ """++ # Expand dataset+ data = self.data * 1000++ # Test higher volume IVFFlat index with clustering+ config = {+ "method": "sparse",+ "vectormethod": "sentence-transformers",+ "path": "sparse-encoder-testing/splade-bert-tiny-nq",+ "ivfsparse": {"sample": 1.0},+ }+ scoring = ScoringFactory.create(config)+ scoring.index((uid, {"text": text}, tags) for uid, text, tags in data)++ # Generate temp file path+ index = os.path.join(tempfile.gettempdir(), "scoring")+ os.makedirs(index, exist_ok=True)++ # Save scoring instance+ scoring.save(f"{index}/scoring.sparse.index")++ # Reload scoring instance+ scoring = ScoringFactory.create(config)+ scoring.load(f"{index}/scoring.sparse.index")++ # Run search and validate correct result returned+ results = scoring.search("lottery ticket", 1)+ self.assertGreater(len(results), 0)+ scoring.close()diff --git a/test/python/testvectors/testdense/__init__.py b/test/python/testvectors/testdense/__init__.pynew file mode 100644index 00000000..e69de29bdiff --git a/test/python/testvectors/testcustom.py b/test/python/testvectors/testdense/testcustom.pysimilarity index 100%rename from test/python/testvectors/testcustom.pyrename to test/python/testvectors/testdense/testcustom.pydiff --git a/test/python/testvectors/testexternal.py b/test/python/testvectors/testdense/testexternal.pysimilarity index 100%rename from test/python/testvectors/testexternal.pyrename to test/python/testvectors/testdense/testexternal.pydiff --git a/test/python/testvectors/testhuggingface.py b/test/python/testvectors/testdense/testhuggingface.pysimilarity index 100%rename from test/python/testvectors/testhuggingface.pyrename to test/python/testvectors/testdense/testhuggingface.pydiff --git a/test/python/testvectors/testlitellm.py b/test/python/testvectors/testdense/testlitellm.pysimilarity index 100%rename from test/python/testvectors/testlitellm.pyrename to test/python/testvectors/testdense/testlitellm.pydiff --git a/test/python/testvectors/testllama.py b/test/python/testvectors/testdense/testllama.pysimilarity index 100%rename from test/python/testvectors/testllama.pyrename to test/python/testvectors/testdense/testllama.pydiff --git a/test/python/testvectors/testm2v.py b/test/python/testvectors/testdense/testm2v.pysimilarity index 100%rename from test/python/testvectors/testm2v.pyrename to test/python/testvectors/testdense/testm2v.pydiff --git a/test/python/testvectors/testsbert.py b/test/python/testvectors/testdense/testsbert.pysimilarity index 100%rename from test/python/testvectors/testsbert.pyrename to test/python/testvectors/testdense/testsbert.pydiff --git a/test/python/testvectors/testvectors.py b/test/python/testvectors/testdense/testvectors.pysimilarity index 96%rename from test/python/testvectors/testvectors.pyrename to test/python/testvectors/testdense/testvectors.pyindex d28a7612..e3cb0dc9 100644--- a/test/python/testvectors/testvectors.py+++ b/test/python/testvectors/testdense/testvectors.py@@ -64,5 +64,5 @@ class TestVectors(unittest.TestCase):f.close()# Create the recovery instance with an empty checkpoint file- recovery = Recovery(checkpoint, "id")+ recovery = Recovery(checkpoint, "id", np.load)self.assertIsNone(recovery())diff --git a/test/python/testvectors/testwordvectors.py b/test/python/testvectors/testdense/testwordvectors.pysimilarity index 98%rename from test/python/testvectors/testwordvectors.pyrename to test/python/testvectors/testdense/testwordvectors.pyindex 13eb35e3..7ee98e4b 100644--- a/test/python/testvectors/testwordvectors.py+++ b/test/python/testvectors/testdense/testwordvectors.py@@ -12,7 +12,7 @@ import numpy as npfrom huggingface_hub.errors import HFValidationErrorfrom txtai.vectors import VectorsFactory-from txtai.vectors.words import create, transform+from txtai.vectors.dense.words import create, transformclass TestWordVectors(unittest.TestCase):diff --git a/test/python/testvectors/testsparse/__init__.py b/test/python/testvectors/testsparse/__init__.pynew file mode 100644index 00000000..e69de29bdiff --git a/test/python/testvectors/testsparse/testsbert.py b/test/python/testvectors/testsparse/testsbert.pynew file mode 100644index 00000000..f18af43f--- /dev/null+++ b/test/python/testvectors/testsparse/testsbert.py@@ -0,0 +1,32 @@+"""+Sparse Sentence Transformers module tests+"""++import os+import unittest++from txtai.vectors import SparseVectorsFactory+from txtai.util import SparseArray+++class TestSparseSTVectors(unittest.TestCase):+ """+ SparseSTVectors tests+ """++ def testIndex(self):+ """+ Test indexing with sentence-transformers vectors+ """++ model = SparseVectorsFactory.create({"method": "sentence-transformers", "path": "sparse-encoder-testing/splade-bert-tiny-nq"})+ ids, dimension, batches, stream = model.index([(0, "test", None)])++ self.assertEqual(len(ids), 1)+ self.assertEqual(dimension, 30522)+ self.assertEqual(batches, 1)+ self.assertIsNotNone(os.path.exists(stream))++ # Test shape of serialized embeddings+ with open(stream, "rb") as queue:+ self.assertEqual(SparseArray().load(queue).shape, (1, 30522))diff --git a/test/python/testvectors/testsparse/testvectors.py b/test/python/testvectors/testsparse/testvectors.pynew file mode 100644index 00000000..63e18ade--- /dev/null+++ b/test/python/testvectors/testsparse/testvectors.py@@ -0,0 +1,40 @@+"""+Sparse Vectors module tests+"""++import unittest++from txtai.vectors import SparseVectors, SparseVectorsFactory+++class TestSparseVectors(unittest.TestCase):+ """+ Sparse Vectors tests.+ """++ def testCustom(self):+ """+ Test custom sparse vectors instance+ """++ self.assertIsNotNone(+ SparseVectorsFactory.create({"method": "txtai.vectors.SparseSTVectors", "path": "sparse-encoder-testing/splade-bert-tiny-nq"})+ )++ def testNotSupported(self):+ """+ Test exceptions for unsupported methods+ """++ vectors = SparseVectors(None, None, None)++ self.assertRaises(ValueError, vectors.truncate, None)+ self.assertRaises(ValueError, vectors.quantize, None)++ def testNotFound(self):+ """+ Test unresolvable vector backend+ """++ with self.assertRaises(ImportError):+ SparseVectorsFactory.create({"method": "notfound.vectors"})