💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Honor the nfeatures setting in IVFSparse (#1250)
刘冠杰36c375b86e4e
diff --git a/docs/embeddings/configuration/scoring.md b/docs/embeddings/configuration/scoring.mdindex 03dbae7a..73c05ea8 100644--- a/docs/embeddings/configuration/scoring.md+++ b/docs/embeddings/configuration/scoring.md@@ -54,7 +54,7 @@ Score normalization scales the output between 0 and 1. This setting supports:```yamlivfsparse:sample: percent of data to use for model training (0.0 - 1.0)- nfeatures: top n features to use for model training (int)+ nfeatures: top n features to use for model training, defaults to all features (int)nlist: desired number of clusters (int)nprobe: search probe setting (int)minpoints: minimum number of points for a cluster (int)diff --git a/src/python/txtai/ann/sparse/ivfsparse.py b/src/python/txtai/ann/sparse/ivfsparse.pyindex fd10573a..b3e9b53a 100644--- a/src/python/txtai/ann/sparse/ivfsparse.py+++ b/src/python/txtai/ann/sparse/ivfsparse.py@@ -208,10 +208,10 @@ class IVFSparse(ANN):cluster centroids"""- # Select top n most important features that contribute to L2 vector norm- indices = np.argsort(-norm(train, axis=0))[: self.setting("nfeatures", 25)]- data = train[:, indices]- data = train+ # Select top n most important features that contribute to L2 vector norm.+ # nfeatures is opt-in: the full feature space is used when it isn't set.+ nfeatures = self.setting("nfeatures")+ data = train[:, np.argsort(-norm(train, axis=0))[:nfeatures]] if nfeatures else train# Cluster data using k-meanskmeans = MiniBatchKMeans(n_clusters=clusters, random_state=0, n_init=5).fit(data)diff --git a/test/python/testann/testsparse.py b/test/python/testann/testsparse.pyindex b7b36f48..e12212d8 100644--- a/test/python/testann/testsparse.py+++ b/test/python/testann/testsparse.py@@ -11,6 +11,7 @@ from unittest.mock import patchimport numpy as npfrom scipy.sparse import random+from sklearn.cluster import MiniBatchKMeansfrom sklearn.preprocessing import normalizefrom txtai.ann import SparseANNFactory@@ -168,6 +169,36 @@ class TestSparse(unittest.TestCase):ann.close()+ def testIVFSparseNFeatures(self):+ """+ Test IVFSparse nfeatures setting limits model training to the top n features+ """++ # Generate test data+ data = self.generate(500, 100)++ # Capture the feature count passed to model training+ features = []+ fit = MiniBatchKMeans.fit++ def spy(self, x, *args, **kwargs):+ features.append(x.shape[1])+ return fit(self, x, *args, **kwargs)++ # nfeatures unset trains on the full feature space+ ann = SparseANNFactory.create({"backend": "ivfsparse", "ivfsparse": {"nlist": 2}})+ with patch("txtai.ann.sparse.ivfsparse.MiniBatchKMeans.fit", spy):+ ann.index(data)+ ann.close()++ # nfeatures limits training to the top n features+ ann = SparseANNFactory.create({"backend": "ivfsparse", "ivfsparse": {"nlist": 2, "nfeatures": 10}})+ with patch("txtai.ann.sparse.ivfsparse.MiniBatchKMeans.fit", spy):+ ann.index(data)+ ann.close()++ self.assertEqual(features, [100, 10])+@patch("sqlalchemy.orm.Query.limit")def testPGSparse(self, query):"""