Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Honor the nfeatures setting in IVFSparse (#1250)

刘冠杰36c375b86e4e
diff --git a/docs/embeddings/configuration/scoring.md b/docs/embeddings/configuration/scoring.md
index 03dbae7a..73c05ea8 100644
--- a/docs/embeddings/configuration/scoring.md
+++ b/docs/embeddings/configuration/scoring.md
@@ -54,7 +54,7 @@ Score normalization scales the output between 0 and 1. This setting supports:
```yaml
ivfsparse:
sample: percent of data to use for model training (0.0 - 1.0)
- nfeatures: top n features to use for model training (int)
+ nfeatures: top n features to use for model training, defaults to all features (int)
nlist: desired number of clusters (int)
nprobe: search probe setting (int)
minpoints: minimum number of points for a cluster (int)
diff --git a/src/python/txtai/ann/sparse/ivfsparse.py b/src/python/txtai/ann/sparse/ivfsparse.py
index fd10573a..b3e9b53a 100644
--- a/src/python/txtai/ann/sparse/ivfsparse.py
+++ b/src/python/txtai/ann/sparse/ivfsparse.py
@@ -208,10 +208,10 @@ class IVFSparse(ANN):
cluster centroids
"""
- # Select top n most important features that contribute to L2 vector norm
- indices = np.argsort(-norm(train, axis=0))[: self.setting("nfeatures", 25)]
- data = train[:, indices]
- data = train
+ # Select top n most important features that contribute to L2 vector norm.
+ # nfeatures is opt-in: the full feature space is used when it isn't set.
+ nfeatures = self.setting("nfeatures")
+ data = train[:, np.argsort(-norm(train, axis=0))[:nfeatures]] if nfeatures else train
# Cluster data using k-means
kmeans = MiniBatchKMeans(n_clusters=clusters, random_state=0, n_init=5).fit(data)
diff --git a/test/python/testann/testsparse.py b/test/python/testann/testsparse.py
index b7b36f48..e12212d8 100644
--- a/test/python/testann/testsparse.py
+++ b/test/python/testann/testsparse.py
@@ -11,6 +11,7 @@ from unittest.mock import patch
import numpy as np
from scipy.sparse import random
+from sklearn.cluster import MiniBatchKMeans
from sklearn.preprocessing import normalize
from txtai.ann import SparseANNFactory
@@ -168,6 +169,36 @@ class TestSparse(unittest.TestCase):
ann.close()
+ def testIVFSparseNFeatures(self):
+ """
+ Test IVFSparse nfeatures setting limits model training to the top n features
+ """
+
+ # Generate test data
+ data = self.generate(500, 100)
+
+ # Capture the feature count passed to model training
+ features = []
+ fit = MiniBatchKMeans.fit
+
+ def spy(self, x, *args, **kwargs):
+ features.append(x.shape[1])
+ return fit(self, x, *args, **kwargs)
+
+ # nfeatures unset trains on the full feature space
+ ann = SparseANNFactory.create({"backend": "ivfsparse", "ivfsparse": {"nlist": 2}})
+ with patch("txtai.ann.sparse.ivfsparse.MiniBatchKMeans.fit", spy):
+ ann.index(data)
+ ann.close()
+
+ # nfeatures limits training to the top n features
+ ann = SparseANNFactory.create({"backend": "ivfsparse", "ivfsparse": {"nlist": 2, "nfeatures": 10}})
+ with patch("txtai.ann.sparse.ivfsparse.MiniBatchKMeans.fit", spy):
+ ann.index(data)
+ ann.close()
+
+ self.assertEqual(features, [100, 10])
+
@patch("sqlalchemy.orm.Query.limit")
def testPGSparse(self, query):
"""