💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Fix IVFSparse returning unsorted sparse ANN search results
winklemadc33188a0f89d
diff --git a/src/python/txtai/ann/sparse/ivfsparse.py b/src/python/txtai/ann/sparse/ivfsparse.pyindex e9551f65..4d2aaee0 100644--- a/src/python/txtai/ann/sparse/ivfsparse.py+++ b/src/python/txtai/ann/sparse/ivfsparse.py@@ -273,6 +273,10 @@ class IVFSparse(ANN):indices = np.argpartition(-scores, limit if limit < scores.shape[1] else scores.shape[1] - 1)[:, :limit]scores = np.take_along_axis(scores, indices, axis=1)+ # argpartition doesn't order within the top n - sort each row by score descending+ order = np.argsort(-scores, axis=1)+ indices, scores = np.take_along_axis(indices, order, axis=1), np.take_along_axis(scores, order, axis=1)+return indices, scoresdef scan(self, query, limit, blockids):diff --git a/src/python/txtai/pipeline/text/lateencoder.py b/src/python/txtai/pipeline/text/lateencoder.pyindex e1213153..93310077 100644--- a/src/python/txtai/pipeline/text/lateencoder.py+++ b/src/python/txtai/pipeline/text/lateencoder.py@@ -100,6 +100,10 @@ class LateEncoder(Pipeline):indices = np.argpartition(-scores, limit if limit and limit < scores.shape[0] else scores.shape[0] - 1)[:, :limit]scores = np.take_along_axis(scores, indices, axis=1)+ # argpartition doesn't order within the top n - sort each row by score descending+ order = np.argsort(-scores, axis=1)+ indices, scores = np.take_along_axis(indices, order, axis=1), np.take_along_axis(scores, order, axis=1)+results = []for x, index in enumerate(indices):results.append(list(zip(index.tolist(), scores[x].tolist())))diff --git a/test/python/testann/testsparse.py b/test/python/testann/testsparse.pyindex f3c4bcc2..1584e0e3 100644--- a/test/python/testann/testsparse.py+++ b/test/python/testann/testsparse.py@@ -105,6 +105,24 @@ class TestSparse(unittest.TestCase):ann.close()+ def testIVFSparseSortOrder(self):+ """+ Test IVFSparse returns results sorted by score descending+ """++ # Generate test data+ data = self.generate(50, 30522)++ ann = SparseANNFactory.create({"backend": "ivfsparse"})+ ann.index(data)++ # Each result list must be ranked by score descending+ for results in ann.search(data[:5], 10):+ scores = [score for _, score in results]+ self.assertEqual(scores, sorted(scores, reverse=True))++ ann.close()+@patch("sqlalchemy.orm.Query.limit")def testPGSparse(self, query):"""