💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Preserve keyword document ID types across save/load (#1274) (#1275)
Hua.T8ef454d0bf7e
diff --git a/docs/embeddings/format.md b/docs/embeddings/format.mdindex fc1ade27..e026865f 100644--- a/docs/embeddings/format.md+++ b/docs/embeddings/format.md@@ -57,3 +57,8 @@ Sparse/keyword indexing| ------------------------------------------------------------- | ----------------------------------------------------------------------------- || [Local index](https://www.sqlite.org/) | Metadata serialized with MessagePack. Terms stored in SQLite. || [Postgres](https://www.postgresql.org/docs/current/textsearch.html) | Text indexed with Postgres Full Text Search (FTS) |++Local keyword indexes store document IDs as MessagePack blobs in SQLite to preserve+string and integer types. Legacy indexes with plain text IDs can still be loaded,+using their existing integer-inference behavior. Indexes saved with serialized IDs+require a version of txtai that supports this format.diff --git a/src/python/txtai/scoring/terms.py b/src/python/txtai/scoring/terms.pyindex cfb71834..26993e31 100644--- a/src/python/txtai/scoring/terms.py+++ b/src/python/txtai/scoring/terms.py@@ -13,6 +13,7 @@ from collections import Counterfrom threading import RLock# Core library imports+from ..serialize import SerializeFactoryfrom ..util import Librarynp = Library().numpy()@@ -278,9 +279,13 @@ class Terms:# Index id - lengthself.lengths.append(length)- # Cast ids to int if every id is an integer- if all(uid.isdigit() for uid in self.ids):+ # Retain integer inference for legacy indexes with plain TEXT ids.+ if all(isinstance(uid, str) and uid.isdigit() for uid in self.ids):self.ids = [int(uid) for uid in self.ids]+ else:+ # Serialized ids preserve types, including numeric strings and mixed string/integer ids.+ serializer = SerializeFactory.create()+ self.ids = [serializer.loadbytes(uid) if isinstance(uid, bytes) else uid for uid in self.ids]# Clear cacheself.weights.cache_clear()@@ -299,9 +304,10 @@ class Terms:# Clear documents tableself.cursor.execute(Terms.DELETE_DOCUMENTS)- # Save document attributes+ # Serialize ids as BLOBs so SQLite TEXT affinity does not discard their types.+ serializer = SerializeFactory.create()for i, uid in enumerate(self.ids):- self.cursor.execute(Terms.INSERT_DOCUMENT, [i, uid, 1 if i in self.deletes else 0, self.lengths[i]])+ self.cursor.execute(Terms.INSERT_DOCUMENT, [i, serializer.savebytes(uid), 1 if i in self.deletes else 0, self.lengths[i]])# Temporary databaseif not self.path:diff --git a/test/python/testscoring/testkeyword.py b/test/python/testscoring/testkeyword.pyindex e48b9bf5..856da201 100644--- a/test/python/testscoring/testkeyword.py+++ b/test/python/testscoring/testkeyword.py@@ -3,6 +3,7 @@ Keyword scoring tests"""import os+import sqlite3import tempfileimport unittest@@ -163,6 +164,82 @@ class TestKeyword(unittest.TestCase):self.assertEqual(scoring.count(), 0)self.assertEqual(scoring.search("bear", 1), [])+ def testTermsIdRoundTrip(self):+ """+ Test that saving and loading preserves keyword document IDs and their content+ """++ texts = ["orchard apple", "coastal ocean", "forest trail"]+ cases = [["001", "1", "01"], [0, 1, -1], [1, "1", "notes"], ["²", "³", "¹"], [0, 1, 2], ["one", "two", "three"]]+ for method in ["bm25", "tfidf", "sif"]:+ for content in [False, True]:+ for ids in cases:+ with self.subTest(method=method, content=content, ids=ids), tempfile.TemporaryDirectory() as directory:+ config = {"method": method, "terms": True, "content": content}+ scoring = ScoringFactory.create(config)+ try:+ scoring.index([(uid, text, None) for uid, text in zip(ids, texts)])+ expected = scoring.batchsearch(texts)+ path, copy = os.path.join(directory, "index"), os.path.join(directory, "copy")++ # Cover an initial save, an in-place save and a copy to a different path.+ for target in [path, path, copy]:+ scoring.save(target)+ scoring.close()+ scoring = ScoringFactory.create(config)+ scoring.load(target)+ self.assertEqual(scoring.batchsearch(texts), expected)++ # IDs must still identify the same document when deleting after a reload.+ scoring.delete([ids[0]])+ scoring.save(copy)+ scoring.close()+ scoring = ScoringFactory.create(config)+ scoring.load(copy)+ self.assertEqual(scoring.count(), 2)+ self.assertEqual(scoring.search(texts[0]), [])+ self.assertEqual(scoring.batchsearch(texts[1:]), expected[1:])+ finally:+ scoring.close()++ def testTermsLegacyIds(self):+ """+ Test reading legacy text IDs and saving additional documents into those indexes+ """++ texts = ["orchard apple", "coastal ocean"]+ for ids in [[0, 1], ["one", "two"]]:+ for content in [False, True]:+ with self.subTest(ids=ids, content=content), tempfile.TemporaryDirectory() as directory:+ config = {"method": "bm25", "terms": True, "content": content}+ scoring = ScoringFactory.create(config)+ try:+ scoring.index([(uid, text, None) for uid, text in zip(ids, texts)])+ expected = scoring.batchsearch(texts)+ path = os.path.join(directory, "index")+ scoring.save(path)+ scoring.close()++ # Older indexes stored IDs as plain SQLite TEXT, without type information.+ with sqlite3.connect(path + ".terms") as connection:+ connection.executemany("UPDATE documents SET id = ? WHERE indexid = ?", [(str(uid), i) for i, uid in enumerate(ids)])+ connection.close()++ scoring = ScoringFactory.create(config)+ scoring.load(path)+ self.assertEqual(scoring.batchsearch(texts), expected)++ scoring.upsert([("003", "forest trail", None)])+ expected = scoring.batchsearch([*texts, "forest trail"])+ scoring.save(path)+ scoring.close()+ scoring = ScoringFactory.create(config)+ scoring.load(path)+ self.assertEqual(scoring.count(), 3)+ self.assertEqual(scoring.batchsearch([*texts, "forest trail"]), expected)+ finally:+ scoring.close()+def testWildcardEscapedCharacters(self):"""Test explicit escaping while preserving SQL LIKE wildcard behavior