💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add support for wildcards in scoring term search, closes #990
davidmezzettid75be702dd26
diff --git a/src/python/txtai/scoring/pgtext.py b/src/python/txtai/scoring/pgtext.pyindex 7b3ab68c..aa149db3 100644--- a/src/python/txtai/scoring/pgtext.py+++ b/src/python/txtai/scoring/pgtext.py@@ -3,6 +3,7 @@ PGText module"""import os+import re# Conditional importtry:@@ -71,6 +72,9 @@ class PGText(Scoring):return Nonedef search(self, query, limit=3):+ # Replace wildcards with prefix wildcard term+ query = re.sub(r"(?<!\:)\*", ":*", query)+# Run queryquery = (self.database.query(self.table.c["indexid"], text("ts_rank(vector, plainto_tsquery(:language, :query)) rank"))diff --git a/src/python/txtai/scoring/terms.py b/src/python/txtai/scoring/terms.pyindex cd7f7b96..8d2cabb9 100644--- a/src/python/txtai/scoring/terms.py+++ b/src/python/txtai/scoring/terms.py@@ -3,6 +3,7 @@ Terms module"""import functools+import loggingimport osimport sqlite3import sys@@ -13,6 +14,9 @@ from threading import RLockimport numpy as np+# Logging configuration+logger = logging.getLogger(__name__)+class Terms:"""@@ -30,6 +34,7 @@ class Terms:INSERT_TERM = "INSERT OR REPLACE INTO terms VALUES (?, ?, ?)"SELECT_TERMS = "SELECT ids, freqs FROM terms WHERE term = ?"+ WILDCARD_TERMS = "SELECT term FROM terms WHERE term LIKE ?"# Documents tableCREATE_DOCUMENTS = """@@ -45,6 +50,9 @@ class Terms:INSERT_DOCUMENT = "INSERT OR REPLACE INTO documents VALUES (?, ?, ?, ?)"SELECT_DOCUMENTS = "SELECT indexid, id, deleted, length FROM documents ORDER BY indexid"+ # Operators+ ASTERISK = "__asterisk__"+def __init__(self, config, score, idf):"""Creates a new terms index.@@ -173,6 +181,12 @@ class Terms:# Initialize scores arrayscores = np.zeros(len(self.ids), dtype=np.float32)+ # Apply term expansion+ terms = self.expand(terms)++ # Debug logging for terms+ logger.debug(" ".join(["%s"] * len(terms)), *terms)+# Score less common termsterms, skipped, hasscores = Counter(terms), {}, Falsefor term, freq in terms.items():@@ -192,6 +206,19 @@ class Terms:# Merge in common term scores and return top n matchesreturn self.topn(scores, limit, hasscores, skipped)+ def escape(self, query):+ """+ Escapes query operators.++ Args:+ query: query++ Returns:+ query with query operators escaped+ """++ return query.replace("*", Terms.ASTERISK)+def count(self):"""Number of elements in the scoring index.@@ -392,6 +419,33 @@ class Terms:return uids, freqs+ def expand(self, terms):+ """+ Expands a list of terms using the following term expansion rules.++ - Asterisks (*) for wildcard expressions++ Args:+ terms: list of terms++ Returns:+ expanded terms list+ """++ results = []+ for term in terms:+ # Wilcard expressions+ if Terms.ASTERISK in term:+ # Require a prefix or suffix+ if term.replace(Terms.ASTERISK, "").strip():+ term = term.replace(Terms.ASTERISK, "%")+ result = self.cursor.execute(Terms.WILDCARD_TERMS, [term])+ results.extend([t for t, in result])+ else:+ results.append(term)++ return results+@functools.lru_cache(maxsize=500)def weights(self, term):"""diff --git a/src/python/txtai/scoring/tfidf.py b/src/python/txtai/scoring/tfidf.pyindex e1c5f29c..1bc64744 100644--- a/src/python/txtai/scoring/tfidf.py+++ b/src/python/txtai/scoring/tfidf.py@@ -156,6 +156,9 @@ class TFIDF(Scoring):def search(self, query, limit=3):# Check if term index availableif self.terms:+ # Escape query operators+ query = self.terms.escape(query) if isinstance(query, str) else [self.terms.escape(q) for q in query]+# Parse query into termsquery = self.tokenize(query) if isinstance(query, str) else querydiff --git a/test/python/testscoring/testkeyword.py b/test/python/testscoring/testkeyword.pyindex 6f0b8446..8b84a537 100644--- a/test/python/testscoring/testkeyword.py+++ b/test/python/testscoring/testkeyword.py@@ -275,6 +275,10 @@ class TestKeyword(unittest.TestCase):index, _ = scoring.batchsearch(["bear"], 1)[0][0]self.assertEqual(index, 3)+ # Run wildcard search+ index, _ = scoring.search("bea*", 1)[0]+ self.assertEqual(index, 3)+# Test save/reloadself.save(scoring, config, f"scoring.{config['method']}.search")