Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Add support for wildcards in scoring term search, closes #990

davidmezzettid75be702dd26
diff --git a/src/python/txtai/scoring/pgtext.py b/src/python/txtai/scoring/pgtext.py
index 7b3ab68c..aa149db3 100644
--- a/src/python/txtai/scoring/pgtext.py
+++ b/src/python/txtai/scoring/pgtext.py
@@ -3,6 +3,7 @@ PGText module
"""
import os
+import re
# Conditional import
try:
@@ -71,6 +72,9 @@ class PGText(Scoring):
return None
def search(self, query, limit=3):
+ # Replace wildcards with prefix wildcard term
+ query = re.sub(r"(?<!\:)\*", ":*", query)
+
# Run query
query = (
self.database.query(self.table.c["indexid"], text("ts_rank(vector, plainto_tsquery(:language, :query)) rank"))
diff --git a/src/python/txtai/scoring/terms.py b/src/python/txtai/scoring/terms.py
index cd7f7b96..8d2cabb9 100644
--- a/src/python/txtai/scoring/terms.py
+++ b/src/python/txtai/scoring/terms.py
@@ -3,6 +3,7 @@ Terms module
"""
import functools
+import logging
import os
import sqlite3
import sys
@@ -13,6 +14,9 @@ from threading import RLock
import numpy as np
+# Logging configuration
+logger = logging.getLogger(__name__)
+
class Terms:
"""
@@ -30,6 +34,7 @@ class Terms:
INSERT_TERM = "INSERT OR REPLACE INTO terms VALUES (?, ?, ?)"
SELECT_TERMS = "SELECT ids, freqs FROM terms WHERE term = ?"
+ WILDCARD_TERMS = "SELECT term FROM terms WHERE term LIKE ?"
# Documents table
CREATE_DOCUMENTS = """
@@ -45,6 +50,9 @@ class Terms:
INSERT_DOCUMENT = "INSERT OR REPLACE INTO documents VALUES (?, ?, ?, ?)"
SELECT_DOCUMENTS = "SELECT indexid, id, deleted, length FROM documents ORDER BY indexid"
+ # Operators
+ ASTERISK = "__asterisk__"
+
def __init__(self, config, score, idf):
"""
Creates a new terms index.
@@ -173,6 +181,12 @@ class Terms:
# Initialize scores array
scores = np.zeros(len(self.ids), dtype=np.float32)
+ # Apply term expansion
+ terms = self.expand(terms)
+
+ # Debug logging for terms
+ logger.debug(" ".join(["%s"] * len(terms)), *terms)
+
# Score less common terms
terms, skipped, hasscores = Counter(terms), {}, False
for term, freq in terms.items():
@@ -192,6 +206,19 @@ class Terms:
# Merge in common term scores and return top n matches
return self.topn(scores, limit, hasscores, skipped)
+ def escape(self, query):
+ """
+ Escapes query operators.
+
+ Args:
+ query: query
+
+ Returns:
+ query with query operators escaped
+ """
+
+ return query.replace("*", Terms.ASTERISK)
+
def count(self):
"""
Number of elements in the scoring index.
@@ -392,6 +419,33 @@ class Terms:
return uids, freqs
+ def expand(self, terms):
+ """
+ Expands a list of terms using the following term expansion rules.
+
+ - Asterisks (*) for wildcard expressions
+
+ Args:
+ terms: list of terms
+
+ Returns:
+ expanded terms list
+ """
+
+ results = []
+ for term in terms:
+ # Wilcard expressions
+ if Terms.ASTERISK in term:
+ # Require a prefix or suffix
+ if term.replace(Terms.ASTERISK, "").strip():
+ term = term.replace(Terms.ASTERISK, "%")
+ result = self.cursor.execute(Terms.WILDCARD_TERMS, [term])
+ results.extend([t for t, in result])
+ else:
+ results.append(term)
+
+ return results
+
@functools.lru_cache(maxsize=500)
def weights(self, term):
"""
diff --git a/src/python/txtai/scoring/tfidf.py b/src/python/txtai/scoring/tfidf.py
index e1c5f29c..1bc64744 100644
--- a/src/python/txtai/scoring/tfidf.py
+++ b/src/python/txtai/scoring/tfidf.py
@@ -156,6 +156,9 @@ class TFIDF(Scoring):
def search(self, query, limit=3):
# Check if term index available
if self.terms:
+ # Escape query operators
+ query = self.terms.escape(query) if isinstance(query, str) else [self.terms.escape(q) for q in query]
+
# Parse query into terms
query = self.tokenize(query) if isinstance(query, str) else query
diff --git a/test/python/testscoring/testkeyword.py b/test/python/testscoring/testkeyword.py
index 6f0b8446..8b84a537 100644
--- a/test/python/testscoring/testkeyword.py
+++ b/test/python/testscoring/testkeyword.py
@@ -275,6 +275,10 @@ class TestKeyword(unittest.TestCase):
index, _ = scoring.batchsearch(["bear"], 1)[0][0]
self.assertEqual(index, 3)
+ # Run wildcard search
+ index, _ = scoring.search("bea*", 1)[0]
+ self.assertEqual(index, 3)
+
# Test save/reload
self.save(scoring, config, f"scoring.{config['method']}.search")