Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Fix scoring.index with no tokens parsed resulting in error, closes #337

davidmezzettic346df20dbe0
diff --git a/src/python/txtai/scoring/base.py b/src/python/txtai/scoring/base.py
index 8a402541..92e59d51 100644
--- a/src/python/txtai/scoring/base.py
+++ b/src/python/txtai/scoring/base.py
@@ -60,27 +60,29 @@ class Scoring:
# Parse documents
tokenlists = self.parse(documents)
- # Calculate total token frequency
- self.tokens = sum(self.wordfreq.values())
+ # Build index if tokens parsed
+ if self.wordfreq:
+ # Calculate total token frequency
+ self.tokens = sum(self.wordfreq.values())
- # Calculate average frequency per token
- self.avgfreq = self.tokens / len(self.wordfreq.values())
+ # Calculate average frequency per token
+ self.avgfreq = self.tokens / len(self.wordfreq.values())
- # Calculate average document length in tokens
- self.avgdl = self.tokens / self.total
+ # Calculate average document length in tokens
+ self.avgdl = self.tokens / self.total
- # Compute IDF scores
- for word, freq in self.docfreq.items():
- self.idf[word] = self.computeidf(freq)
+ # Compute IDF scores
+ for word, freq in self.docfreq.items():
+ self.idf[word] = self.computeidf(freq)
- # Average IDF score per token
- self.avgidf = sum(self.idf.values()) / len(self.idf)
+ # Average IDF score per token
+ self.avgidf = sum(self.idf.values()) / len(self.idf)
- # Filter for tags that appear in at least 1% of the documents
- self.tags = {tag: number for tag, number in self.tags.items() if number >= self.total * 0.005}
+ # Filter for tags that appear in at least 1% of the documents
+ self.tags = {tag: number for tag, number in self.tags.items() if number >= self.total * 0.005}
- # Process document terms, if necessary
- self.terms(tokenlists)
+ # Process document terms, if necessary
+ self.terms(tokenlists)
def parse(self, documents):
"""
@@ -197,7 +199,7 @@ class Scoring:
# Check if document terms available
if self.docterms:
- query = Tokenizer.tokenize(query)
+ query = Tokenizer.tokenize(query) if isinstance(query, str) else query
scores = {}
for token in query:
@@ -256,7 +258,7 @@ class Scoring:
Loads a saved Scoring object from path.
Args:
- path: directory path to load model
+ path: directory path to load scoring index
"""
with open(path, "rb") as handle:
@@ -267,7 +269,7 @@ class Scoring:
Saves a Scoring object to path.
Args:
- path: directory path to save model
+ path: directory path to save scoring index
"""
with open(path, "wb") as handle:
diff --git a/src/python/txtai/scoring/factory.py b/src/python/txtai/scoring/factory.py
index 015e93b0..40c1d27e 100644
--- a/src/python/txtai/scoring/factory.py
+++ b/src/python/txtai/scoring/factory.py
@@ -9,13 +9,13 @@ from .sif import SIF
class ScoringFactory:
"""
- Methods to create Scoring models.
+ Methods to create Scoring indexes.
"""
@staticmethod
def create(config):
"""
- Factory method to construct a Scoring object.
+ Factory method to construct a Scoring instance.
Args:
config: scoring configuration parameters - supports bm25, sif, tfidf
@@ -35,7 +35,7 @@ class ScoringFactory:
if method == "sif":
return SIF(config)
if method == "tfidf":
- # Default scoring object implements tf-idf
+ # Default scoring class implements tf-idf
return Scoring(config)
return None
diff --git a/test/python/testscoring.py b/test/python/testscoring.py
index a6555555..b2d7521f 100644
--- a/test/python/testscoring.py
+++ b/test/python/testscoring.py
@@ -74,6 +74,7 @@ class TestScoring(unittest.TestCase):
self.weights(config)
self.search(config)
self.content(config)
+ self.empty(config)
def index(self, config, data=None):
"""
@@ -139,8 +140,8 @@ class TestScoring(unittest.TestCase):
document = (1, ["bear", "wins"], None)
- model = self.index(config)
- weights = model.weights(document[1])
+ scoring = self.index(config)
+ weights = scoring.weights(document[1])
# Default weights
self.assertNotEqual(weights[0], weights[1])
@@ -150,8 +151,8 @@ class TestScoring(unittest.TestCase):
uid, text, _ = data[3]
data[3] = (uid, text, "wins")
- model = self.index(config, data)
- weights = model.weights(document[1])
+ scoring = self.index(config, data)
+ weights = scoring.weights(document[1])
# Modified weights
self.assertEqual(weights[0], weights[1])
@@ -164,11 +165,11 @@ class TestScoring(unittest.TestCase):
method: scoring method
"""
- model = ScoringFactory.create({**config, **{"terms": True}})
- model.index(self.data)
+ scoring = ScoringFactory.create({**config, **{"terms": True}})
+ scoring.index(self.data)
# Run search and validate correct result returned
- index, _ = model.search("bear", 1)[0]
+ index, _ = scoring.search("bear", 1)[0]
self.assertEqual(index, 3)
def content(self, config):
@@ -179,21 +180,36 @@ class TestScoring(unittest.TestCase):
config: scoring config
"""
- model = ScoringFactory.create({**config, **{"terms": True, "content": True}})
- model.index(self.data)
+ scoring = ScoringFactory.create({**config, **{"terms": True, "content": True}})
+ scoring.index(self.data)
# Test text with content
text = "Great news today"
- model.index([(model.total, text, None)])
+ scoring.index([(scoring.total, text, None)])
# Run search and validate correct result returned
- result = model.search("great news", 1)[0]["text"]
+ result = scoring.search("great news", 1)[0]["text"]
self.assertEqual(result, text)
# Test reading text from dictionary
text = "Feel good story: baby panda born"
- model.index([(model.total, {"text": text}, None)])
+ scoring.index([(scoring.total, {"text": text}, None)])
# Run search and validate correct result returned
- result = model.search("feel good story", 1)[0]["text"]
+ result = scoring.search("feel good story", 1)[0]["text"]
self.assertEqual(result, text)
+
+ def empty(self, config):
+ """
+ Tests scoring index properly handles an index call when no data present.
+
+ Args:
+ config: scoring config
+ """
+
+ # Create scoring index with no data
+ scoring = ScoringFactory.create(config)
+ scoring.index([])
+
+ # Assert index call returns and index has a count of 0
+ self.assertEqual(scoring.total, 0)