💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Fix scoring.index with no tokens parsed resulting in error, closes #337
davidmezzettic346df20dbe0
diff --git a/src/python/txtai/scoring/base.py b/src/python/txtai/scoring/base.pyindex 8a402541..92e59d51 100644--- a/src/python/txtai/scoring/base.py+++ b/src/python/txtai/scoring/base.py@@ -60,27 +60,29 @@ class Scoring:# Parse documentstokenlists = self.parse(documents)- # Calculate total token frequency- self.tokens = sum(self.wordfreq.values())+ # Build index if tokens parsed+ if self.wordfreq:+ # Calculate total token frequency+ self.tokens = sum(self.wordfreq.values())- # Calculate average frequency per token- self.avgfreq = self.tokens / len(self.wordfreq.values())+ # Calculate average frequency per token+ self.avgfreq = self.tokens / len(self.wordfreq.values())- # Calculate average document length in tokens- self.avgdl = self.tokens / self.total+ # Calculate average document length in tokens+ self.avgdl = self.tokens / self.total- # Compute IDF scores- for word, freq in self.docfreq.items():- self.idf[word] = self.computeidf(freq)+ # Compute IDF scores+ for word, freq in self.docfreq.items():+ self.idf[word] = self.computeidf(freq)- # Average IDF score per token- self.avgidf = sum(self.idf.values()) / len(self.idf)+ # Average IDF score per token+ self.avgidf = sum(self.idf.values()) / len(self.idf)- # Filter for tags that appear in at least 1% of the documents- self.tags = {tag: number for tag, number in self.tags.items() if number >= self.total * 0.005}+ # Filter for tags that appear in at least 1% of the documents+ self.tags = {tag: number for tag, number in self.tags.items() if number >= self.total * 0.005}- # Process document terms, if necessary- self.terms(tokenlists)+ # Process document terms, if necessary+ self.terms(tokenlists)def parse(self, documents):"""@@ -197,7 +199,7 @@ class Scoring:# Check if document terms availableif self.docterms:- query = Tokenizer.tokenize(query)+ query = Tokenizer.tokenize(query) if isinstance(query, str) else queryscores = {}for token in query:@@ -256,7 +258,7 @@ class Scoring:Loads a saved Scoring object from path.Args:- path: directory path to load model+ path: directory path to load scoring index"""with open(path, "rb") as handle:@@ -267,7 +269,7 @@ class Scoring:Saves a Scoring object to path.Args:- path: directory path to save model+ path: directory path to save scoring index"""with open(path, "wb") as handle:diff --git a/src/python/txtai/scoring/factory.py b/src/python/txtai/scoring/factory.pyindex 015e93b0..40c1d27e 100644--- a/src/python/txtai/scoring/factory.py+++ b/src/python/txtai/scoring/factory.py@@ -9,13 +9,13 @@ from .sif import SIFclass ScoringFactory:"""- Methods to create Scoring models.+ Methods to create Scoring indexes."""@staticmethoddef create(config):"""- Factory method to construct a Scoring object.+ Factory method to construct a Scoring instance.Args:config: scoring configuration parameters - supports bm25, sif, tfidf@@ -35,7 +35,7 @@ class ScoringFactory:if method == "sif":return SIF(config)if method == "tfidf":- # Default scoring object implements tf-idf+ # Default scoring class implements tf-idfreturn Scoring(config)return Nonediff --git a/test/python/testscoring.py b/test/python/testscoring.pyindex a6555555..b2d7521f 100644--- a/test/python/testscoring.py+++ b/test/python/testscoring.py@@ -74,6 +74,7 @@ class TestScoring(unittest.TestCase):self.weights(config)self.search(config)self.content(config)+ self.empty(config)def index(self, config, data=None):"""@@ -139,8 +140,8 @@ class TestScoring(unittest.TestCase):document = (1, ["bear", "wins"], None)- model = self.index(config)- weights = model.weights(document[1])+ scoring = self.index(config)+ weights = scoring.weights(document[1])# Default weightsself.assertNotEqual(weights[0], weights[1])@@ -150,8 +151,8 @@ class TestScoring(unittest.TestCase):uid, text, _ = data[3]data[3] = (uid, text, "wins")- model = self.index(config, data)- weights = model.weights(document[1])+ scoring = self.index(config, data)+ weights = scoring.weights(document[1])# Modified weightsself.assertEqual(weights[0], weights[1])@@ -164,11 +165,11 @@ class TestScoring(unittest.TestCase):method: scoring method"""- model = ScoringFactory.create({**config, **{"terms": True}})- model.index(self.data)+ scoring = ScoringFactory.create({**config, **{"terms": True}})+ scoring.index(self.data)# Run search and validate correct result returned- index, _ = model.search("bear", 1)[0]+ index, _ = scoring.search("bear", 1)[0]self.assertEqual(index, 3)def content(self, config):@@ -179,21 +180,36 @@ class TestScoring(unittest.TestCase):config: scoring config"""- model = ScoringFactory.create({**config, **{"terms": True, "content": True}})- model.index(self.data)+ scoring = ScoringFactory.create({**config, **{"terms": True, "content": True}})+ scoring.index(self.data)# Test text with contenttext = "Great news today"- model.index([(model.total, text, None)])+ scoring.index([(scoring.total, text, None)])# Run search and validate correct result returned- result = model.search("great news", 1)[0]["text"]+ result = scoring.search("great news", 1)[0]["text"]self.assertEqual(result, text)# Test reading text from dictionarytext = "Feel good story: baby panda born"- model.index([(model.total, {"text": text}, None)])+ scoring.index([(scoring.total, {"text": text}, None)])# Run search and validate correct result returned- result = model.search("feel good story", 1)[0]["text"]+ result = scoring.search("feel good story", 1)[0]["text"]self.assertEqual(result, text)++ def empty(self, config):+ """+ Tests scoring index properly handles an index call when no data present.++ Args:+ config: scoring config+ """++ # Create scoring index with no data+ scoring = ScoringFactory.create(config)+ scoring.index([])++ # Assert index call returns and index has a count of 0+ self.assertEqual(scoring.total, 0)