💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Update scoring algorithms, closes #351
davidmezzettibc91beeec59b
diff --git a/src/python/txtai/scoring/base.py b/src/python/txtai/scoring/base.pyindex 92e59d51..5e28204d 100644--- a/src/python/txtai/scoring/base.py+++ b/src/python/txtai/scoring/base.py@@ -232,12 +232,11 @@ class Scoring:if self.documents:results = []for x, score in topn:- if self.documents:- data = self.documents[x]- if isinstance(data, dict):- results.append({"id": x, "text": data.get("text"), "score": score, "data": data})- else:- results.append({"id": x, "text": data, "score": score})+ data = self.documents[x]+ if isinstance(data, dict):+ results.append({"id": x, "text": data.get("text"), "score": score, "data": data})+ else:+ results.append({"id": x, "text": data, "score": score})return results@@ -275,31 +274,31 @@ class Scoring:with open(path, "wb") as handle:pickle.dump(self.__dict__, handle, protocol=__pickle__)- def computeidf(self, freq):+ def computefreq(self, tokens):"""- Computes an idf score for word frequency.+ Computes token frequency.Args:- freq: word frequency+ tokens: input tokensReturns:- idf score+ {token: count}"""- return math.log(self.total / (1 + freq))+ return Counter(tokens)- def computefreq(self, tokens):+ def computeidf(self, freq):"""- Computes token frequency.+ Computes an idf score for word frequency.Args:- tokens: input tokens+ freq: word frequencyReturns:- {token: count}+ idf score"""- return Counter(tokens)+ return math.log((self.total + 1) / (freq + 1)) + 1# pylint: disable=W0613def score(self, freq, idf, length):@@ -315,4 +314,4 @@ class Scoring:token score"""- return (freq / length) * idf+ return idf * math.sqrt(freq) * (1 / math.sqrt(length))diff --git a/src/python/txtai/scoring/bm25.py b/src/python/txtai/scoring/bm25.pyindex 266cba8f..980908c6 100644--- a/src/python/txtai/scoring/bm25.py+++ b/src/python/txtai/scoring/bm25.py@@ -16,7 +16,7 @@ class BM25(Scoring):super().__init__(config)# BM25 configurable parameters- self.k1 = self.config.get("k1", 0.1)+ self.k1 = self.config.get("k1", 1.2)self.b = self.config.get("b", 0.75)def computeidf(self, freq):diff --git a/src/python/txtai/scoring/sif.py b/src/python/txtai/scoring/sif.pyindex d222ce58..e4bee2f6 100644--- a/src/python/txtai/scoring/sif.py+++ b/src/python/txtai/scoring/sif.py@@ -14,7 +14,7 @@ class SIF(Scoring):super().__init__(config)# SIF configurable parameters- self.a = self.config.get("a", 0.001)+ self.a = self.config.get("a", 1e-3)def computefreq(self, tokens):# Default method computes frequency for a single entry