Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Update scoring algorithms, closes #351

davidmezzettibc91beeec59b
diff --git a/src/python/txtai/scoring/base.py b/src/python/txtai/scoring/base.py
index 92e59d51..5e28204d 100644
--- a/src/python/txtai/scoring/base.py
+++ b/src/python/txtai/scoring/base.py
@@ -232,12 +232,11 @@ class Scoring:
if self.documents:
results = []
for x, score in topn:
- if self.documents:
- data = self.documents[x]
- if isinstance(data, dict):
- results.append({"id": x, "text": data.get("text"), "score": score, "data": data})
- else:
- results.append({"id": x, "text": data, "score": score})
+ data = self.documents[x]
+ if isinstance(data, dict):
+ results.append({"id": x, "text": data.get("text"), "score": score, "data": data})
+ else:
+ results.append({"id": x, "text": data, "score": score})
return results
@@ -275,31 +274,31 @@ class Scoring:
with open(path, "wb") as handle:
pickle.dump(self.__dict__, handle, protocol=__pickle__)
- def computeidf(self, freq):
+ def computefreq(self, tokens):
"""
- Computes an idf score for word frequency.
+ Computes token frequency.
Args:
- freq: word frequency
+ tokens: input tokens
Returns:
- idf score
+ {token: count}
"""
- return math.log(self.total / (1 + freq))
+ return Counter(tokens)
- def computefreq(self, tokens):
+ def computeidf(self, freq):
"""
- Computes token frequency.
+ Computes an idf score for word frequency.
Args:
- tokens: input tokens
+ freq: word frequency
Returns:
- {token: count}
+ idf score
"""
- return Counter(tokens)
+ return math.log((self.total + 1) / (freq + 1)) + 1
# pylint: disable=W0613
def score(self, freq, idf, length):
@@ -315,4 +314,4 @@ class Scoring:
token score
"""
- return (freq / length) * idf
+ return idf * math.sqrt(freq) * (1 / math.sqrt(length))
diff --git a/src/python/txtai/scoring/bm25.py b/src/python/txtai/scoring/bm25.py
index 266cba8f..980908c6 100644
--- a/src/python/txtai/scoring/bm25.py
+++ b/src/python/txtai/scoring/bm25.py
@@ -16,7 +16,7 @@ class BM25(Scoring):
super().__init__(config)
# BM25 configurable parameters
- self.k1 = self.config.get("k1", 0.1)
+ self.k1 = self.config.get("k1", 1.2)
self.b = self.config.get("b", 0.75)
def computeidf(self, freq):
diff --git a/src/python/txtai/scoring/sif.py b/src/python/txtai/scoring/sif.py
index d222ce58..e4bee2f6 100644
--- a/src/python/txtai/scoring/sif.py
+++ b/src/python/txtai/scoring/sif.py
@@ -14,7 +14,7 @@ class SIF(Scoring):
super().__init__(config)
# SIF configurable parameters
- self.a = self.config.get("a", 0.001)
+ self.a = self.config.get("a", 1e-3)
def computefreq(self, tokens):
# Default method computes frequency for a single entry