💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Only normalize topn scores
davidmezzetti0a8638c9babb
diff --git a/src/python/txtai/scoring/base.py b/src/python/txtai/scoring/base.pyindex c9c35d0f..05af3493 100644--- a/src/python/txtai/scoring/base.py+++ b/src/python/txtai/scoring/base.py@@ -213,23 +213,39 @@ class Scoring:scores[x] += self.docterms[token][x]- # Check if score normalization enabled- if self.normalize:- # Calculate max score = 4 * average score- maxscore = 4 * self.score(self.avgfreq, self.avgidf, self.avgdl)-- # Normalize scores between 0 - 1 using maxscore- for x in scores:- scores[x] = min(scores[x] / maxscore, 1.0)-- # Sort and get topn results- topn = sorted(scores.items(), key=lambda x: x[1], reverse=True)[:limit]+ # Get topn matching results+ topn = self.topn(scores, limit)# Format results and add content if availablereturn self.results(topn)return None+ def topn(self, scores, limit):+ """+ Extracts topn search results.++ Args:+ scores: all scores+ limit: maximum results++ Returns:+ topn results+ """++ # Sort and get topn results+ scores = sorted(scores.items(), key=lambda x: x[1], reverse=True)[:limit]++ # Normalize scores, if enabled+ if self.normalize:+ # Calculate max score = 4 * average score+ maxscore = 4 * self.score(self.avgfreq, self.avgidf, self.avgdl)++ # Normalize scores between 0 - 1 using maxscore+ scores = [(x, min(score / maxscore, 1.0)) for x, score in scores]++ return scores+def results(self, topn):"""Resolves a list of (id, score) with document content, if available. Otherwise, the original input is returned.