💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Add bm25s library to benchmarks (#757)
a0346f102085fe9f600cb2666524
diff --git a/examples/benchmarks.py b/examples/benchmarks.pyindex 09ad4f08..4de59037 100644--- a/examples/benchmarks.py+++ b/examples/benchmarks.py@@ -2,7 +2,7 @@Runs benchmark evaluations with the BEIR dataset.Install txtai and the following dependencies to run:- pip install txtai pytrec_eval rank-bm25 elasticsearch psutil+ pip install txtai pytrec_eval rank-bm25 elasticsearch psutil bm25s"""import argparse@@ -24,6 +24,8 @@ from pytrec_eval import RelevanceEvaluatorfrom elasticsearch import Elasticsearchfrom elasticsearch.helpers import bulk+import bm25s+from txtai.embeddings import Embeddingsfrom txtai.pipeline import Extractor, LLM, Tokenizerfrom txtai.scoring import ScoringFactory@@ -387,6 +389,50 @@ class Elastic(Index):return es+class BM25S(Index):+ """+ BM25 as implemented by BM25S+ """++ def search(self, queries, limit):+ tokenizer = Tokenizer()++ tokens = [tokenizer(x) for x in queries]++ results, scores = self.backend.retrieve(tokens, corpus=self.corpus_ids, k=limit, n_threads=4)++ # List of queries => list of matches (docid, score)+ x = []++ for a, b in zip(results, scores):+ x.append([(str(c), float(d)) for c, d in zip(a, b)])++ return x++ def index(self):+ tokenizer = Tokenizer()+ corpus_ids = []+ corpus_lst = []++ for uid, text, _ in self.rows():+ corpus_ids.append(uid)+ corpus_lst.append(text)++ self.corpus_ids = corpus_ids++ if os.path.exists(self.output) and not self.refresh:+ model = bm25s.BM25.load(self.output)+ else:+ tokens = [tokenizer(x) for x in corpus_lst]++ model = bm25s.BM25(method="lucene", k1=1.2, b=0.75)+ model.index(tokens, leave_progress=False)++ model.save(self.output)++ return model++def relevance(path):"""Loads relevance data for evaluation.@@ -440,6 +486,8 @@ def create(method, path, config, output, refresh):return SQLiteFTS(path, config, output, refresh)if method == "rank":return RankBM25(path, config, output, refresh)+ if method == "bm25s":+ return BM25S(path, config, output, refresh)# Defaultreturn Embed(path, config, output, refresh)@@ -563,7 +611,7 @@ def benchmarks(args):"climate-fever","scifact",]- methods = ["embed", "es", "hybrid", "rank", "scoring", "sqlite"]+ methods = ["embed", "es", "hybrid", "rank", "scoring", "sqlite", "bm25s"]mode = "w"# Run and save benchmarks