💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Keep RAG reference ids aligned with texts when empty texts are skipped (#1473)
Rohit Behera5ca17bbea841
diff --git a/src/python/txtai/pipeline/llm/rag.py b/src/python/txtai/pipeline/llm/rag.pyindex cf1d0619..10f2958e 100644--- a/src/python/txtai/pipeline/llm/rag.py+++ b/src/python/txtai/pipeline/llm/rag.py@@ -247,16 +247,14 @@ class RAG(Pipeline):# Tokenize textsegments, tokenlist = [], []if texts:- for text in texts:+ # Add index id to segments to preserve ordering after filters+ for x, text in enumerate(texts):# Run tokenizer method, if available, otherwise returns original texttokens = self.tokenize(text)if tokens:- segments.append(text)+ segments.append((x, text))tokenlist.append(tokens)- # Add index id to segments to preserve ordering after filters- segments = list(enumerate(segments))-# Get list of (id, score) - sorted by highest score per queryif isinstance(self.similarity, Similarity):# Score using similarity pipelinediff --git a/test/python/testpipeline/testllm/testrag.py b/test/python/testpipeline/testllm/testrag.pyindex 6de48b92..9086de3b 100644--- a/test/python/testpipeline/testllm/testrag.py+++ b/test/python/testpipeline/testllm/testrag.py@@ -121,6 +121,11 @@ class TestRAG(unittest.TestCase):answers = rag([(question, question, question, True)], self.data)self.assertTrue(self.data[answers[0][2]].startswith("Giants hit 3 HRs"))+ # Test reference field indexes into texts when empty texts are skipped+ data = [""] + self.data+ answers = rag([(question, question, question, True)], data)+ self.assertTrue(data[answers[0][2]].startswith("Giants hit 3 HRs"))+def testPrompt(self):"""Test a user prompt with templating