Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Keep RAG reference ids aligned with texts when empty texts are skipped (#1473)

Rohit Behera5ca17bbea841
diff --git a/src/python/txtai/pipeline/llm/rag.py b/src/python/txtai/pipeline/llm/rag.py
index cf1d0619..10f2958e 100644
--- a/src/python/txtai/pipeline/llm/rag.py
+++ b/src/python/txtai/pipeline/llm/rag.py
@@ -247,16 +247,14 @@ class RAG(Pipeline):
# Tokenize text
segments, tokenlist = [], []
if texts:
- for text in texts:
+ # Add index id to segments to preserve ordering after filters
+ for x, text in enumerate(texts):
# Run tokenizer method, if available, otherwise returns original text
tokens = self.tokenize(text)
if tokens:
- segments.append(text)
+ segments.append((x, text))
tokenlist.append(tokens)
- # Add index id to segments to preserve ordering after filters
- segments = list(enumerate(segments))
-
# Get list of (id, score) - sorted by highest score per query
if isinstance(self.similarity, Similarity):
# Score using similarity pipeline
diff --git a/test/python/testpipeline/testllm/testrag.py b/test/python/testpipeline/testllm/testrag.py
index 6de48b92..9086de3b 100644
--- a/test/python/testpipeline/testllm/testrag.py
+++ b/test/python/testpipeline/testllm/testrag.py
@@ -121,6 +121,11 @@ class TestRAG(unittest.TestCase):
answers = rag([(question, question, question, True)], self.data)
self.assertTrue(self.data[answers[0][2]].startswith("Giants hit 3 HRs"))
+ # Test reference field indexes into texts when empty texts are skipped
+ data = [""] + self.data
+ answers = rag([(question, question, question, True)], data)
+ self.assertTrue(data[answers[0][2]].startswith("Giants hit 3 HRs"))
+
def testPrompt(self):
"""
Test a user prompt with templating