💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Preserve local filenames in RetrieveTask
dingpuyu3c0776619973
diff --git a/src/python/txtai/workflow/task/retrieve.py b/src/python/txtai/workflow/task/retrieve.pyindex cfad597d..622fab36 100644--- a/src/python/txtai/workflow/task/retrieve.py+++ b/src/python/txtai/workflow/task/retrieve.py@@ -44,11 +44,11 @@ class RetrieveTask(Task):self.safeoutput = SafeOpen(safeopen=directory if safeopen else safeopen, allowurl=False)def prepare(self, element):- # Extract file path from URL- path = urlparse(element).path-# Validate input element- url, _ = self.safeinput.valid(element)+ url, exists = self.safeinput.valid(element)++ # Preserve local filenames; only parse remote inputs as URLs+ path = url if exists else urlparse(element).pathif url:if self.flatten:# Flatten directory structure (default)diff --git a/test/python/testworkflow.py b/test/python/testworkflow.pyindex 403a7e1b..9c1d2754 100644--- a/test/python/testworkflow.py+++ b/test/python/testworkflow.py@@ -10,6 +10,8 @@ import tempfileimport sysimport unittest+from itertools import product+import numpy as npimport torch@@ -352,6 +354,29 @@ class TestWorkflow(unittest.TestCase):results = list(workflow(["file://" + Utils.PATH + "/books.jpg"]))self.assertTrue(results[0].endswith("books.jpg") and "txtai" in results[0])+ def testRetrieveLocalFilenames(self):+ """+ Test that local filenames are preserved instead of parsed as URLs+ """++ with tempfile.TemporaryDirectory(dir=".") as directory:+ directory = os.path.relpath(directory)+ names = ["report#1.txt", "report#2.txt", "report;1.txt", "plain.txt"]+ paths = [os.path.join(directory, name) for name in names]+ for path, name in zip(paths, names):+ with open(path, "w", encoding="utf-8") as output:+ output.write(name)++ for flatten, prefix, safeopen in product((True, False), ("", "file://"), (False, directory)):+ with self.subTest(flatten=flatten, prefix=prefix, safeopen=safeopen):+ task = RetrieveTask(directory=os.path.join(directory, "output"), flatten=flatten, safeopen=safeopen)+ results = list(Workflow([task])([prefix + path for path in paths]))++ self.assertEqual([os.path.basename(path) for path in results], names)+ for path, name in zip(results, names):+ with open(path, encoding="utf-8") as result:+ self.assertEqual(result.read(), name)+def testScheduleWorkflow(self):"""Test workflow schedules