Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Preserve local filenames in RetrieveTask

dingpuyu3c0776619973
diff --git a/src/python/txtai/workflow/task/retrieve.py b/src/python/txtai/workflow/task/retrieve.py
index cfad597d..622fab36 100644
--- a/src/python/txtai/workflow/task/retrieve.py
+++ b/src/python/txtai/workflow/task/retrieve.py
@@ -44,11 +44,11 @@ class RetrieveTask(Task):
self.safeoutput = SafeOpen(safeopen=directory if safeopen else safeopen, allowurl=False)
def prepare(self, element):
- # Extract file path from URL
- path = urlparse(element).path
-
# Validate input element
- url, _ = self.safeinput.valid(element)
+ url, exists = self.safeinput.valid(element)
+
+ # Preserve local filenames; only parse remote inputs as URLs
+ path = url if exists else urlparse(element).path
if url:
if self.flatten:
# Flatten directory structure (default)
diff --git a/test/python/testworkflow.py b/test/python/testworkflow.py
index 403a7e1b..9c1d2754 100644
--- a/test/python/testworkflow.py
+++ b/test/python/testworkflow.py
@@ -10,6 +10,8 @@ import tempfile
import sys
import unittest
+from itertools import product
+
import numpy as np
import torch
@@ -352,6 +354,29 @@ class TestWorkflow(unittest.TestCase):
results = list(workflow(["file://" + Utils.PATH + "/books.jpg"]))
self.assertTrue(results[0].endswith("books.jpg") and "txtai" in results[0])
+ def testRetrieveLocalFilenames(self):
+ """
+ Test that local filenames are preserved instead of parsed as URLs
+ """
+
+ with tempfile.TemporaryDirectory(dir=".") as directory:
+ directory = os.path.relpath(directory)
+ names = ["report#1.txt", "report#2.txt", "report;1.txt", "plain.txt"]
+ paths = [os.path.join(directory, name) for name in names]
+ for path, name in zip(paths, names):
+ with open(path, "w", encoding="utf-8") as output:
+ output.write(name)
+
+ for flatten, prefix, safeopen in product((True, False), ("", "file://"), (False, directory)):
+ with self.subTest(flatten=flatten, prefix=prefix, safeopen=safeopen):
+ task = RetrieveTask(directory=os.path.join(directory, "output"), flatten=flatten, safeopen=safeopen)
+ results = list(Workflow([task])([prefix + path for path in paths]))
+
+ self.assertEqual([os.path.basename(path) for path in results], names)
+ for path, name in zip(results, names):
+ with open(path, encoding="utf-8") as result:
+ self.assertEqual(result.read(), name)
+
def testScheduleWorkflow(self):
"""
Test workflow schedules