Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Decode local file URIs in FileTask (#1508)

Hua.Tfb2ed0691827
diff --git a/src/python/txtai/workflow/task/file.py b/src/python/txtai/workflow/task/file.py
index 9f878147..5834bb17 100644
--- a/src/python/txtai/workflow/task/file.py
+++ b/src/python/txtai/workflow/task/file.py
@@ -5,6 +5,8 @@ FileTask module
import os
import re
+from urllib.request import url2pathname
+
from .base import Task
@@ -21,12 +23,25 @@ class FileTask(Task):
if not isinstance(element, str):
return False
- # Replace file prefixes
- element = re.sub(FileTask.FILE, "", element)
+ # Normalize file paths
+ element = self.normalize(element)
# Only accept file paths that exist
return super().accept(element) and os.path.exists(element)
def prepare(self, element):
- # Replace file prefixes
- return re.sub(FileTask.FILE, "", element)
+ return self.normalize(element)
+
+ def normalize(self, element):
+ """
+ Normalizes a local file path, decoding file URIs when necessary.
+
+ Args:
+ element: file path or URI
+
+ Returns:
+ local file path
+ """
+
+ # Decode file URIs while preserving literal percent sequences in plain paths.
+ return url2pathname(re.sub(FileTask.FILE, "", element)) if element.startswith("file://") else element
diff --git a/test/python/testworkflow.py b/test/python/testworkflow.py
index 839bb74e..61daff23 100644
--- a/test/python/testworkflow.py
+++ b/test/python/testworkflow.py
@@ -13,6 +13,7 @@ import unittest
from datetime import datetime, timezone
from itertools import product
+from pathlib import Path
from unittest.mock import patch
import numpy as np
@@ -355,6 +356,18 @@ class TestWorkflow(unittest.TestCase):
results = list(workflow(inputs))
self.assertEqual(results, inputs)
+ def testFileURIWorkflow(self):
+ """
+ Test encoded file URIs and plain paths with literal percent sequences.
+ """
+
+ with tempfile.TemporaryDirectory() as directory:
+ path = Path(directory) / "notes %20 #1.txt"
+ path.write_text("research note", encoding="utf-8")
+
+ workflow = Workflow([FileTask(lambda paths: [Path(p).read_text(encoding="utf-8") for p in paths])])
+ self.assertEqual(list(workflow([str(path), path.as_uri()])), ["research note", "research note"])
+
def testImageWorkflow(self):
"""
Test an image task