💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows
Python agentsaiai-agentsembeddingsinformation-retrievallanguage-modellarge-language-modelsllmnlppythonragretrieval-augmented-generationsearchsearch-enginesemantic-searchsentence-embeddingstransformerstxtaivector-databasevector-search
Validate object storage download paths
r4venrudyc19756498ee7
diff --git a/src/python/txtai/cloud/storage.py b/src/python/txtai/cloud/storage.pyindex e3108741..0a4cd931 100644--- a/src/python/txtai/cloud/storage.py+++ b/src/python/txtai/cloud/storage.py@@ -4,6 +4,8 @@ Object storage moduleimport os+from pathlib import Path+# Conditional importtry:from libcloud.storage.providers import get_driver, DRIVERS@@ -77,17 +79,19 @@ class ObjectStorage(Cloud):# Download files in container. Optionally filter with a provided prefix.else:+ root = Path(path).resolve()container = self.client.get_container(self.config["container"])for obj in container.list_objects(prefix=self.config.get("prefix")):- # Derive local path and directory- localpath = os.path.join(path, obj.name)- directory = os.path.dirname(localpath)+ # Validate the resolved file destination+ localpath = (root / obj.name).resolve()+ if root not in localpath.parents or localpath.is_dir():+ raise IOError(f"Invalid object path: {obj.name}")# Create local directory, if necessary- os.makedirs(directory, exist_ok=True)+ os.makedirs(localpath.parent, exist_ok=True)# Download file locally- obj.download(localpath, overwrite_existing=True)+ obj.download(str(localpath), overwrite_existing=True)return pathdiff --git a/test/python/testcloud.py b/test/python/testcloud.pyindex 4136a733..be967f81 100644--- a/test/python/testcloud.py+++ b/test/python/testcloud.py@@ -7,13 +7,83 @@ import tempfileimport timeimport unittest-from unittest.mock import patch+from unittest.mock import Mock, patchfrom huggingface_hub import hf_hub_download-from txtai.cloud import Cloud+from txtai.cloud import Cloud, ObjectStoragefrom txtai.embeddings import Embeddings+class TestObjectStorage(unittest.TestCase):+ """+ Object storage path tests.+ """++ def testLoadPaths(self):+ """+ Test nested object paths and prefix filtering+ """++ with tempfile.TemporaryDirectory() as directory:+ cloud = ObjectStorage({"provider": "local", "container": "test", "key": directory, "prefix": "index"})+ cloud.client = Mock()+ obj = Mock()+ obj.name = "index/nested/config.json"+ container = cloud.client.get_container.return_value+ container.list_objects.return_value = [obj]++ path = os.path.join(directory, "output")+ self.assertEqual(cloud.load(path), path)+ container.list_objects.assert_called_once_with(prefix="index")+ obj.download.assert_called_once_with(os.path.realpath(os.path.join(path, obj.name)), overwrite_existing=True)+ self.assertTrue(os.path.isdir(os.path.join(path, "index", "nested")))++ def testInvalidLoadPaths(self):+ """+ Test invalid object destinations before filesystem changes+ """++ with tempfile.TemporaryDirectory() as directory:+ cloud = ObjectStorage({"provider": "local", "container": "test", "key": directory})+ cloud.client = Mock()+ obj = Mock()+ cloud.client.get_container.return_value.list_objects.return_value = [obj]+ path = os.path.join(directory, "output")+ os.makedirs(os.path.join(path, "nested"))++ for name in ["../other/file", os.path.join(directory, "other", "file"), "../output-other/file", ".", "nested"]:+ with self.subTest(name=name), patch("txtai.cloud.storage.os.makedirs") as makedirs:+ obj.name = name+ with self.assertRaises(IOError):+ cloud.load(path)+ makedirs.assert_not_called()+ obj.download.assert_not_called()++ def testLoadLink(self):+ """+ Test object destinations through a local directory link+ """++ with tempfile.TemporaryDirectory() as directory:+ cloud = ObjectStorage({"provider": "local", "container": "test", "key": directory})+ cloud.client = Mock()+ obj = Mock()+ obj.name = "nested/config.json"+ cloud.client.get_container.return_value.list_objects.return_value = [obj]+ path = os.path.join(directory, "output")+ os.makedirs(path)+ try:+ os.symlink(directory, os.path.join(path, "nested"), target_is_directory=True)+ except OSError as e:+ if getattr(e, "winerror", None) != 1314:+ raise+ self.skipTest("Creating symbolic links requires permission")++ with self.assertRaises(IOError):+ cloud.load(path)+ obj.download.assert_not_called()++class TestCloud(unittest.TestCase):"""Cloud tests.