Skip to content

armgit / txtai

Public Imported from GitHub
0

💡 All-in-one AI framework for semantic search, LLM orchestration and language model workflows

Validate object storage download paths

r4venrudyc19756498ee7
diff --git a/src/python/txtai/cloud/storage.py b/src/python/txtai/cloud/storage.py
index e3108741..0a4cd931 100644
--- a/src/python/txtai/cloud/storage.py
+++ b/src/python/txtai/cloud/storage.py
@@ -4,6 +4,8 @@ Object storage module
import os
+from pathlib import Path
+
# Conditional import
try:
from libcloud.storage.providers import get_driver, DRIVERS
@@ -77,17 +79,19 @@ class ObjectStorage(Cloud):
# Download files in container. Optionally filter with a provided prefix.
else:
+ root = Path(path).resolve()
container = self.client.get_container(self.config["container"])
for obj in container.list_objects(prefix=self.config.get("prefix")):
- # Derive local path and directory
- localpath = os.path.join(path, obj.name)
- directory = os.path.dirname(localpath)
+ # Validate the resolved file destination
+ localpath = (root / obj.name).resolve()
+ if root not in localpath.parents or localpath.is_dir():
+ raise IOError(f"Invalid object path: {obj.name}")
# Create local directory, if necessary
- os.makedirs(directory, exist_ok=True)
+ os.makedirs(localpath.parent, exist_ok=True)
# Download file locally
- obj.download(localpath, overwrite_existing=True)
+ obj.download(str(localpath), overwrite_existing=True)
return path
diff --git a/test/python/testcloud.py b/test/python/testcloud.py
index 4136a733..be967f81 100644
--- a/test/python/testcloud.py
+++ b/test/python/testcloud.py
@@ -7,13 +7,83 @@ import tempfile
import time
import unittest
-from unittest.mock import patch
+from unittest.mock import Mock, patch
from huggingface_hub import hf_hub_download
-from txtai.cloud import Cloud
+from txtai.cloud import Cloud, ObjectStorage
from txtai.embeddings import Embeddings
+class TestObjectStorage(unittest.TestCase):
+ """
+ Object storage path tests.
+ """
+
+ def testLoadPaths(self):
+ """
+ Test nested object paths and prefix filtering
+ """
+
+ with tempfile.TemporaryDirectory() as directory:
+ cloud = ObjectStorage({"provider": "local", "container": "test", "key": directory, "prefix": "index"})
+ cloud.client = Mock()
+ obj = Mock()
+ obj.name = "index/nested/config.json"
+ container = cloud.client.get_container.return_value
+ container.list_objects.return_value = [obj]
+
+ path = os.path.join(directory, "output")
+ self.assertEqual(cloud.load(path), path)
+ container.list_objects.assert_called_once_with(prefix="index")
+ obj.download.assert_called_once_with(os.path.realpath(os.path.join(path, obj.name)), overwrite_existing=True)
+ self.assertTrue(os.path.isdir(os.path.join(path, "index", "nested")))
+
+ def testInvalidLoadPaths(self):
+ """
+ Test invalid object destinations before filesystem changes
+ """
+
+ with tempfile.TemporaryDirectory() as directory:
+ cloud = ObjectStorage({"provider": "local", "container": "test", "key": directory})
+ cloud.client = Mock()
+ obj = Mock()
+ cloud.client.get_container.return_value.list_objects.return_value = [obj]
+ path = os.path.join(directory, "output")
+ os.makedirs(os.path.join(path, "nested"))
+
+ for name in ["../other/file", os.path.join(directory, "other", "file"), "../output-other/file", ".", "nested"]:
+ with self.subTest(name=name), patch("txtai.cloud.storage.os.makedirs") as makedirs:
+ obj.name = name
+ with self.assertRaises(IOError):
+ cloud.load(path)
+ makedirs.assert_not_called()
+ obj.download.assert_not_called()
+
+ def testLoadLink(self):
+ """
+ Test object destinations through a local directory link
+ """
+
+ with tempfile.TemporaryDirectory() as directory:
+ cloud = ObjectStorage({"provider": "local", "container": "test", "key": directory})
+ cloud.client = Mock()
+ obj = Mock()
+ obj.name = "nested/config.json"
+ cloud.client.get_container.return_value.list_objects.return_value = [obj]
+ path = os.path.join(directory, "output")
+ os.makedirs(path)
+ try:
+ os.symlink(directory, os.path.join(path, "nested"), target_is_directory=True)
+ except OSError as e:
+ if getattr(e, "winerror", None) != 1314:
+ raise
+ self.skipTest("Creating symbolic links requires permission")
+
+ with self.assertRaises(IOError):
+ cloud.load(path)
+ obj.download.assert_not_called()
+
+
class TestCloud(unittest.TestCase):
"""
Cloud tests.