Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: pin managed dataset media URLs to the Hub and scope extracted archive members (#14840)

Daniel Hanf443cf93f8f0
diff --git a/studio/backend/tests/test_dataset_media_decode_guard.py b/studio/backend/tests/test_dataset_media_decode_guard.py
index 6a66692bf8b..6941de690a0 100644
--- a/studio/backend/tests/test_dataset_media_decode_guard.py
+++ b/studio/backend/tests/test_dataset_media_decode_guard.py
@@ -273,3 +273,190 @@ def test_managed_snapshot_files_of_its_training_dataset_still_decode(tmp_path, m
other.write_bytes(_png((9, 9, 9)))
with pytest.raises(PermissionError):
_decode({"bytes": None, "path": str(other)}, {})
+
+
+ "path",
+ [
+ "http://127.0.0.1:9/hf://datasets/other/private@main/secret.png",
+ "http://127.0.0.1:9/{endpoint}/datasets/other/private/resolve/main/secret.png",
+ "{endpoint}.example.invalid/datasets/other/private/resolve/main/secret.png",
+ "zip://secret.png::http://127.0.0.1:9/hf://datasets/other/private@main/a.zip",
+ "http://127.0.0.1:9/secret.png::hf://datasets/other/private@main/a.zip",
+ "hf://datasets/other/private@main/secret.png\nhttp://127.0.0.1:9/",
+ ],
+)
+def test_managed_url_merely_containing_a_hub_path_is_never_fetched(hub_fetches, path):
+ from datasets import config
+ with pytest.raises(PermissionError):
+ _decode({"bytes": None, "path": path.format(endpoint = config.HF_ENDPOINT)}, {})
+ assert hub_fetches == []
+
+
+def test_managed_archive_member_on_the_hub_keeps_its_token(hub_fetches):
+ path = "zip://a.png::hf://datasets/alice/own@main/images.zip"
+ _decode({"bytes": None, "path": path}, {"alice/own": "alice-token"})
+ assert hub_fetches == [(path, "alice-token")]
+
+
+def extracted_root(tmp_path, monkeypatch):
+ from datasets import config
+
+ from utils.security import dataset_media
+
+ root = tmp_path / "datasets-cache" / "downloads" / "extracted"
+ root.mkdir(parents = True)
+ monkeypatch.setattr(config, "EXTRACTED_DATASETS_PATH", str(root))
+ monkeypatch.setattr(dataset_media, "_extracted_sources", {}, raising = False)
+ return root
+
+
+def _zip(path, color):
+ import zipfile
+
+ path.parent.mkdir(parents = True, exist_ok = True)
+ with zipfile.ZipFile(path, "w") as archive:
+ archive.writestr("member.png", _png(color))
+ return path
+
+
+def _extract(archive, account):
+ from datasets.utils.extract import ExtractManager
+ return run_as(account, ExtractManager().extract, str(archive))
+
+
+def test_managed_member_of_its_own_extracted_archive_still_decodes(extracted_root):
+ from utils.security import dataset_media
+
+ archive = run_as(
+ ALICE, lambda: _zip(storage_roots.dataset_uploads_root() / "own.zip", (5, 6, 7))
+ )
+ member = {"bytes": None, "path": str(extracted_root / _extract(archive, ALICE) / "member.png")}
+ assert _decode(member, {}).getpixel((0, 0)) == (5, 6, 7)
+ # A later process reads the record left beside the extraction.
+ getattr(dataset_media, "_extracted_sources", {}).clear()
+ assert _decode(member, {}).getpixel((0, 0)) == (5, 6, 7)
+
+
+def test_managed_member_of_another_accounts_extracted_archive_is_refused(extracted_root, tmp_path):
+ from utils.account_context import OWNER
+
+ from utils.security import dataset_media
+
+ archive = _zip(tmp_path / "owner-private" / "private.zip", (1, 1, 1))
+ member = {"bytes": None, "path": str(extracted_root / _extract(archive, OWNER) / "member.png")}
+ with pytest.raises(PermissionError):
+ _decode(member, {})
+ assert _decode(member, {}, OWNER).getpixel((0, 0)) == (1, 1, 1)
+ getattr(dataset_media, "_extracted_sources", {}).clear()
+ with pytest.raises(PermissionError):
+ _decode(member, {})
+
+ # An extraction nothing recorded (or whose record names a different archive) has no readable source.
+ unrecorded = extracted_root / ("0" * 64) / "member.png"
+ unrecorded.parent.mkdir()
+ unrecorded.write_bytes(_png((2, 2, 2)))
+ own = run_as(ALICE, lambda: _zip(storage_roots.dataset_uploads_root() / "own.zip", (3, 3, 3)))
+ (extracted_root / ("0" * 64 + ".unsloth-source")).write_text(str(own))
+ with pytest.raises(PermissionError):
+ _decode({"bytes": None, "path": str(unrecorded)}, {})
+
+
+def test_managed_extraction_record_is_read_once_per_process(extracted_root, monkeypatch):
+ from utils.security import dataset_media
+
+ archive = run_as(
+ ALICE, lambda: _zip(storage_roots.dataset_uploads_root() / "own.zip", (5, 6, 7))
+ )
+ member = {"bytes": None, "path": str(extracted_root / _extract(archive, ALICE) / "member.png")}
+ dataset_media._extracted_sources.clear()
+ reads = []
+ real_open = os.open
+
+ def counting_open(file, *args, **kwargs):
+ if str(file).endswith(".unsloth-source"):
+ reads.append(file)
+ return real_open(file, *args, **kwargs)
+
+ monkeypatch.setattr(dataset_media.os, "open", counting_open)
+ for _ in range(3):
+ assert _decode(member, {}).getpixel((0, 0)) == (5, 6, 7)
+ assert len(reads) == 1
+
+
+def test_managed_extraction_record_is_published_whole(extracted_root):
+ from utils.security import dataset_media
+
+ archive = run_as(
+ ALICE, lambda: _zip(storage_roots.dataset_uploads_root() / "own.zip", (5, 6, 7))
+ )
+ from datasets.utils.extract import ExtractManager
+
+ top = ExtractManager(cache_dir = None)._get_output_path(str(archive))
+ # A record left empty by an interrupted writer is replaced by the next extraction.
+ (extracted_root / (os.path.basename(top) + ".unsloth-source")).write_text("")
+ member = {"bytes": None, "path": str(extracted_root / _extract(archive, ALICE) / "member.png")}
+ dataset_media._extracted_sources.clear()
+ assert _decode(member, {}).getpixel((0, 0)) == (5, 6, 7)
+ assert not [p for p in os.listdir(extracted_root) if ".unsloth-source." in p]
+
+
+def test_managed_oversized_extraction_record_is_refused_unread(extracted_root, monkeypatch):
+ from utils.security import dataset_media
+
+ archive = run_as(
+ ALICE, lambda: _zip(storage_roots.dataset_uploads_root() / "own.zip", (5, 6, 7))
+ )
+ top = _extract(archive, ALICE)
+ member = {"bytes": None, "path": str(extracted_root / top / "member.png")}
+ sidecar = extracted_root / (top + ".unsloth-source")
+ # A record padded far past any path length is refused without being read whole.
+ sidecar.write_text(str(archive) + "\0" * (16 << 20))
+ dataset_media._extracted_sources.clear()
+ import tracemalloc
+
+ tracemalloc.start()
+ try:
+ assert dataset_media._extracted_source(member["path"]) == ""
+ peak = tracemalloc.get_traced_memory()[1]
+ finally:
+ tracemalloc.stop()
+ assert peak < (1 << 20)
+ with pytest.raises(PermissionError):
+ _decode(member, {})
+
+
[email protected](not hasattr(os, "mkfifo"), reason = "needs FIFOs")
+def test_managed_extraction_record_planted_as_fifo_is_refused_without_blocking(
+ extracted_root, monkeypatch
+):
+ import threading
+
+ from utils.security import dataset_media
+
+ archive = run_as(
+ ALICE, lambda: _zip(storage_roots.dataset_uploads_root() / "own.zip", (5, 6, 7))
+ )
+ top = _extract(archive, ALICE)
+ member = {"bytes": None, "path": str(extracted_root / top / "member.png")}
+ sidecar = extracted_root / (top + ".unsloth-source")
+ sidecar.unlink()
+ os.mkfifo(sidecar)
+ dataset_media._extracted_sources.clear()
+ result = []
+ reader = threading.Thread(
+ target = lambda: result.append(dataset_media._extracted_source(member["path"])),
+ daemon = True,
+ )
+ reader.start()
+ reader.join(5)
+ if reader.is_alive():
+ # Unblock the stuck open so the thread can end, then fail.
+ with open(sidecar, "wb"):
+ pass
+ reader.join(5)
+ pytest.fail("reading a FIFO extraction record blocked")
+ assert result == [""]
+ with pytest.raises(PermissionError):
+ _decode(member, {})
diff --git a/studio/backend/utils/security/dataset_media.py b/studio/backend/utils/security/dataset_media.py
index 6a72cfb7b45..95ee7685f94 100644
--- a/studio/backend/utils/security/dataset_media.py
+++ b/studio/backend/utils/security/dataset_media.py
@@ -3,19 +3,10 @@
"""Keep a managed account's dataset media to files it may read, and validate Arrow data files.
-A ``datasets`` Image / Audio / Video / Pdf / Nifti cell holding a ``path`` and no ``bytes`` is opened by the
-feature's decoder in whichever process reads the row: a local path is read from disk, and a Hub URL is fetched
-with ``token_per_repo_id[repo]``, which falls back to the process's own Hub credentials for any repo the dataset
-was not loaded from. Any dataset can carry such cells, so for a managed account a local path must be one the
-account may read (its own roots, a Hub snapshot it can see or its training dataset's, an archive ``datasets``
-extracted), a Hub URL is fetched with the token the caller supplied for its repo or anonymously, and any other
-URL is refused. The owner is unchanged.
-
-pyarrow's IPC reader trusts the lengths and offsets an ``.arrow`` file declares, so the ``arrow`` builder validates
-each table in full before ``datasets`` converts its rows: a malformed file raises ``ArrowInvalid`` (a ValueError)
-instead of reading past its buffers. ``datasets``' own cache files are memory-mapped by its reader, not this builder.
-
-``install()`` patches ``datasets`` once it is imported, without importing it at startup; it is idempotent.
+Media decoders open a cell's ``path`` locally or fetch Hub URLs with the server's token as fallback, so for a
+managed account local paths must be readable by it (extracted archive members only via a recorded, readable
+source archive), Hub URLs use only a caller-supplied token, and other URLs are refused. ``.arrow`` tables are
+validated in full because pyarrow's IPC reader trusts declared offsets. ``install()`` patches lazily, idempotent.
"""
import functools
@@ -23,7 +14,9 @@ import importlib.abc
import importlib.util
import os
import re
+import stat
import sys
+import tempfile
_DECODERS = {
"datasets.features.image": "Image",
@@ -33,10 +26,15 @@ _DECODERS = {
"datasets.features.nifti": "Nifti",
}
_ARROW = "datasets.packaged_modules.arrow.arrow"
+_EXTRACT = "datasets.utils.extract"
+_SOURCE_SUFFIX = ".unsloth-source"
+_SOURCE_MAX_BYTES = 16384
+# Protocols ``datasets`` chains in front of an outer location ("zip://member::<outer>").
_MARK = "_unsloth_dataset_media_guard"
_SNAPSHOT_REPO = re.compile(r"[\\/]datasets--([^\\/]+?)--([^\\/]+)[\\/]")
-# Hub dataset repos the process was started to train on (a worker serves one account's job).
_dataset_repos: set[str] = set()
+# Extraction dir (real path) -> source archive.
+_extracted_sources: dict[str, str] = {}
class MediaPathRefused(PermissionError, ValueError):
@@ -77,13 +75,40 @@ def _training_snapshot(path: str) -> bool:
)
-def _extracted_archive(path: str) -> bool:
+def _extracted_source(path: str):
+ """The archive an extracted file came from, or None when it is not under the extraction cache."""
from datasets import config
- root = os.path.realpath(str(config.EXTRACTED_DATASETS_PATH))
- return os.path.realpath(path).startswith(root.rstrip(os.sep) + os.sep)
+ from datasets.utils.extract import ExtractManager
-
-def _require_readable(path: str) -> None:
+ root = os.path.realpath(str(config.EXTRACTED_DATASETS_PATH)).rstrip(os.sep) + os.sep
+ real = os.path.realpath(path)
+ if not real.startswith(root):
+ return None
+ top = root + real[len(root) :].split(os.sep, 1)[0]
+ source = _extracted_sources.get(top)
+ if source is None:
+ try:
+ # A record planted as a FIFO or a link (an archive member can name it) must not block or redirect the read.
+ flags = os.O_RDONLY | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_NOFOLLOW", 0)
+ fd = os.open(top + _SOURCE_SUFFIX, flags)
+ with os.fdopen(fd, "rb") as f:
+ if not stat.S_ISREG(os.fstat(f.fileno()).st_mode):
+ return ""
+ raw = f.read(_SOURCE_MAX_BYTES + 1)
+ if len(raw) > _SOURCE_MAX_BYTES:
+ return ""
+ source = raw.decode("utf-8")
+ except (OSError, UnicodeDecodeError):
+ return ""
+ # The directory is named by a hash of the archive's path, so a recorded source can not name another one.
+ expected = ExtractManager(cache_dir = None)._get_output_path(source)
+ if os.path.realpath(expected) != top:
+ return ""
+ _extracted_sources[top] = source
+ return source
+
+
+def _require_readable(path: str, depth: int = 0) -> None:
from fastapi import HTTPException
from core.training.account_jobs import visible_cached_path
@@ -91,9 +116,49 @@ def _require_readable(path: str) -> None:
path = os.path.abspath(os.path.expanduser(path))
try:
visible_cached_path(path, "dataset")
+ return
except HTTPException:
- if not (_training_snapshot(path) or _extracted_archive(path)):
- _refuse(path)
+ pass
+ if _training_snapshot(path):
+ return
+ source = _extracted_source(path)
+ if not source or depth >= 4:
+ _refuse(path)
+ try:
+ _require_readable(source, depth + 1)
+ except MediaPathRefused:
+ _refuse(path)
+
+
+def _record_extraction(extract):
+ @functools.wraps(extract)
+ def recorded(self, input_path, *args, **kwargs):
+ output_path = extract(self, input_path, *args, **kwargs)
+ if output_path != input_path:
+ source, top = os.path.abspath(str(input_path)), os.path.realpath(str(output_path))
+ _extracted_sources[top] = source
+ # Atomic rename: readers never see a partial record.
+ temp = None
+ try:
+ fd, temp = tempfile.mkstemp(
+ dir = os.path.dirname(top), prefix = os.path.basename(top) + _SOURCE_SUFFIX + "."
+ )
+ with os.fdopen(fd, "w", encoding = "utf-8") as f:
+ f.write(source)
+ os.chmod(temp, 0o644) # mkstemp's 0600 would hide the record from other users of the cache
+ os.replace(temp, top + _SOURCE_SUFFIX)
+ temp = None
+ except OSError:
+ pass
+ finally:
+ if temp is not None:
+ try:
+ os.unlink(temp)
+ except OSError:
+ pass
+ return output_path
+
+ return recorded
def require_readable_media(path) -> None:
@@ -107,6 +172,40 @@ _READ_THROUGH_PROTOCOLS = frozenset(
)
+def _anchored(pattern: str):
+ """``pattern``'s ``{field}`` placeholders as groups and the rest literal, matched against a whole string."""
+ parts = re.split(r"{([^:}]+)(?::[^}]+)?}", pattern)
+ regex = "".join(
+ f"(?P<{part}>.+)" if index % 2 else re.escape(part) for index, part in enumerate(parts)
+ )
+ return re.compile(regex)
+
+
+def _hub_fields(source: str):
+ """Repo fields of a Hub dataset file URL, else None. Matched whole: string_to_dict searches anywhere."""
+ from datasets import config
+ from datasets.utils.py_utils import string_to_dict
+
+ endpoint = str(config.HF_ENDPOINT).rstrip("/")
+ if source.startswith(endpoint + "/"):
+ pattern = config.HUB_DATASETS_URL
+ elif source.startswith("hf://"):
+ pattern = config.HUB_DATASETS_HFFS_URL
+ else:
+ return None
+ match = _anchored(pattern).fullmatch(source)
+ if match is None:
+ return None
+ # The decoders key the token on string_to_dict's answer: it must name the same repo.
+ try:
+ theirs = string_to_dict(source, pattern)
+ except ValueError:
+ theirs = None
+ if not theirs or theirs.get("repo_id") != match.group("repo_id"):
+ return None
+ return match.groupdict()
+
+
def _permitted_tokens(value, token_per_repo_id):
"""The token mapping the decoder may use for ``value``; raises for a path the account may not read."""
if isinstance(value, str):
@@ -117,9 +216,7 @@ def _permitted_tokens(value, token_per_repo_id):
return token_per_repo_id
if raw is not None or not isinstance(path, str) or not path:
return token_per_repo_id
- from datasets import config
from datasets.utils.file_utils import is_local_path
- from datasets.utils.py_utils import string_to_dict
# A chained URL ("zip://inner::<outer>") reads the outer location, but only an archive,
# compression or cache hop reads through to the next one: `file://x::hf://...` opens x.
@@ -129,16 +226,7 @@ def _permitted_tokens(value, token_per_repo_id):
if is_local_path(source):
_require_readable(source)
return token_per_repo_id
- # The decoders' own lookup: whatever key they will read is the one given a token here.
- pattern = (
- config.HUB_DATASETS_URL
- if source.startswith(config.HF_ENDPOINT)
- else config.HUB_DATASETS_HFFS_URL
- )
- try:
- fields = string_to_dict(source, pattern)
- except ValueError:
- fields = None
+ fields = _hub_fields(source)
if fields is None:
# Not a Hub file the decoder can key a token on: it would dial it with the server's credentials.
_refuse(path)
@@ -189,6 +277,11 @@ def _patch_attribute(owner, attribute, wrapper) -> None:
def _patch(module) -> None:
+ if module.__name__ == _EXTRACT:
+ owner = getattr(module, "ExtractManager", None)
+ if owner is not None:
+ _patch_attribute(owner, "extract", _record_extraction)
+ return
if module.__name__ == _ARROW:
owner = getattr(module, "Arrow", None)
if owner is not None:
@@ -218,7 +311,7 @@ class _Finder(importlib.abc.MetaPathFinder):
path,
target = None,
):
- if name not in _DECODERS and name != _ARROW:
+ if name not in _DECODERS and name not in (_ARROW, _EXTRACT):
return None
sys.meta_path.remove(self)
try:
@@ -232,7 +325,7 @@ class _Finder(importlib.abc.MetaPathFinder):
def install() -> None:
- for name in (*_DECODERS, _ARROW):
+ for name in (*_DECODERS, _ARROW, _EXTRACT):
if name in sys.modules:
_patch(sys.modules[name])
if not any(isinstance(finder, _Finder) for finder in sys.meta_path):