Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: pin managed dataset media URLs to the Hub and scope extracted archive members (#14840)
Daniel Hanf443cf93f8f0
diff --git a/studio/backend/tests/test_dataset_media_decode_guard.py b/studio/backend/tests/test_dataset_media_decode_guard.pyindex 6a66692bf8b..6941de690a0 100644--- a/studio/backend/tests/test_dataset_media_decode_guard.py+++ b/studio/backend/tests/test_dataset_media_decode_guard.py@@ -273,3 +273,190 @@ def test_managed_snapshot_files_of_its_training_dataset_still_decode(tmp_path, mother.write_bytes(_png((9, 9, 9)))with pytest.raises(PermissionError):_decode({"bytes": None, "path": str(other)}, {})+++ "path",+ [+ "http://127.0.0.1:9/hf://datasets/other/private@main/secret.png",+ "http://127.0.0.1:9/{endpoint}/datasets/other/private/resolve/main/secret.png",+ "{endpoint}.example.invalid/datasets/other/private/resolve/main/secret.png",+ "zip://secret.png::http://127.0.0.1:9/hf://datasets/other/private@main/a.zip",+ "http://127.0.0.1:9/secret.png::hf://datasets/other/private@main/a.zip",+ "hf://datasets/other/private@main/secret.png\nhttp://127.0.0.1:9/",+ ],+)+def test_managed_url_merely_containing_a_hub_path_is_never_fetched(hub_fetches, path):+ from datasets import config+ with pytest.raises(PermissionError):+ _decode({"bytes": None, "path": path.format(endpoint = config.HF_ENDPOINT)}, {})+ assert hub_fetches == []+++def test_managed_archive_member_on_the_hub_keeps_its_token(hub_fetches):+ path = "zip://a.png::hf://datasets/alice/own@main/images.zip"+ _decode({"bytes": None, "path": path}, {"alice/own": "alice-token"})+ assert hub_fetches == [(path, "alice-token")]+++def extracted_root(tmp_path, monkeypatch):+ from datasets import config++ from utils.security import dataset_media++ root = tmp_path / "datasets-cache" / "downloads" / "extracted"+ root.mkdir(parents = True)+ monkeypatch.setattr(config, "EXTRACTED_DATASETS_PATH", str(root))+ monkeypatch.setattr(dataset_media, "_extracted_sources", {}, raising = False)+ return root+++def _zip(path, color):+ import zipfile++ path.parent.mkdir(parents = True, exist_ok = True)+ with zipfile.ZipFile(path, "w") as archive:+ archive.writestr("member.png", _png(color))+ return path+++def _extract(archive, account):+ from datasets.utils.extract import ExtractManager+ return run_as(account, ExtractManager().extract, str(archive))+++def test_managed_member_of_its_own_extracted_archive_still_decodes(extracted_root):+ from utils.security import dataset_media++ archive = run_as(+ ALICE, lambda: _zip(storage_roots.dataset_uploads_root() / "own.zip", (5, 6, 7))+ )+ member = {"bytes": None, "path": str(extracted_root / _extract(archive, ALICE) / "member.png")}+ assert _decode(member, {}).getpixel((0, 0)) == (5, 6, 7)+ # A later process reads the record left beside the extraction.+ getattr(dataset_media, "_extracted_sources", {}).clear()+ assert _decode(member, {}).getpixel((0, 0)) == (5, 6, 7)+++def test_managed_member_of_another_accounts_extracted_archive_is_refused(extracted_root, tmp_path):+ from utils.account_context import OWNER++ from utils.security import dataset_media++ archive = _zip(tmp_path / "owner-private" / "private.zip", (1, 1, 1))+ member = {"bytes": None, "path": str(extracted_root / _extract(archive, OWNER) / "member.png")}+ with pytest.raises(PermissionError):+ _decode(member, {})+ assert _decode(member, {}, OWNER).getpixel((0, 0)) == (1, 1, 1)+ getattr(dataset_media, "_extracted_sources", {}).clear()+ with pytest.raises(PermissionError):+ _decode(member, {})++ # An extraction nothing recorded (or whose record names a different archive) has no readable source.+ unrecorded = extracted_root / ("0" * 64) / "member.png"+ unrecorded.parent.mkdir()+ unrecorded.write_bytes(_png((2, 2, 2)))+ own = run_as(ALICE, lambda: _zip(storage_roots.dataset_uploads_root() / "own.zip", (3, 3, 3)))+ (extracted_root / ("0" * 64 + ".unsloth-source")).write_text(str(own))+ with pytest.raises(PermissionError):+ _decode({"bytes": None, "path": str(unrecorded)}, {})+++def test_managed_extraction_record_is_read_once_per_process(extracted_root, monkeypatch):+ from utils.security import dataset_media++ archive = run_as(+ ALICE, lambda: _zip(storage_roots.dataset_uploads_root() / "own.zip", (5, 6, 7))+ )+ member = {"bytes": None, "path": str(extracted_root / _extract(archive, ALICE) / "member.png")}+ dataset_media._extracted_sources.clear()+ reads = []+ real_open = os.open++ def counting_open(file, *args, **kwargs):+ if str(file).endswith(".unsloth-source"):+ reads.append(file)+ return real_open(file, *args, **kwargs)++ monkeypatch.setattr(dataset_media.os, "open", counting_open)+ for _ in range(3):+ assert _decode(member, {}).getpixel((0, 0)) == (5, 6, 7)+ assert len(reads) == 1+++def test_managed_extraction_record_is_published_whole(extracted_root):+ from utils.security import dataset_media++ archive = run_as(+ ALICE, lambda: _zip(storage_roots.dataset_uploads_root() / "own.zip", (5, 6, 7))+ )+ from datasets.utils.extract import ExtractManager++ top = ExtractManager(cache_dir = None)._get_output_path(str(archive))+ # A record left empty by an interrupted writer is replaced by the next extraction.+ (extracted_root / (os.path.basename(top) + ".unsloth-source")).write_text("")+ member = {"bytes": None, "path": str(extracted_root / _extract(archive, ALICE) / "member.png")}+ dataset_media._extracted_sources.clear()+ assert _decode(member, {}).getpixel((0, 0)) == (5, 6, 7)+ assert not [p for p in os.listdir(extracted_root) if ".unsloth-source." in p]+++def test_managed_oversized_extraction_record_is_refused_unread(extracted_root, monkeypatch):+ from utils.security import dataset_media++ archive = run_as(+ ALICE, lambda: _zip(storage_roots.dataset_uploads_root() / "own.zip", (5, 6, 7))+ )+ top = _extract(archive, ALICE)+ member = {"bytes": None, "path": str(extracted_root / top / "member.png")}+ sidecar = extracted_root / (top + ".unsloth-source")+ # A record padded far past any path length is refused without being read whole.+ sidecar.write_text(str(archive) + "\0" * (16 << 20))+ dataset_media._extracted_sources.clear()+ import tracemalloc++ tracemalloc.start()+ try:+ assert dataset_media._extracted_source(member["path"]) == ""+ peak = tracemalloc.get_traced_memory()[1]+ finally:+ tracemalloc.stop()+ assert peak < (1 << 20)+ with pytest.raises(PermissionError):+ _decode(member, {})++[email protected](not hasattr(os, "mkfifo"), reason = "needs FIFOs")+def test_managed_extraction_record_planted_as_fifo_is_refused_without_blocking(+ extracted_root, monkeypatch+):+ import threading++ from utils.security import dataset_media++ archive = run_as(+ ALICE, lambda: _zip(storage_roots.dataset_uploads_root() / "own.zip", (5, 6, 7))+ )+ top = _extract(archive, ALICE)+ member = {"bytes": None, "path": str(extracted_root / top / "member.png")}+ sidecar = extracted_root / (top + ".unsloth-source")+ sidecar.unlink()+ os.mkfifo(sidecar)+ dataset_media._extracted_sources.clear()+ result = []+ reader = threading.Thread(+ target = lambda: result.append(dataset_media._extracted_source(member["path"])),+ daemon = True,+ )+ reader.start()+ reader.join(5)+ if reader.is_alive():+ # Unblock the stuck open so the thread can end, then fail.+ with open(sidecar, "wb"):+ pass+ reader.join(5)+ pytest.fail("reading a FIFO extraction record blocked")+ assert result == [""]+ with pytest.raises(PermissionError):+ _decode(member, {})diff --git a/studio/backend/utils/security/dataset_media.py b/studio/backend/utils/security/dataset_media.pyindex 6a72cfb7b45..95ee7685f94 100644--- a/studio/backend/utils/security/dataset_media.py+++ b/studio/backend/utils/security/dataset_media.py@@ -3,19 +3,10 @@"""Keep a managed account's dataset media to files it may read, and validate Arrow data files.-A ``datasets`` Image / Audio / Video / Pdf / Nifti cell holding a ``path`` and no ``bytes`` is opened by the-feature's decoder in whichever process reads the row: a local path is read from disk, and a Hub URL is fetched-with ``token_per_repo_id[repo]``, which falls back to the process's own Hub credentials for any repo the dataset-was not loaded from. Any dataset can carry such cells, so for a managed account a local path must be one the-account may read (its own roots, a Hub snapshot it can see or its training dataset's, an archive ``datasets``-extracted), a Hub URL is fetched with the token the caller supplied for its repo or anonymously, and any other-URL is refused. The owner is unchanged.--pyarrow's IPC reader trusts the lengths and offsets an ``.arrow`` file declares, so the ``arrow`` builder validates-each table in full before ``datasets`` converts its rows: a malformed file raises ``ArrowInvalid`` (a ValueError)-instead of reading past its buffers. ``datasets``' own cache files are memory-mapped by its reader, not this builder.--``install()`` patches ``datasets`` once it is imported, without importing it at startup; it is idempotent.+Media decoders open a cell's ``path`` locally or fetch Hub URLs with the server's token as fallback, so for a+managed account local paths must be readable by it (extracted archive members only via a recorded, readable+source archive), Hub URLs use only a caller-supplied token, and other URLs are refused. ``.arrow`` tables are+validated in full because pyarrow's IPC reader trusts declared offsets. ``install()`` patches lazily, idempotent."""import functools@@ -23,7 +14,9 @@ import importlib.abcimport importlib.utilimport osimport re+import statimport sys+import tempfile_DECODERS = {"datasets.features.image": "Image",@@ -33,10 +26,15 @@ _DECODERS = {"datasets.features.nifti": "Nifti",}_ARROW = "datasets.packaged_modules.arrow.arrow"+_EXTRACT = "datasets.utils.extract"+_SOURCE_SUFFIX = ".unsloth-source"+_SOURCE_MAX_BYTES = 16384+# Protocols ``datasets`` chains in front of an outer location ("zip://member::<outer>")._MARK = "_unsloth_dataset_media_guard"_SNAPSHOT_REPO = re.compile(r"[\\/]datasets--([^\\/]+?)--([^\\/]+)[\\/]")-# Hub dataset repos the process was started to train on (a worker serves one account's job)._dataset_repos: set[str] = set()+# Extraction dir (real path) -> source archive.+_extracted_sources: dict[str, str] = {}class MediaPathRefused(PermissionError, ValueError):@@ -77,13 +75,40 @@ def _training_snapshot(path: str) -> bool:)-def _extracted_archive(path: str) -> bool:+def _extracted_source(path: str):+ """The archive an extracted file came from, or None when it is not under the extraction cache."""from datasets import config- root = os.path.realpath(str(config.EXTRACTED_DATASETS_PATH))- return os.path.realpath(path).startswith(root.rstrip(os.sep) + os.sep)+ from datasets.utils.extract import ExtractManager--def _require_readable(path: str) -> None:+ root = os.path.realpath(str(config.EXTRACTED_DATASETS_PATH)).rstrip(os.sep) + os.sep+ real = os.path.realpath(path)+ if not real.startswith(root):+ return None+ top = root + real[len(root) :].split(os.sep, 1)[0]+ source = _extracted_sources.get(top)+ if source is None:+ try:+ # A record planted as a FIFO or a link (an archive member can name it) must not block or redirect the read.+ flags = os.O_RDONLY | getattr(os, "O_NONBLOCK", 0) | getattr(os, "O_NOFOLLOW", 0)+ fd = os.open(top + _SOURCE_SUFFIX, flags)+ with os.fdopen(fd, "rb") as f:+ if not stat.S_ISREG(os.fstat(f.fileno()).st_mode):+ return ""+ raw = f.read(_SOURCE_MAX_BYTES + 1)+ if len(raw) > _SOURCE_MAX_BYTES:+ return ""+ source = raw.decode("utf-8")+ except (OSError, UnicodeDecodeError):+ return ""+ # The directory is named by a hash of the archive's path, so a recorded source can not name another one.+ expected = ExtractManager(cache_dir = None)._get_output_path(source)+ if os.path.realpath(expected) != top:+ return ""+ _extracted_sources[top] = source+ return source+++def _require_readable(path: str, depth: int = 0) -> None:from fastapi import HTTPExceptionfrom core.training.account_jobs import visible_cached_path@@ -91,9 +116,49 @@ def _require_readable(path: str) -> None:path = os.path.abspath(os.path.expanduser(path))try:visible_cached_path(path, "dataset")+ returnexcept HTTPException:- if not (_training_snapshot(path) or _extracted_archive(path)):- _refuse(path)+ pass+ if _training_snapshot(path):+ return+ source = _extracted_source(path)+ if not source or depth >= 4:+ _refuse(path)+ try:+ _require_readable(source, depth + 1)+ except MediaPathRefused:+ _refuse(path)+++def _record_extraction(extract):+ @functools.wraps(extract)+ def recorded(self, input_path, *args, **kwargs):+ output_path = extract(self, input_path, *args, **kwargs)+ if output_path != input_path:+ source, top = os.path.abspath(str(input_path)), os.path.realpath(str(output_path))+ _extracted_sources[top] = source+ # Atomic rename: readers never see a partial record.+ temp = None+ try:+ fd, temp = tempfile.mkstemp(+ dir = os.path.dirname(top), prefix = os.path.basename(top) + _SOURCE_SUFFIX + "."+ )+ with os.fdopen(fd, "w", encoding = "utf-8") as f:+ f.write(source)+ os.chmod(temp, 0o644) # mkstemp's 0600 would hide the record from other users of the cache+ os.replace(temp, top + _SOURCE_SUFFIX)+ temp = None+ except OSError:+ pass+ finally:+ if temp is not None:+ try:+ os.unlink(temp)+ except OSError:+ pass+ return output_path++ return recordeddef require_readable_media(path) -> None:@@ -107,6 +172,40 @@ _READ_THROUGH_PROTOCOLS = frozenset()+def _anchored(pattern: str):+ """``pattern``'s ``{field}`` placeholders as groups and the rest literal, matched against a whole string."""+ parts = re.split(r"{([^:}]+)(?::[^}]+)?}", pattern)+ regex = "".join(+ f"(?P<{part}>.+)" if index % 2 else re.escape(part) for index, part in enumerate(parts)+ )+ return re.compile(regex)+++def _hub_fields(source: str):+ """Repo fields of a Hub dataset file URL, else None. Matched whole: string_to_dict searches anywhere."""+ from datasets import config+ from datasets.utils.py_utils import string_to_dict++ endpoint = str(config.HF_ENDPOINT).rstrip("/")+ if source.startswith(endpoint + "/"):+ pattern = config.HUB_DATASETS_URL+ elif source.startswith("hf://"):+ pattern = config.HUB_DATASETS_HFFS_URL+ else:+ return None+ match = _anchored(pattern).fullmatch(source)+ if match is None:+ return None+ # The decoders key the token on string_to_dict's answer: it must name the same repo.+ try:+ theirs = string_to_dict(source, pattern)+ except ValueError:+ theirs = None+ if not theirs or theirs.get("repo_id") != match.group("repo_id"):+ return None+ return match.groupdict()++def _permitted_tokens(value, token_per_repo_id):"""The token mapping the decoder may use for ``value``; raises for a path the account may not read."""if isinstance(value, str):@@ -117,9 +216,7 @@ def _permitted_tokens(value, token_per_repo_id):return token_per_repo_idif raw is not None or not isinstance(path, str) or not path:return token_per_repo_id- from datasets import configfrom datasets.utils.file_utils import is_local_path- from datasets.utils.py_utils import string_to_dict# A chained URL ("zip://inner::<outer>") reads the outer location, but only an archive,# compression or cache hop reads through to the next one: `file://x::hf://...` opens x.@@ -129,16 +226,7 @@ def _permitted_tokens(value, token_per_repo_id):if is_local_path(source):_require_readable(source)return token_per_repo_id- # The decoders' own lookup: whatever key they will read is the one given a token here.- pattern = (- config.HUB_DATASETS_URL- if source.startswith(config.HF_ENDPOINT)- else config.HUB_DATASETS_HFFS_URL- )- try:- fields = string_to_dict(source, pattern)- except ValueError:- fields = None+ fields = _hub_fields(source)if fields is None:# Not a Hub file the decoder can key a token on: it would dial it with the server's credentials._refuse(path)@@ -189,6 +277,11 @@ def _patch_attribute(owner, attribute, wrapper) -> None:def _patch(module) -> None:+ if module.__name__ == _EXTRACT:+ owner = getattr(module, "ExtractManager", None)+ if owner is not None:+ _patch_attribute(owner, "extract", _record_extraction)+ returnif module.__name__ == _ARROW:owner = getattr(module, "Arrow", None)if owner is not None:@@ -218,7 +311,7 @@ class _Finder(importlib.abc.MetaPathFinder):path,target = None,):- if name not in _DECODERS and name != _ARROW:+ if name not in _DECODERS and name not in (_ARROW, _EXTRACT):return Nonesys.meta_path.remove(self)try:@@ -232,7 +325,7 @@ class _Finder(importlib.abc.MetaPathFinder):def install() -> None:- for name in (*_DECODERS, _ARROW):+ for name in (*_DECODERS, _ARROW, _EXTRACT):if name in sys.modules:_patch(sys.modules[name])if not any(isinstance(finder, _Finder) for finder in sys.meta_path):