Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: a sandbox read waits out a legacy move's staging window (#11876)

Daniel Han77ecdbb14ae8
diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.py
index 9609f61cb2d..2f6002c31fc 100644
--- a/studio/backend/core/inference/tools.py
+++ b/studio/backend/core/inference/tools.py
@@ -10564,15 +10564,9 @@ def _legacy_lock_peek(name: str) -> "threading.Lock | None":
_LEGACY_SHARED_BUCKET = "_invalid"
-def _legacy_session_dir(session_id: str) -> "str | None":
- """This session's directory at the legacy root, while one is still there.
-
- Both names, like the migration itself: a chat from before the upgrade whose
- id starts with the derived prefix kept its folder under the literal id.
- """
- if not is_owner_context():
- return None
- legacy_root = _legacy_sandbox_root()
+def _legacy_names(session_id: str) -> "list[str]":
+ """Every name this session's folder can have at the legacy root, which is also the key its move
+ is locked under."""
names = [_sandbox_name(session_id)]
if not _usable_session_id(session_id):
# Before this change an id the filesystem could not hold shared one bucket with every other such chat: read
@@ -10582,13 +10576,50 @@ def _legacy_session_dir(session_id: str) -> "str | None":
# Only the derived-prefix case: an id the old code could hold kept its folder under the literal name while
# _sandbox_name now hashes it. A fallback name is nobody's chat: every session-less call ran in there.
names.append(session_id)
- for name in names:
+ return names
+
+
+def _marked_sandbox_after_moves(root: str, session_id: str) -> "str | None | bool":
+ """_marked_sandbox_in with every legacy move of this session held off, or False when no move of
+ it ever started. Under whichever name it was moved from: a chat whose id starts with the
+ derived prefix moves under the literal id, and its staging tree is marked with the derived one."""
+ locks = [_legacy_lock_peek(name) for name in _legacy_names(session_id)]
+ locks = [lock for lock in locks if lock is not None]
+ if not locks:
+ return False
+ # All at once, so no move can start between the wait and the look. A mover only ever holds one of these, so taking
+ # several in a fixed order cannot deadlock against it.
+ with contextlib.ExitStack() as held:
+ for lock in locks:
+ held.enter_context(lock)
+ return _marked_sandbox_in(root, session_id)
+
+
+def _legacy_session_dir(session_id: str) -> "str | None":
+ """This session's directory at the legacy root, while one is still there.
+
+ Both names, like the migration itself: a chat from before the upgrade whose
+ id starts with the derived prefix kept its folder under the literal id.
+ """
+ if not is_owner_context():
+ return None
+ legacy_root = _legacy_sandbox_root()
+ for name in _legacy_names(session_id):
candidate = os.path.join(legacy_root, name)
- if not os.path.isdir(candidate) or os.path.islink(candidate):
+ if os.path.islink(candidate):
continue
+ if os.path.isdir(candidate):
+ lock = _legacy_lock_for(name)
+ else:
+ # Gone from the legacy root can mean sitting in staging, where neither root holds it and the caller would
+ # fall through to a destination that does not exist yet. The same durable trace
+ # _migrate_one_legacy_session waits on: an entry means a move began, so waiting on it lets the move land.
+ lock = _legacy_lock_peek(name)
+ if lock is None:
+ continue
# Under this session's move lock, and checked again inside it: the move is a rename, so a path handed back
# mid-move lists nothing and 404s every card. One that already ran sends the caller to the destination.
- with _legacy_lock_for(name):
+ with lock:
if os.path.isdir(candidate) and not os.path.islink(candidate):
return candidate
return None
@@ -10958,6 +10989,14 @@ def resolve_sandbox_workdir(session_id: str | None = None) -> str:
# A migration that moved the tree but could not rename it into place leaves the only copy under a marked name,
# at any root.
ours = _marked_sandbox_in(root, session_id)
+ if ours and _STAGING_SUFFIX in os.path.basename(ours):
+ # A live move marks its staging tree before renaming it into place, so this can also be one still moving,
+ # which the rename is about to take away. Movers hold the session's lock for the whole move, so once it is
+ # free a staging tree that is still there is a stranded one, and one that landed or rolled back is found
+ # where it went.
+ settled = _marked_sandbox_after_moves(root, session_id)
+ if settled is not False:
+ ours = settled
if ours:
return ours
# Right after an upgrade the files can still be at the legacy root: the move runs in the background and can
@@ -10965,6 +11004,13 @@ def resolve_sandbox_workdir(session_id: str | None = None) -> str:
legacy = _legacy_session_dir(session_id)
if legacy:
return legacy
+ if not os.path.isdir(workdir):
+ # The lookup above can have waited out a move the first scan ran ahead of. One whose rename and rollback
+ # both failed leaves the only copy in a marked staging tree, which that scan never saw. Only a session a
+ # move has touched pays for the second listing.
+ ours = _marked_sandbox_after_moves(root, session_id)
+ if ours:
+ return ours
if not _root_is_ours() and not _owned_by_session(workdir, session_id):
# In a root the user pointed us at this chat can be in a fallback whose name nothing recomputes, and a read
# that stops here shows an empty sandbox and 404s the file cards already in the transcript.
diff --git a/studio/backend/tests/test_sandbox_files_and_storage_roots.py b/studio/backend/tests/test_sandbox_files_and_storage_roots.py
index 74d20d6e801..87bfd29a60e 100644
--- a/studio/backend/tests/test_sandbox_files_and_storage_roots.py
+++ b/studio/backend/tests/test_sandbox_files_and_storage_roots.py
@@ -682,13 +682,153 @@ def test_the_legacy_migration_is_startup_work(tmp_path, monkeypatch):
Path(tools.resolve_sandbox_workdir("__LOCALID_upgrade"))
assert (legacy / "sales.csv").is_file()
- tools.migrate_legacy_sandbox_in_background()
- for _ in range(50):
- if not legacy.exists():
- break
- time.sleep(0.05)
+ # Joined, not polled: the legacy folder is gone for the whole staging window, well before the
+ # move lands, so its absence says nothing about whether the migration has finished.
+ mover = tools.migrate_legacy_sandbox_in_background()
+ mover.join(60)
+ assert not mover.is_alive(), "the background migration never finished"
resolved = Path(tools.resolve_sandbox_workdir("__LOCALID_upgrade"))
assert (resolved / "sales.csv").is_file(), "the file did not follow the migration"
+ # A read falls back to the legacy root while a session is still there, so the file being
+ # readable does not by itself show that anything moved.
+ assert not legacy.exists(), "the session was left at the legacy root"
+ assert resolved.resolve().is_relative_to(Path(tools.sandbox_root()).resolve())
+
+
[email protected]("paused_after", ["move", "mark"])
+ "session_id",
+ [
+ "__LOCALID_reading",
+ # Kept its legacy folder under the literal id, so its move is locked under that name while
+ # its marker carries the derived one.
+ "_id-reading",
+ ],
+)
+def test_a_read_does_not_answer_from_inside_a_legacy_move_s_staging_window(
+ tmp_path, monkeypatch, paused_after, session_id
+):
+ """A listing or download that lands while a session sits in staging must wait for the move.
+
+ Through that window the session is in neither root. Before the staging tree is marked a read
+ falls through to the destination before it exists; after, it finds the staging tree, which
+ the rename is about to take away. Either way the sandbox lists empty and every file card 404s.
+ """
+ fake_home = tmp_path / "userprofile"
+ fake_home.mkdir()
+ _shared_setup_11(fake_home, monkeypatch, tmp_path)
+
+ session = fake_home / "studio_sandbox" / session_id
+ session.mkdir(parents = True)
+ (session / "data.csv").write_text("a\n")
+
+ tools = _shared_setup_6()
+
+ staged = threading.Event()
+ release = threading.Event()
+ real_move = shutil.move
+
+ real_mark = tools._mark_sandbox
+
+ def pause_in_staging():
+ staged.set() # the source is gone and the destination is not in place yet
+ release.wait(10)
+
+ def gated_move(source, destination, *args, **kwargs):
+ moved = real_move(source, destination, *args, **kwargs)
+ if paused_after == "move" and tools._STAGING_SUFFIX in os.path.basename(destination):
+ pause_in_staging()
+ return moved
+
+ def gated_mark(path, name):
+ marked = real_mark(path, name)
+ if paused_after == "mark" and tools._STAGING_SUFFIX in os.path.basename(path):
+ pause_in_staging()
+ return marked
+
+ monkeypatch.setattr(tools.shutil, "move", gated_move)
+ monkeypatch.setattr(tools, "_mark_sandbox", gated_mark)
+
+ mover = tools.migrate_legacy_sandbox_in_background()
+ assert staged.wait(10), "the migration never reached the staging window"
+
+ result = {}
+ reader = threading.Thread(
+ target = lambda: result.update(
+ workdir = Path(tools.resolve_sandbox_workdir(session_id)),
+ ),
+ daemon = True,
+ )
+ reader.start()
+ reader.join(1.0)
+ returned_early = not reader.is_alive()
+ release.set()
+ reader.join(10)
+ mover.join(10)
+
+ assert not returned_early, "the read answered from inside the staging window"
+ assert "workdir" in result, "the read never returned"
+ assert (result["workdir"] / "data.csv").is_file(), f"{result['workdir']} lost its files"
+
+
+def test_a_read_that_waited_out_a_stranded_move_finds_the_staging_tree(tmp_path, monkeypatch):
+ """A move whose rename and rollback both fail leaves the only copy in its marked staging tree.
+
+ A read whose first scan ran before the staging tree was marked waits the move out at the
+ legacy lookup, finds the legacy folder gone, and must look for the staging tree again rather
+ than hand back a destination that will never exist.
+ """
+ fake_home = tmp_path / "userprofile"
+ fake_home.mkdir()
+ _shared_setup_11(fake_home, monkeypatch, tmp_path)
+
+ session = fake_home / "studio_sandbox" / "__LOCALID_stranded"
+ session.mkdir(parents = True)
+ (session / "data.csv").write_text("a\n")
+
+ tools = _shared_setup_6()
+
+ staged = threading.Event()
+ release = threading.Event()
+ real_move = shutil.move
+ real_rename = os.rename
+
+ def gated_move(source, destination, *args, **kwargs):
+ moved = real_move(source, destination, *args, **kwargs)
+ if tools._STAGING_SUFFIX in os.path.basename(destination):
+ staged.set() # in staging and not yet marked
+ release.wait(10)
+ return moved
+
+ def failing_rename(source, destination, *args, **kwargs):
+ if tools._STAGING_SUFFIX in os.path.basename(os.fspath(source)):
+ raise OSError("rename refused") # both the rename into place and the rollback
+ return real_rename(source, destination, *args, **kwargs)
+
+ monkeypatch.setattr(tools.shutil, "move", gated_move)
+ monkeypatch.setattr(tools.os, "rename", failing_rename)
+
+ mover = tools.migrate_legacy_sandbox_in_background()
+ assert staged.wait(10), "the migration never reached the staging window"
+
+ result = {}
+ reader = threading.Thread(
+ target = lambda: result.update(
+ workdir = Path(tools.resolve_sandbox_workdir("__LOCALID_stranded")),
+ ),
+ daemon = True,
+ )
+ reader.start()
+ reader.join(1.0)
+ returned_early = not reader.is_alive()
+ release.set()
+ reader.join(10)
+ mover.join(10)
+
+ assert not returned_early, "the read answered from inside the staging window"
+ assert "workdir" in result, "the read never returned"
+ assert tools._STAGING_SUFFIX in result["workdir"].name, result["workdir"]
+ assert (result["workdir"] / "data.csv").is_file(), f"{result['workdir']} lost its files"
def test_the_migration_is_serialised(tmp_path, monkeypatch):