Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
A first tool call must not answer from inside a legacy move's staging window (#10953)
Daniel Han0bb3c06b883e
diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.pyindex 5f05021cf8d..0bcba5464cf 100644--- a/studio/backend/core/inference/tools.py+++ b/studio/backend/core/inference/tools.py@@ -8002,28 +8002,46 @@ def _staged_move(source: str, target: str, name: str) -> None:the original still in place; the next launch would read that as a session the new rootalready has and strand the files. Filled under a name nothing resolves to, then renamed,which on one filesystem is atomic."""+ global _legacy_sandbox_migratedstaging = f"{target}{_STAGING_SUFFIX}{uuid.uuid4().hex[:8]}"+ # Announced for exactly as long as neither end of the move is where a reader looks. Anything+ # deciding there is nothing to migrate has to consult this first, or it decides it during the+ # one moment the evidence is missing.+ with _legacy_locks_guard:+ _legacy_moves_in_flight.add(name)try:- shutil.move(source, staging)- except OSError:- # Half filled and ours, and the source is still where it was.- shutil.rmtree(staging, ignore_errors = True)- raise- # Marked here, not after the rename: across filesystems the move has already removed the legacy copy, so from this- # instant the staging tree is the only one there is and a kill before the marker would leave it unfindable.- _preserve_foreign_marker(staging, name)- _mark_sandbox(staging, name)- try:- os.rename(staging, target)- except OSError:- # The move already took the legacy copy, so this tree is the only one and deleting it would lose the files. It- # is marked, so _marked_sandbox_in finds it; put it back and let the next pass retry.try:- os.rename(staging, source)+ shutil.move(source, staging)except OSError:- logger.warning("Sandbox %s left at %s: could not be moved in", name, staging)- raise- _mark_sandbox(target, name)+ # Half filled and ours, and the source is still where it was.+ shutil.rmtree(staging, ignore_errors = True)+ raise+ # Marked here, not after the rename: across filesystems the move has already removed the legacy copy, so from+ # this instant the staging tree is the only one there is and a kill before the marker would leave it unfindable.+ _preserve_foreign_marker(staging, name)+ _mark_sandbox(staging, name)+ try:+ os.rename(staging, target)+ except OSError:+ # The move already took the legacy copy, so this tree is the only one and deleting it would lose the files.+ # It is marked, so _marked_sandbox_in finds it; put it back and let the next pass retry.+ try:+ os.rename(staging, source)+ except OSError:+ logger.warning("Sandbox %s left at %s: could not be moved in", name, staging)+ else:+ # There is something to migrate again, so say so. A whole-tree pass that ran while this sat in staging+ # saw an empty legacy root and can already have called the migration finished; left standing, that+ # retires the very retry this rollback exists to allow and the chat keeps an empty sandbox until the+ # process restarts.+ _legacy_sandbox_migrated = False+ raise+ _mark_sandbox(target, name)+ finally:+ global _legacy_moves_done+ with _legacy_locks_guard:+ _legacy_moves_in_flight.discard(name)+ _legacy_moves_done += 1# Bookkeeping only, never held across a move: starting the background pass and the sweep. Anything that copies a tree@@ -8042,6 +8060,24 @@ def _legacy_lock_for(name: str) -> threading.Lock:return _legacy_session_locks.setdefault(name, threading.Lock())+# Sessions whose move is running right this instant, held by _staged_move across the whole of+# it. Neither root shows the tree while that runs, so this is the only thing that separates+# "the legacy copy is gone because it arrived" from "it is gone because it is in staging".+_legacy_moves_in_flight: "set[str]" = set()+# Every move that has finished, succeeded or rolled back. The set above only answers "right+# now", which cannot see a move that both started and ended inside a whole-tree pass: by the+# end it is empty again although the pass listed the legacy root mid-staging and, if that move+# rolled back, the source is now sitting there unlisted.+_legacy_moves_done = 0+++def _legacy_lock_peek(name: str) -> "threading.Lock | None":+ """The lock covering this session's move, if one was ever started. Never creates the entry: a+ name with nothing at the legacy root must not leave one behind."""+ with _legacy_locks_guard:+ return _legacy_session_locks.get(name)++# Where every id the old code could not use as a directory name went. One bucket for all of them, which is what this# change stops doing, so it is never moved up as though it were a chat: several chats' files are in there._LEGACY_SHARED_BUCKET = "_invalid"@@ -8079,12 +8115,38 @@ def _legacy_session_dir(session_id: str) -> "str | None":def _migrate_one_legacy_session(root: str, name: str) -> None:"""Bring one session up from the legacy root, without waiting for the rest."""- if not is_owner_context() or _legacy_sandbox_migrated:+ if not is_owner_context():+ return+ # The legacy root, not the done flag and not the session directory. _staged_move renames the+ # tree aside into staging before renaming it into place, and through that window neither root+ # holds it; a failing rename then rolls it back. Anything read on one side of that and used on+ # the other describes an instant that has passed, and the caller leaves without files that are+ # sitting at the legacy root again. The flag is no safer than the directory, since a pass can+ # look during the same window, find the root empty, and call the migration finished over a+ # move it never saw. The root is the stable question: removed once the migration is genuinely+ # done, present for the whole of any move, and still there when one failed, which is exactly+ # when this should try rather than skip.+ legacy_root = _legacy_sandbox_root()+ if not os.path.isdir(legacy_root):return- source = os.path.join(_legacy_sandbox_root(), name)- if os.path.islink(source) or not os.path.isdir(source):+ source = os.path.join(legacy_root, name)+ if os.path.islink(source):return- with _legacy_lock_for(name):+ if os.path.isdir(source):+ lock = _legacy_lock_for(name)+ else:+ # Staged away, or never there at all? Only a mover takes the tree, and it takes this lock+ # before it does, so an entry is the durable trace of that. Durable is the point: entries+ # are never removed, so unlike the in-flight set this reading cannot go stale between here+ # and the wait below. No entry means no move ever began for this name, and none can begin+ # without the source appearing, which only a rollback does, and only after a move. Looking+ # without inserting is also what keeps the table bounded by the chats that had a legacy+ # folder, rather than growing one entry per chat for as long as a failed migration leaves+ # the root in place.+ lock = _legacy_lock_peek(name)+ if lock is None:+ return+ with lock:if not os.path.isdir(source):return # the background pass got there first# Through the resolver, like the whole-tree pass: at a shared root the plain name can be the user's own, and@@ -8127,21 +8189,27 @@ def _migrate_legacy_sandbox(root: str) -> None:if _legacy_sandbox_migrated:return# Only when nothing movable is left: a file locked on Windows is retryable, and one attempt strands it once- # the destination exists.- if _migrate_legacy_sandbox_locked(root):- _legacy_sandbox_migrated = True+ # the destination exists. The flag is committed inside, in the same guarded moment as the check that earns+ # it, so a rollback cannot be decided against and then overwritten by a verdict formed before it happened.+ _migrate_legacy_sandbox_locked(root)def _migrate_legacy_sandbox_locked(root: str) -> bool:"""True when the legacy root holds nothing that could still be moved. A collision is not afailure: the new root already has that session, and the legacy copy is deliberately left forthe user to find."""+ global _legacy_sandbox_migratedlegacy = _legacy_sandbox_root()try:+ # Read before the tree is looked at, so any move that runs from here on is counted.+ with _legacy_locks_guard:+ moves_before = _legacy_moves_doneif os.path.realpath(legacy) == os.path.realpath(root) or not os.path.isdir(legacy):+ _legacy_sandbox_migrated = Truereturn Trueos.makedirs(root, exist_ok = True)moved = 0+ own_moves = 0complete = Truefor name in os.listdir(legacy):source = os.path.join(legacy, name)@@ -8160,6 +8228,7 @@ def _migrate_legacy_sandbox_locked(root: str) -> bool:with _legacy_lock_for(name):if not os.path.isdir(source) or os.path.exists(target):continue # a request path moved it while we waited+ own_moves += 1 # before the call: one that raises still advances the count_staged_move(source, target, name)moved += 1except OSError as error:@@ -8169,6 +8238,19 @@ def _migrate_legacy_sandbox_locked(root: str) -> bool:logger.warning("Could not move sandbox %s: %s", name, error)if moved:logger.info("Moved %d chat sandbox folder(s) from %s to %s", moved, legacy, root)+ # Any move but this pass's own, overlapping any part of it. Such a session was in neither root while the+ # listing above ran, so that listing is not evidence about it: it may have arrived, or rolled back and be+ # sitting at the legacy root right now, unlisted. Counting rather than asking what is in flight is the point,+ # since one that started and ended inside the pass leaves nothing in flight to find. The legacy root has to+ # stay too, as that is where a rollback renames back into. Reporting unfinished is what brings the next pass.+ with _legacy_locks_guard:+ overlapped = (+ bool(_legacy_moves_in_flight) or _legacy_moves_done - moves_before != own_moves+ )+ if not overlapped and complete:+ _legacy_sandbox_migrated = True+ if overlapped:+ return False# Empty only: a leftover is a collision the user should still find.try:os.rmdir(legacy)diff --git a/studio/backend/tests/test_sandbox_files_and_storage_roots.py b/studio/backend/tests/test_sandbox_files_and_storage_roots.pyindex 322b7b9692b..027b1b3f8b0 100644--- a/studio/backend/tests/test_sandbox_files_and_storage_roots.py+++ b/studio/backend/tests/test_sandbox_files_and_storage_roots.py@@ -719,6 +719,387 @@ def test_the_migration_is_serialised(tmp_path, monkeypatch):assert (workdir / "data.csv").is_file(), f"{workdir} lost its files"+def test_a_first_tool_call_waits_out_a_move_already_in_staging(tmp_path, monkeypatch):+ """The window above, forced rather than raced for.++ ``_staged_move`` renames the tree aside into staging before renaming it into place, so in+ between the legacy copy is gone and the destination does not exist yet. A first tool call+ landing there used to read the missing source as nothing to do, create an empty sandbox+ under the name the pending rename needs, and hand the chat a directory with none of its+ files in it. The test above only hits this when the scheduler happens to line the two up,+ which on CI was about one run in a hundred; this one blocks the mover inside the window."""+ import shutil+ import threading++ fake_home = tmp_path / "userprofile"+ fake_home.mkdir()+ _shared_setup_11(fake_home, monkeypatch, tmp_path)++ session = fake_home / "studio_sandbox" / "__LOCALID_staged"+ session.mkdir(parents = True)+ (session / "data.csv").write_text("a\n")++ tools = _shared_setup_6()++ staged = threading.Event()+ release = threading.Event()+ real_move = shutil.move++ def gated_move(source, destination, *args, **kwargs):+ moved = real_move(source, destination, *args, **kwargs)+ if "__LOCALID_staged" in str(destination):+ staged.set() # the source is gone and the destination is not in place yet+ release.wait(10)+ return moved++ monkeypatch.setattr(tools.shutil, "move", gated_move)++ mover = threading.Thread(+ target = lambda: tools._migrate_legacy_sandbox(tools.sandbox_root()),+ daemon = True,+ )+ mover.start()+ assert staged.wait(10), "the migration never reached the staging window"++ result = {}+ caller = threading.Thread(+ target = lambda: result.update(+ workdir = Path(tools.get_sandbox_workdir("__LOCALID_staged")),+ ),+ daemon = True,+ )+ caller.start()+ caller.join(1.0)+ # The point of the test: answering from inside the window is what loses the files.+ returned_early = not caller.is_alive()+ release.set()+ caller.join(10)+ mover.join(10)++ assert not returned_early, "the first tool call answered from inside the staging window"+ assert "workdir" in result, "the first tool call never returned"+ assert (result["workdir"] / "data.csv").is_file(), f"{result['workdir']} lost its files"+++def test_the_migrated_flag_cannot_be_set_over_a_move_still_in_staging(tmp_path, monkeypatch):+ """A whole-tree pass must not call the migration finished over a move still in staging.++ While one session sits in staging, neither root holds it, so a pass that lists the legacy+ root right then finds it empty and moves nothing. Reporting that as finished retires the+ retry a rollback would need, and removes the legacy root a rollback renames back into. The+ first tool call underneath has to come back with the files either way."""+ import shutil+ import threading++ fake_home = tmp_path / "userprofile"+ fake_home.mkdir()+ _shared_setup_11(fake_home, monkeypatch, tmp_path)++ session = fake_home / "studio_sandbox" / "__LOCALID_flag"+ session.mkdir(parents = True)+ (session / "data.csv").write_text("a\n")++ tools = _shared_setup_6()++ staged = threading.Event()+ release = threading.Event()+ real_move = shutil.move++ def gated_move(source, destination, *args, **kwargs):+ moved = real_move(source, destination, *args, **kwargs)+ if "__LOCALID_flag" in str(destination):+ staged.set()+ release.wait(10)+ return moved++ monkeypatch.setattr(tools.shutil, "move", gated_move)++ root = tools.sandbox_root()+ mover = threading.Thread(+ target = lambda: tools._migrate_one_legacy_session(root, "__LOCALID_flag"),+ daemon = True,+ )+ mover.start()+ assert staged.wait(10), "the per-session move never reached the staging window"++ tools._migrate_legacy_sandbox(root) # sees an empty legacy root while this one is in staging+ assert (+ not tools._legacy_sandbox_migrated+ ), "a pass that ran through another session's staging window called the migration finished"++ result = {}+ caller = threading.Thread(+ target = lambda: result.update(+ workdir = Path(tools.get_sandbox_workdir("__LOCALID_flag")),+ ),+ daemon = True,+ )+ caller.start()+ caller.join(1.0)+ returned_early = not caller.is_alive()+ release.set()+ caller.join(10)+ mover.join(10)++ assert not returned_early, "the first tool call answered from inside the staging window"+ assert "workdir" in result, "the first tool call never returned"+ assert (result["workdir"] / "data.csv").is_file(), f"{result['workdir']} lost its files"+++def test_a_rolled_back_move_puts_the_migration_back_on_the_table(tmp_path, monkeypatch):+ """_staged_move restores the legacy copy when the final rename fails, and says so in its+ own comment: put it back and let the next pass retry. With the done flag already set by a+ pass that ran while this sat in staging there is no next pass, and every later call short+ circuits on the flag, so the chat keeps an empty sandbox until the process restarts."""+ import os+ import shutil+ import threading++ fake_home = tmp_path / "userprofile"+ fake_home.mkdir()+ _shared_setup_11(fake_home, monkeypatch, tmp_path)++ legacy = fake_home / "studio_sandbox" / "__LOCALID_rollback"+ legacy.mkdir(parents = True)+ (legacy / "data.csv").write_text("a\n")++ tools = _shared_setup_6()++ staged = threading.Event()+ release = threading.Event()+ real_move = shutil.move+ real_rename = os.rename+ rename_failed = []++ def gated_move(source, destination, *args, **kwargs):+ moved = real_move(source, destination, *args, **kwargs)+ if "__LOCALID_rollback" in str(destination):+ staged.set()+ release.wait(10)+ return moved++ def failing_rename(source, destination, *args, **kwargs):+ # Only the staging -> target step, which is the one that fails on a live handle.+ if str(destination).endswith("__LOCALID_rollback") and not rename_failed:+ rename_failed.append(True)+ raise OSError(39, "Directory not empty")+ return real_rename(source, destination, *args, **kwargs)++ monkeypatch.setattr(tools.shutil, "move", gated_move)+ monkeypatch.setattr(tools.os, "rename", failing_rename)++ root = tools.sandbox_root()+ mover = threading.Thread(+ target = lambda: tools._migrate_one_legacy_session(root, "__LOCALID_rollback"),+ daemon = True,+ )+ mover.start()+ assert staged.wait(10), "the per-session move never reached the staging window"++ # The pass itself now declines to finish over a move in staging, so the flag is set by hand+ # here: it stands for the one gap that check cannot close, between a pass reading an empty+ # in-flight set and assigning the flag, during which a request-path move can start.+ tools._migrate_legacy_sandbox(root)+ tools._legacy_sandbox_migrated = True++ release.set()+ mover.join(10)++ assert rename_failed, "the rename was never made to fail"+ assert (legacy / "data.csv").is_file(), "the rollback did not restore the legacy copy"+ assert (+ not tools._legacy_sandbox_migrated+ ), "a restored legacy copy left the migration marked finished, so nothing retries it"++ # And the retry the rollback asked for actually happens on the next call.+ workdir = Path(tools.get_sandbox_workdir("__LOCALID_rollback"))+ assert (workdir / "data.csv").is_file(), f"{workdir} never got the restored files"+++def test_a_move_that_begins_and_ends_inside_the_pass_still_counts(tmp_path, monkeypatch):+ """The reversed interleaving: the rollback finishes before the pass looks.++ Asking what is in flight only answers "right now". A move that starts after the pass has+ listed the legacy root and rolls back before the pass reaches its check leaves nothing in+ flight to find, yet the listing never saw it and the restored source is sitting at the+ legacy root unlisted. The pass would call the migration done over it."""+ import os+ import shutil++ fake_home = tmp_path / "userprofile"+ fake_home.mkdir()+ _shared_setup_11(fake_home, monkeypatch, tmp_path)++ legacy = fake_home / "studio_sandbox" / "__LOCALID_inside"+ legacy.mkdir(parents = True)+ (legacy / "data.csv").write_text("a\n")++ tools = _shared_setup_6()+ root = tools.sandbox_root()++ real_listdir = os.listdir+ real_rename = os.rename+ rename_failed = []+ ran = []++ def failing_rename(source, destination, *args, **kwargs):+ if str(destination).endswith("__LOCALID_inside") and not rename_failed:+ rename_failed.append(True)+ raise OSError(39, "Directory not empty")+ return real_rename(source, destination, *args, **kwargs)++ def listdir_then_move(path, *args, **kwargs):+ entries = real_listdir(path, *args, **kwargs)+ # Once, right after the pass reads the legacy root: a whole move, start to rollback,+ # inside the pass and invisible to it.+ if not ran and os.path.realpath(str(path)) == os.path.realpath(+ str(fake_home / "studio_sandbox")+ ):+ ran.append(True)+ monkeypatch.setattr(tools.os, "rename", failing_rename)+ try:+ tools._migrate_one_legacy_session(root, "__LOCALID_inside")+ except OSError:+ pass+ monkeypatch.setattr(tools.os, "rename", real_rename)+ return [e for e in entries if e != "__LOCALID_inside"]+ return entries++ monkeypatch.setattr(tools.os, "listdir", listdir_then_move)+ monkeypatch.setattr(tools.shutil, "move", shutil.move)++ tools._migrate_legacy_sandbox(root)++ assert ran, "the nested move never ran"+ assert rename_failed, "the rename was never made to fail"+ assert (legacy / "data.csv").is_file(), "the rollback did not restore the legacy copy"+ assert (+ not tools._legacy_sandbox_migrated+ ), "the pass finished over a move it never listed, so the restored copy is stranded"++ workdir = Path(tools.get_sandbox_workdir("__LOCALID_inside"))+ assert (workdir / "data.csv").is_file(), f"{workdir} never got the restored files"+++def test_a_rollback_between_the_two_reads_is_not_read_as_nothing_to_do(tmp_path, monkeypatch):+ """Whether the session directory is there and whether a move is running are two reads.++ A failing rename between them restores the tree, so the absence the first read saw and the+ quiet the second saw describe different instants and neither is now. A caller splitting its+ decision across that pair leaves without the files, which are back at the legacy root, and+ caches an empty sandbox in their place. Nothing else picks it up either: the rolled-back+ mover is the live background migration, so _start_legacy_migration hands back that same+ thread rather than starting the pass that would have found it."""+ import os+ import shutil+ import threading++ fake_home = tmp_path / "userprofile"+ fake_home.mkdir()+ _shared_setup_11(fake_home, monkeypatch, tmp_path)++ legacy = fake_home / "studio_sandbox" / "__LOCALID_split"+ legacy.mkdir(parents = True)+ (legacy / "data.csv").write_text("a\n")++ tools = _shared_setup_6()+ root = tools.sandbox_root()++ real_isdir = os.path.isdir+ real_rename = os.rename+ real_move = shutil.move+ staged = threading.Event()+ release = threading.Event()+ rolled_back = threading.Event()+ finish = threading.Event()+ rename_failed = []+ split = []++ def failing_rename(source, destination, *args, **kwargs):+ if str(destination).endswith("__LOCALID_split") and not rename_failed:+ rename_failed.append(True)+ raise OSError(39, "Directory not empty")+ return real_rename(source, destination, *args, **kwargs)++ def gated_move(source, destination, *args, **kwargs):+ moved = real_move(source, destination, *args, **kwargs)+ if "__LOCALID_split" in str(destination):+ staged.set()+ release.wait(10)+ return moved++ monkeypatch.setattr(tools.os, "rename", failing_rename)+ monkeypatch.setattr(tools.shutil, "move", gated_move)++ def run_mover():+ try:+ tools._migrate_one_legacy_session(root, "__LOCALID_split")+ finally:+ rolled_back.set()+ finish.wait(10) # stay alive, so this is still the running background migration++ mover = threading.Thread(target = run_mover, daemon = True)+ mover.start()+ monkeypatch.setattr(tools, "_legacy_background", mover, raising = False)+ assert staged.wait(10), "the move never reached the staging window"++ def isdir_then_roll_back(path, *args, **kwargs):+ answer = real_isdir(path, *args, **kwargs)+ # The caller has just read the staged-away session. Let the rollback finish before it+ # asks anything else, so its two reads straddle the restore.+ if not split and str(path).endswith("__LOCALID_split") and not answer:+ split.append(True)+ release.set()+ rolled_back.wait(10)+ return answer++ monkeypatch.setattr(tools.os.path, "isdir", isdir_then_roll_back)++ workdir = Path(tools.get_sandbox_workdir("__LOCALID_split"))++ monkeypatch.setattr(tools.os.path, "isdir", real_isdir)+ finish.set()+ mover.join(10)++ assert rename_failed, "the rename was never made to fail"+ assert split, "the caller never split its reads across the rollback"+ assert (workdir / "data.csv").is_file(), f"{workdir} lost its files across the rollback"+++def test_a_stalled_migration_does_not_grow_a_lock_per_chat(tmp_path, monkeypatch):+ """The legacy root staying put is the normal shape of a migration that keeps failing, and+ every uncached session consults it. _legacy_session_locks is documented as bounded by the+ chats that had a legacy folder, so a chat with nothing there must not leave an entry, or a+ stalled migration turns the table into a per-chat cache that only a restart clears."""+ fake_home = tmp_path / "userprofile"+ fake_home.mkdir()+ _shared_setup_11(fake_home, monkeypatch, tmp_path)++ legacy = fake_home / "studio_sandbox"+ legacy.mkdir(parents = True)+ (legacy / "__LOCALID_had_one").mkdir()+ (legacy / "__LOCALID_had_one" / "data.csv").write_text("a\n")+ # Left behind so the root itself survives, the way a migration that cannot finish leaves it.+ (legacy / "_invalid").mkdir()++ tools = _shared_setup_6()+ tools._legacy_session_locks.clear()+ root = tools.sandbox_root()++ for index in range(25):+ tools._migrate_one_legacy_session(root, f"__LOCALID_never_there{index}")++ assert legacy.is_dir(), "this test needs the legacy root to still be there"+ assert (+ set(tools._legacy_session_locks) == set()+ ), f"chats with nothing at the legacy root left locks behind: {sorted(tools._legacy_session_locks)}"++ # The chat that does have one still migrates, and is still allowed its entry.+ tools._migrate_one_legacy_session(root, "__LOCALID_had_one")+ assert (Path(root) / "__LOCALID_had_one" / "data.csv").is_file(), "the real move did not happen"+ assert set(tools._legacy_session_locks) == {"__LOCALID_had_one"}++def test_every_reported_file_is_downloadable(tmp_path, monkeypatch):"""The walk and the download route must agree, or the card advertises afile that always 404s."""