Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: rewind files more than four folders deep (#15439)
Nilay771b3bc371ec
diff --git a/studio/backend/core/inference/file_history.py b/studio/backend/core/inference/file_history.pyindex 51f31dbb173..869481a0209 100644--- a/studio/backend/core/inference/file_history.py+++ b/studio/backend/core/inference/file_history.py@@ -140,6 +140,23 @@ def _key(snap_value) -> tuple:return (snap_value[0], snap_value[1], snap_value[2] if len(snap_value) > 2 else None)+def _probe(workdir: str, rels: "Iterable[str]") -> dict:+ """Regular files among *rels*, keyed as a walk keys them; one behind a linked folder counts as gone."""+ root = os.path.realpath(workdir)+ found = {}+ for rel in rels:+ path = os.path.join(root, *rel.split("/"))+ try:+ if os.path.realpath(os.path.dirname(path)) != os.path.dirname(path):+ continue+ info = os.lstat(path)+ except OSError:+ continue+ if stat_mod.S_ISREG(info.st_mode):+ found[rel] = (info.st_mtime_ns, info.st_size, None)+ return found++class FileHistory:"""One per account. Every method is blocking: call it from a worker thread."""@@ -288,10 +305,12 @@ class FileHistory:continuenew = self._capture(workdir, rel, key, budget)if old is not None and old.kind == new.kind == "blob" and old.blob == new.blob:- old.mtime_ns, old.digest, old.mode = new.mtime_ns, new.digest, new.mode+ # Same content, so a known digest still holds; a probe that hashed nothing must not drop it.+ old.mtime_ns, old.mode = new.mtime_ns, new.mode+ old.digest = old.digest if new.digest is None else new.digestself._db.execute("UPDATE workdir_state SET mtime_ns=?, digest=?, mode=? WHERE workdir=? AND rel_path=?",- (new.mtime_ns, new.digest, new.mode, workdir, rel),+ (old.mtime_ns, old.digest, old.mode, workdir, rel),)continuechanges[rel] = (old, new)@@ -326,7 +345,13 @@ class FileHistory:def _walk_sync(self, workdir: str, walk) -> dict:snap, complete = walk(workdir)- return self._sync(workdir, snap, None, complete)+ changes = self._sync(workdir, snap, None, complete)+ if not complete:+ # A capped walk never reached these: read them directly, or a rewind trusts a stale state+ # and overwrites the user's later edit.+ missed = {rel for rel in self._load_state(workdir) if rel not in snap}+ changes.update(self._sync(workdir, _probe(workdir, missed), missed))+ return changesdef _record(self,diff --git a/studio/backend/core/inference/tools.py b/studio/backend/core/inference/tools.pyindex 0726dfde081..62480b3859e 100644--- a/studio/backend/core/inference/tools.py+++ b/studio/backend/core/inference/tools.py@@ -20922,6 +20922,8 @@ _MAX_REPORTED_FILES = 25_MAX_SANDBOX_PATH_SEGMENTS = 4_MAX_SNAPSHOT_FILES = 2000 # a shard-writing script must not blow up the result_MAX_SNAPSHOT_DIRS = 2000 # nor a directory-writing one stall the next call+_MAX_HISTORY_FILES = 20000+_MAX_HISTORY_DIRS = 20000def _user_path_parts(parts: "list[str]", root: "str | None" = None) -> "list[str]":@@ -21009,32 +21011,40 @@ def _snapshot_skips_dir(base: str, name: str) -> bool:return name in _SNAPSHOT_SKIP_DIRS or os.path.isfile(os.path.join(base, name, "pyvenv.cfg"))-def _snapshot_workdir_files(workdir: str | None) -> "dict[str, tuple]":+def _snapshot_workdir_files(workdir: str | None, history: bool = False) -> "dict[str, tuple]":"""Maps every regular file to a change key; digests are added only on coarse-timestamp volumes."""snapshot: "dict[str, tuple]" = {}if not workdir or not os.path.isdir(workdir):return snapshot+ max_files, max_dirs = (+ (_MAX_HISTORY_FILES, _MAX_HISTORY_DIRS)+ if history+ else (_MAX_SNAPSHOT_FILES, _MAX_SNAPSHOT_DIRS)+ )# Directories are budgeted separately: empty folders never reach the file cap.visited = 0hash_budget = 0 if _volume_timestamps_finely(workdir) else _MAX_SNAPSHOT_HASH_BYTES# walked to include nested outputs such as outputs/report.csv that a top-level listing drops.for base, dirs, names in os.walk(workdir):visited += 1- if visited > _MAX_SNAPSHOT_DIRS:+ if visited > max_dirs:return _CappedSnapshot(snapshot)# depth 0 is the workdir itself, whose files are one segment.- relative = base[len(workdir) :].strip(os.sep)- depth = len(_user_path_parts(relative.split(os.sep) if relative else []))+ folder = base[len(workdir) :].strip(os.sep)+ depth = len(_user_path_parts(folder.split(os.sep) if folder else []))+ prefix = folder.replace(os.sep, "/") + "/" if folder else ""# skip noisy dot directories except attachments; dot files like .gitignore remain valid artifacts.dirs[:] = ([]- if depth >= _MAX_SANDBOX_PATH_SEGMENTS - 1+ if not history and depth >= _MAX_SANDBOX_PATH_SEGMENTS - 1else [dfor d in dirsif (not d.startswith(".") or (base == workdir and d == _ATTACHMENTS_DIR))and _servable_segment(d)and not _snapshot_skips_dir(base, d)+ # history drops scratch anyway; a torch.compile cache there would spend the whole budget.+ and not (history and base == workdir and not file_history.tracked(d + "/", ()))])for name in names:@@ -21049,7 +21059,7 @@ def _snapshot_workdir_files(workdir: str | None) -> "dict[str, tuple]":stat = os.lstat(path)if not S_ISREG(stat.st_mode):continue- relative = os.path.relpath(path, workdir).replace(os.sep, "/")+ relative = prefix + namecontent = Noneif hash_budget and hash_budget >= stat.st_size:content = _content_key(path, stat.st_size)@@ -21058,7 +21068,7 @@ def _snapshot_workdir_files(workdir: str | None) -> "dict[str, tuple]":snapshot[relative] = (stat.st_mtime_ns, stat.st_size, content)except OSError:continue- if len(snapshot) >= _MAX_SNAPSHOT_FILES:+ if len(snapshot) >= max_files:return _CappedSnapshot(snapshot)return snapshot@@ -21098,7 +21108,7 @@ def _call_finished(token: "dict | None") -> None:returnif token.get("history") is not None and not token["history"].done:# An exit that skipped the file sentinels (a launch error): the folder may still have moved.- _history_end(token, _snapshot_workdir_files(token["workdir"]))+ _history_end(token)with _calls_lock:running = _workdir_calls.get(token["workdir"])if running is None:@@ -21111,41 +21121,40 @@ def _call_finished(token: "dict | None") -> None:_workdir_calls.pop(token["workdir"], None)-def _history_snapshot(snap: "dict[str, tuple]") -> "tuple[dict, bool]":+def _history_walk(workdir: str) -> "tuple[dict, bool]":"""The walk as file history keeps it: without scratch scripts and Studio's own trees. The flagsays whether the walk was whole, since a capped one cannot prove a file is gone."""+ snap = _snapshot_workdir_files(workdir, history = True)with _scratch_lock:scratch = set(_active_scratch)kept = {rel: key for rel, key in snap.items() if file_history.tracked(rel, scratch)}return kept, not isinstance(snap, _CappedSnapshot)-def _history_begin(call_token: dict, session_id: "str | None", before: "dict[str, tuple]") -> None:+def _history_begin(call_token: dict, session_id: "str | None") -> None:if file_history.current_turn() is None:return- snap, complete = _history_snapshot(before)+ snap, complete = _history_walk(call_token["workdir"])call_token["history"] = file_history.begin_call(session_id, call_token.get("workdir"), snap, complete = complete)-def _history_end(call_token: "dict | None", after: "dict[str, tuple]") -> None:+def _history_end(call_token: "dict | None") -> None:if not call_token or call_token.get("history") is None:return+ snap, complete = _history_walk(call_token["workdir"])+ # Checked after the walk: an upload landing mid-walk is in `snap` and must not read as ours.if call_token.get("shared"):- # An upload or another call wrote here meanwhile: what changed is not all this turn's.call_token["history"].ambiguous = True- snap, complete = _history_snapshot(after)file_history.end_call(call_token["history"], snap, complete = complete)def _walk_lists(workdir: str, rel: str) -> bool:- """Whether `_snapshot_workdir_files` would list *rel*, so a file edit_file records is never+ """Whether `_history_walk` would list *rel*, so a file edit_file records is nevertaken for a deletion by the next python or terminal walk."""parts = rel.split("/")- if len(_user_path_parts(parts)) > _MAX_SANDBOX_PATH_SEGMENTS or not all(- map(_servable_segment, parts)- ):+ if not all(map(_servable_segment, parts)):return Falsebase = workdirfor part in parts[:-1]:@@ -21176,7 +21185,7 @@ def _history_begin_file(session_id: "str | None", target: str) -> dict:logger.warning("file history: could not open the index", exc_info = True)return {}if baseline:- snap, complete = _history_snapshot(_snapshot_workdir_files(workdir))+ snap, complete = _history_walk(workdir)# A capped walk can miss the very file being edited; without it, the edit reads as a creation.snap = {**_history_file_snapshot(workdir, rel), **snap}else:@@ -21224,7 +21233,7 @@ def _created_file_sentinels(# card beats one that names, and downloads, another chat's file.return ""after = _snapshot_workdir_files(workdir)- _history_end(token, after)+ _history_end(token)if token is not None and token.get("shared"):# Calls started mid-walk write into `after`, indistinguishable from ours; only known sharing skips.return ""@@ -21344,7 +21353,7 @@ def _python_exec(call_token = _call_started(workdir)# Snapshot mtimes to detect new and overwritten files._before = _snapshot_workdir_files(workdir)- _history_begin(call_token, session_id, _before)+ _history_begin(call_token, session_id)try:# In the workdir: sys.path[0] keeps earlier helpers importable and __file__ in the sandbox.fd, tmp_path = tempfile.mkstemp(suffix = ".py", prefix = "studio_exec_", dir = workdir)@@ -21592,7 +21601,7 @@ def _bash_exec(call_token = _call_started(workdir)# Same pre-run snapshot as _python_exec, so files a command writes are reported, not "(no output)"._before = _snapshot_workdir_files(workdir)- _history_begin(call_token, session_id, _before)+ _history_begin(call_token, session_id)safe_env = (_build_bypass_env(workdir)if disable_sandboxdiff --git a/studio/backend/routes/chat_history.py b/studio/backend/routes/chat_history.pyindex dad2d4ad38b..88f8cff3383 100644--- a/studio/backend/routes/chat_history.py+++ b/studio/backend/routes/chat_history.py@@ -1948,9 +1948,6 @@ def rewind_thread_files(status_code = 409, detail = "This chat is still generating. Rewind it once it finishes.")- def walk(workdir: str):- return tools._history_snapshot(tools._snapshot_workdir_files(workdir))-def busy(workdir: str) -> bool:with tools._calls_lock:return any(os.path.realpath(w) == workdir for w in tools._workdir_calls)@@ -1959,7 +1956,7 @@ def rewind_thread_files(report = file_history.store().rewind(thread_id,payload.turn_ids,- walk,+ tools._history_walk,dry_run = payload.dry_run,force = payload.force,busy = busy,diff --git a/studio/backend/tests/test_file_history.py b/studio/backend/tests/test_file_history.pyindex 292c63f2311..5d903eec5ae 100644--- a/studio/backend/tests/test_file_history.py+++ b/studio/backend/tests/test_file_history.py@@ -30,7 +30,7 @@ def chat(tmp_path, monkeypatch):def _walk(workdir):- return tools._history_snapshot(tools._snapshot_workdir_files(workdir))+ return tools._history_walk(workdir)def _rewind(history, turn_ids, **kwargs):@@ -221,7 +221,7 @@ def test_a_walk_cut_short_by_the_folder_cap_is_not_complete(chat, monkeypatch):(workdir / name / "f.txt").write_text(name, encoding = "utf-8")assert _walk(str(workdir))[1] is True# Fewer files than the file cap, so only the folder cap can say the walk missed some.- monkeypatch.setattr(tools, "_MAX_SNAPSHOT_DIRS", 2)+ monkeypatch.setattr(tools, "_MAX_HISTORY_DIRS", 2)assert _walk(str(workdir))[1] is False@@ -288,9 +288,7 @@ def test_a_folder_swapped_for_a_link_is_never_written_through(chat, tmp_path):(elsewhere / "a.txt").write_text("2\n", encoding = "utf-8")workdir.rename(tmp_path / "moved")workdir.symlink_to(elsewhere, target_is_directory = True)- report = history.rewind(- "chat-1", ["u1"], lambda wd: tools._history_snapshot(tools._snapshot_workdir_files(wd))- )+ report = history.rewind("chat-1", ["u1"], _walk)assert report.errors and (elsewhere / "a.txt").read_text(encoding = "utf-8") == "2\n"@@ -310,7 +308,7 @@ def test_an_edit_missed_by_a_capped_first_walk_restores_the_original(chat, monke(workdir / "a").mkdir()(workdir / "z").mkdir()(workdir / "z" / "keep.txt").write_text("original\n", encoding = "utf-8")- monkeypatch.setattr(tools, "_MAX_SNAPSHOT_DIRS", 1)+ monkeypatch.setattr(tools, "_MAX_HISTORY_DIRS", 1)file_history.set_turn("chat-1", "u1")_edit("z/keep.txt", "original", "changed")report = _rewind(history, ["u1"])@@ -389,3 +387,111 @@ def test_an_upload_during_a_call_is_never_undone_with_it(chat):report = _rewind(history, ["u1"])assert "upload.txt" in report.conflictsassert (workdir / "upload.txt").read_text(encoding = "utf-8") == "mine\n"+++def _route_rewind(monkeypatch, turn_ids, **body):+ from routes import chat_history+ from state import active_generations++ monkeypatch.setattr(chat_history, "get_chat_thread", lambda thread_id: {"id": thread_id})+ monkeypatch.setattr(active_generations, "active_thread_ids", lambda scope = None: set())+ request = chat_history.ChatRewindRequest(turn_ids = turn_ids, **body)+ return chat_history.rewind_thread_files("chat-1", request, current_subject = "user")++[email protected](os.name != "posix", reason = "uses a POSIX shell")+def test_files_deep_in_the_folder_are_rewound(chat, monkeypatch):+ workdir, _history = chat+ route = workdir / "web" / "src" / "app" / "api" / "route.ts"+ helpers = workdir / "src" / "app" / "core" / "util" / "helpers.py"+ for path in (route, helpers):+ path.parent.mkdir(parents = True)+ path.write_text("v1\n", encoding = "utf-8")+ file_history.set_turn("chat-1", "u1")+ _terminal("printf 'v2\\n' > web/src/app/api/route.ts")+ _edit("src/app/core/util/helpers.py", "v1", "v2")++ preview = _route_rewind(monkeypatch, ["u1"], dry_run = True)+ assert preview["restored"] == ["src/app/core/util/helpers.py", "web/src/app/api/route.ts"]+ assert _route_rewind(monkeypatch, ["u1"])["restored"] == preview["restored"]+ assert route.read_text(encoding = "utf-8") == helpers.read_text(encoding = "utf-8") == "v1\n"++[email protected](os.name != "posix", reason = "uses a POSIX shell")+def test_a_file_past_the_download_card_walk_is_rewound(chat, monkeypatch):+ workdir, _history = chat+ names = [f"d{d:02}/f{f:03}.txt" for d in range(25) for f in range(100)]+ for name in names:+ (workdir / name).parent.mkdir(exist_ok = True)+ (workdir / name).write_text("v1\n", encoding = "utf-8")+ listed = tools._snapshot_workdir_files(str(workdir))+ missed = next(name for name in names if name not in listed)+ file_history.set_turn("chat-1", "u1")+ _terminal(f"printf 'v2\\n' > {missed}")++ assert _route_rewind(monkeypatch, ["u1"])["restored"] == [missed]+ assert (workdir / missed).read_text(encoding = "utf-8") == "v1\n"+++def test_a_large_scratch_folder_does_not_hide_a_deletion(chat, monkeypatch):+ workdir, history = chat+ (workdir / "keep.txt").write_text("v1\n", encoding = "utf-8")+ cache = workdir / tools._SANDBOX_TEMP_DIRNAME / "torchinductor_user" / "fx" / "ab" / "cd"+ cache.mkdir(parents = True)+ for i in range(60):+ (cache / f"{i}.py").write_text("x\n", encoding = "utf-8")+ monkeypatch.setattr(tools, "_MAX_HISTORY_FILES", 50)+ assert _walk(str(workdir))[1] is True+ file_history.set_turn("chat-1", "u1")+ execute_tool("python", {"code": "import os; os.remove('keep.txt')"}, session_id = "s")++ assert _rewind(history, ["u1"]).recreated == ["keep.txt"]+ assert (workdir / "keep.txt").read_text(encoding = "utf-8") == "v1\n"+++def test_an_upload_during_the_last_walk_is_never_undone_with_it(chat, monkeypatch):+ workdir, history = chat+ file_history.set_turn("chat-1", "u1")+ _edit("seed.txt", "", "s\n") # the baseline+ real_walk, walks = tools._snapshot_workdir_files, []++ def walk_with_an_upload(path, history = False):+ walks.append(history)+ if history and walks.count(True) == 2:+ tools.save_sandbox_upload("s", "upload.txt", [b"mine\n"])+ return real_walk(path, history = history)++ monkeypatch.setattr(tools, "_snapshot_workdir_files", walk_with_an_upload)+ execute_tool("python", {"code": "open('out.txt', 'w').write('x')"}, session_id = "s")+ monkeypatch.setattr(tools, "_snapshot_workdir_files", real_walk)++ report = _rewind(history, ["u1"])+ assert "upload.txt" in report.conflicts+ assert (workdir / "upload.txt").read_text(encoding = "utf-8") == "mine\n"+++def test_a_rewind_after_a_capped_walk_keeps_a_later_user_edit(chat):+ workdir, history = chat+ (workdir / "src").mkdir()+ keep = workdir / "src" / "keep.txt"+ keep.write_text("original\n", encoding = "utf-8")+ file_history.set_turn("chat-1", "u1")+ _edit("src/keep.txt", "original", "model")+ keep.write_text("user later edit\n", encoding = "utf-8")++ def capped(wd): # a walk whose budget ran out before src/+ snap, _complete = _walk(wd)+ return {rel: key for rel, key in snap.items() if not rel.startswith("src/")}, False++ report = history.rewind("chat-1", ["u1"], capped)+ assert report.conflicts == ["src/keep.txt"]+ assert keep.read_text(encoding = "utf-8") == "user later edit\n"+++def test_a_probe_of_a_missed_file_keeps_its_content_digest(chat):+ workdir, history = chat+ (workdir / "keep.txt").write_text("same\n", encoding = "utf-8")+ info = (workdir / "keep.txt").stat()+ history._walk_sync(str(workdir), lambda wd: ({"keep.txt": (1, info.st_size, "d1")}, True))+ history._walk_sync(str(workdir), lambda wd: ({}, False)) # capped: keep.txt is probed+ assert history._load_state(os.path.realpath(str(workdir)))["keep.txt"].digest == "d1"