Skip to content

Qwen3.8-Flash-Next on any consumer hardware: one-click install for Windows / Linux. Strata inference engine, OpenAI/Anthropic API on localhost, optional image input.

C++

Models folder: move the model files to another folder (strata models move): copied and checked file by file, stoppable, run configs follow

Niko1221a0e09803ac8b
diff --git a/strata_cli/cli.py b/strata_cli/cli.py
index 80e66518e..d36946f3e 100644
--- a/strata_cli/cli.py
+++ b/strata_cli/cli.py
@@ -159,6 +159,13 @@ def parser() -> argparse.ArgumentParser:
models_import.add_argument("--plan", action="store_true", required=True,
help="inspect only; adoption requires a qualified model workflow")
models_import.add_argument("--output", choices=("human", "json"), default="human")
+ models_move = models_sp.add_parser("move", help="move the model files (models, packs, mtp) to another folder: copied, checked, then the old copies removed")
+ models_move.add_argument("to", metavar="NEW_FOLDER")
+ models_move.add_argument("--from", dest="source", metavar="OLD_FOLDER", help="the folder the models are in now (default: the data folder in use)")
+ models_move.add_argument("--install-dir", help="absolute path to a Strata installation whose run configs should follow the files")
+ models_move.add_argument("--plan", action="store_true", help="say what would happen and change nothing")
+ models_move.add_argument("--cancel-file", metavar="PATH", help="stop (safely) when this file appears")
+ models_move.add_argument("--output", choices=("human", "json", "jsonl"), default="human")
engine = sp.add_parser("engine", help="engine tuning and component actions")
engine_sp = engine.add_subparsers(dest="engine_command", required=True)
calibrate = engine_sp.add_parser("calibrate", help="measure, review and apply engine tuning")
@@ -495,7 +502,7 @@ def parser() -> argparse.ArgumentParser:
code_sessions.add_argument("--output", choices=("human", "json"), default="human")
for one_shot in (profile_create, profile_list, profile_show, profile_validate, profile_set,
profile_source, profile_edit, profile_history, profile_rollback, doctor,
- components_verify, models_list, models_inspect, models_import, calibrate,
+ components_verify, models_list, models_inspect, models_import, models_move, calibrate,
runtime_start, runtime_status, runtime_verify, runtime_stop, runtime_cleanup, runtime_restart,
*ops_items, bench_run, bench_report, bench_compare, code_sessions, *update_parsers):
one_shot.add_argument("--json", action="store_true", help="alias for --output json")
@@ -547,6 +554,8 @@ def _models(a: argparse.Namespace) -> int:
print(_import_problem(result), file=sys.stderr)
return 5
return _success({"plan": result}, machine=a.output == "json", human=_import_words(result))
+ if a.models_command == "move":
+ return _models_move(a)
if a.models_command == "list":
if a.served:
if a.install_dir:
@@ -603,6 +612,47 @@ def _import_problem(plan: dict) -> str:
return text
+def _models_move(a: argparse.Namespace) -> int:
+ from strata_control import discovery, models, models_move as mm
+ machine = a.output in ("json", "jsonl")
+ old = Path(a.source) if a.source else discovery.data_dir(Path.home() / "Strata")[0]
+ new = Path(a.to).expanduser()
+ if a.plan:
+ result = mm.plan(old, new)
+ if machine:
+ print(json.dumps({"schema_version": 1, "ok": result["ok"], "plan": result}, ensure_ascii=False, allow_nan=False))
+ else:
+ print(result["issue"]["message"] if result["issue"] else
+ f"{len(result['models'])} model(s), {result['models_bytes'] / 1e9:.1f} GB; {result['free_bytes'] / 1e9:.1f} GB free in {new}.")
+ return 0 if result["ok"] else 5
+ try:
+ installs = [models.resolve_install(a.install_dir)[0]] if a.install_dir else [p for p, _s in models.all_installs()]
+ except Exception:
+ installs = []
+ configs = [c for i in installs for c in mm.config_files(i)]
+ cancel = Path(a.cancel_file) if a.cancel_file else None
+
+ def show(event: dict) -> None:
+ if a.output == "jsonl":
+ print(json.dumps(event, ensure_ascii=False, allow_nan=False), flush=True)
+ elif not machine:
+ print(f"\r{event['done'] / 1e9:.1f} of {event['total'] / 1e9:.1f} GB {event['file']}", end="", flush=True)
+ try:
+ summary = mm.move(old, new, on_progress=show, cancelled=(lambda: cancel.exists()) if cancel else None, configs=configs)
+ except mm.MoveError as e:
+ extra = getattr(e, "summary", {})
+ if machine:
+ print(json.dumps({"schema_version": 1, "ok": False, "error": {"code": e.code, "message": str(e), "details": e.details}, **extra}, ensure_ascii=False))
+ else:
+ print(f"\nstrata: {e} {e.details}".rstrip(), file=sys.stderr)
+ return 5
+ if machine:
+ print(json.dumps({"schema_version": 1, "ok": True, "event": "result", **summary}, ensure_ascii=False))
+ else:
+ print("\n" + ("Your models are moved." if summary["done"] else "Moving stopped. Nothing was lost."))
+ return 0 if summary["done"] else 6
+
+
def _import_words(plan: dict) -> str:
"""The plain-words result of `strata models import --plan`: what the files are, what is already set up, what changes (nothing)."""
cand = plan.get("candidate_selection") or {}
diff --git a/strata_control/models_move.py b/strata_control/models_move.py
new file mode 100644
index 000000000..36eb6fc27
--- /dev/null
+++ b/strata_control/models_move.py
@@ -0,0 +1,477 @@
+"""Move the model files (models, packs, mtp) from one data folder to another (0.2.2, Settings > Storage > Models folder).
+
+The desktop app and the Terminal both use this: ``strata models move``. Rules, in the order they matter:
+
+* Nothing is deleted until its copy is whole and checked. One file at a time: copy to ``name.part``, compare the size, read the copy
+ back and compare its SHA-256 with the original's (and with the pinned SHA-256 in the file's ``.done`` mark when it has one), rename it
+ into place, only then remove the original. The extra space needed is never more than the move itself.
+* Stopping is safe. The file in flight is discarded (its ``.part`` is removed), files already moved stay, and the run configs are
+ rewritten for every model whose files are all in the new folder. The data folder in the settings is changed by the caller only
+ when the whole move is done, so the app is never pointed at an incomplete folder.
+* The same drive is a rename (instant). Another drive is a copy.
+* A crash leaves at worst a model in both folders; ``name.part`` files are never trusted.
+"""
+
+from __future__ import annotations
+
+import errno
+import hashlib
+import json
+import os
+import re
+import shutil
+import tempfile
+import time
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Callable
+
+ITEMS = ("models", "packs", "mtp")
+STATE_FILE = "models-move.json"
+MARGIN = 1.05 # room beside the files themselves: 5 %
+CHUNK = 8 * 1024 * 1024
+
+
+class MoveError(Exception):
+ """A move that stopped for a reason the person can act on. ``code`` is one of: not_found, not_writable, inside, disk_full, verify, busy, io."""
+
+ def __init__(self, code: str, message: str, details: str = ""):
+ super().__init__(message)
+ self.code = code
+ self.details = details
+
+
+class Stopped(Exception):
+ """The person asked to stop."""
+
+
+@dataclass
+class Entry:
+ rel: str # path under the data folder, with "/" (models/<tag>/file.gguf)
+ size: int
+ group: str # the model it belongs to: "models/<tag>", "packs/<tag>", "mtp"
+
+
+# ----------------------------------------------------------------------------------------------- what is there
+
+def _item_dirs(data: Path) -> list[Path]:
+ """The models / packs / mtp folders of a data folder (any spelling of the name: ``Models`` and ``models`` are one library)."""
+ try:
+ names = sorted(p for p in data.iterdir() if p.is_dir() and p.name.lower() in ITEMS)
+ except OSError:
+ return []
+ return names
+
+
+def inventory(data: Path) -> list[Entry]:
+ """Every file Strata keeps in a data folder: nothing else is touched. Unfinished downloads (``.part``) are left where they are."""
+ found: list[Entry] = []
+ for top in _item_dirs(data):
+ for root, _dirs, files in os.walk(top):
+ for name in files:
+ if name.endswith(".part"):
+ continue
+ path = Path(root) / name
+ try:
+ size = path.stat().st_size
+ except OSError:
+ continue
+ rel = path.relative_to(data).as_posix()
+ parts = rel.split("/")
+ kind = parts[0].lower()
+ group = kind if kind == "mtp" or len(parts) < 3 else parts[1].lower() # a model's GGUF files and its pack are one model
+ found.append(Entry(rel, size, group))
+ found.sort(key=lambda e: (e.group, e.rel))
+ return found
+
+
+def groups(entries: list[Entry]) -> list[dict]:
+ """The models in a list of files, for the person: name (the folder name), bytes."""
+ out: dict[str, int] = {}
+ for e in entries:
+ out[e.group] = out.get(e.group, 0) + e.size
+ return [{"group": g, "name": g, "bytes": b} for g, b in out.items()]
+
+
+def _near(path: Path) -> Path:
+ """The folder that exists and holds ``path`` (or ``path`` itself)."""
+ p = path
+ while not p.exists() and p.parent != p:
+ p = p.parent
+ return p
+
+
+def same_drive(a: Path, b: Path) -> bool:
+ try:
+ return os.stat(_near(a)).st_dev == os.stat(_near(b)).st_dev
+ except OSError:
+ return False
+
+
+def _same(a: Path, b: Path) -> bool:
+ return os.path.normcase(os.path.abspath(a)) == os.path.normcase(os.path.abspath(b))
+
+
+def _inside(child: Path, parent: Path) -> bool:
+ c, p = os.path.normcase(os.path.abspath(child)), os.path.normcase(os.path.abspath(parent))
+ return c != p and c.startswith(p.rstrip(os.sep) + os.sep)
+
+
+def _write_test(folder: Path) -> str | None:
+ """None when files can be written in ``folder`` (made if missing); else the raw refusal."""
+ try:
+ folder.mkdir(parents=True, exist_ok=True)
+ fd, name = tempfile.mkstemp(prefix=".strata-write-test", dir=folder)
+ os.close(fd)
+ os.remove(name)
+ return None
+ except OSError as e:
+ return f"{errno.errorcode.get(e.errno, e.errno)}: {e.strerror or e}, {e.filename or folder}"
+
+
+# ----------------------------------------------------------------------------------------------- plan
+
+def plan(old: Path, new: Path, *, free=None, write_test: bool = True) -> dict:
+ """What picking ``new`` for the models folder means, without changing anything.
+
+ Returns ``{ok, issue, old, new, same, same_drive, free_bytes, models_bytes, needed_bytes, models, choices}``. ``issue`` is None or
+ ``{code, message, details}``: not_found (the folder or its drive is gone), not_writable, inside (one folder inside the other),
+ same (nothing to do). ``choices`` says which of move / keep / empty can be done and why not.
+ """
+ old, new = Path(old), Path(new)
+ free = free or (lambda p: shutil.disk_usage(p).free)
+ entries = inventory(old) if old.is_dir() else []
+ total = sum(e.size for e in entries)
+ result = {"ok": False, "issue": None, "old": str(old), "new": str(new), "same": False, "same_drive": False, "free_bytes": 0,
+ "models_bytes": total, "needed_bytes": 0, "models": groups(entries), "choices": {}}
+
+ def refuse(code, message, details=""):
+ result["issue"] = {"code": code, "message": message, "details": details}
+ result["choices"] = {k: {"possible": False, "reason": message} for k in ("move", "keep", "empty")}
+ return result
+
+ if _same(old, new):
+ result["same"] = True
+ return refuse("same", "That is the folder Strata already uses.")
+ if _inside(new, old) or _inside(old, new):
+ return refuse("inside", "Pick a folder outside the current one.")
+ if not new.exists():
+ # a folder that will be made is fine when its parent is there; a drive that is not plugged in is not
+ drive = Path(new.anchor) if new.anchor else None
+ if drive is not None and not drive.exists():
+ return refuse("not_found", f"Strata can't find {new}.", f"{drive} does not exist")
+ if write_test:
+ raw = _write_test(new)
+ if raw:
+ return refuse("not_writable", f"Strata can't save files in {new}.", raw)
+ result["same_drive"] = same_drive(old, new)
+ try:
+ result["free_bytes"] = int(free(_near(new)))
+ except OSError:
+ result["free_bytes"] = 0
+ # a same-drive move is a rename: it needs no room
+ needed = 0 if result["same_drive"] else int(total * MARGIN)
+ result["needed_bytes"] = needed
+ can_move = bool(entries) and result["free_bytes"] >= needed
+ reason_move = "" if can_move else ("You have no models to move." if not entries
+ else f"Your models need {_gb(needed)}, this drive has {_gb(result['free_bytes'])} free.")
+ result["choices"] = {"move": {"possible": can_move, "reason": reason_move}, "keep": {"possible": True, "reason": ""},
+ "empty": {"possible": True, "reason": ""}}
+ result["ok"] = True
+ return result
+
+
+def _gb(n: int) -> str:
+ g = n / 1e9
+ return f"{g:.0f} GB" if g >= 10 else f"{g:.1f} GB"
+
+
+# ----------------------------------------------------------------------------------------------- the move
+
+def _sha256_of(path: Path, stop: Callable[[], bool], tick: Callable[[int], None] | None = None) -> str:
+ h = hashlib.sha256()
+ with open(path, "rb") as f:
+ while True:
+ if stop():
+ raise Stopped()
+ block = f.read(CHUNK)
+ if not block:
+ break
+ h.update(block)
+ if tick:
+ tick(len(block))
+ return h.hexdigest()
+
+
+def _pinned(src: Path) -> str | None:
+ """The SHA-256 a finished download wrote in its ``.done`` mark (setup.py: ``sha256 <hex>``), when there is one."""
+ mark = src.with_name(src.name + ".done")
+ try:
+ m = re.search(r"sha256\s+([0-9a-fA-F]{64})", mark.read_text(encoding="utf-8", errors="replace"))
+ except OSError:
+ return None
+ return m.group(1).lower() if m else None
+
+
+def _disk_full(e: OSError) -> bool:
+ return e.errno in (errno.ENOSPC, errno.EDQUOT) or getattr(e, "winerror", None) in (112, 39)
+
+
+def _copy_verified(src: Path, dst: Path, size: int, stop: Callable[[], bool], tick: Callable[[int], None]) -> None:
+ """Copy src to dst (through dst.part), prove the copy is the same file, rename it into place. The source is left alone."""
+ part = dst.with_name(dst.name + ".part")
+ dst.parent.mkdir(parents=True, exist_ok=True)
+ h = hashlib.sha256()
+ try:
+ with open(src, "rb") as fin, open(part, "wb") as fout:
+ while True:
+ if stop():
+ raise Stopped()
+ block = fin.read(CHUNK)
+ if not block:
+ break
+ h.update(block)
+ fout.write(block)
+ tick(len(block))
+ fout.flush()
+ os.fsync(fout.fileno())
+ if part.stat().st_size != size:
+ raise MoveError("verify", f"The copy of {src.name} has the wrong size.", f"{part.stat().st_size} != {size}")
+ want = h.hexdigest()
+ pinned = _pinned(src)
+ if pinned and pinned != want:
+ raise MoveError("verify", f"{src.name} does not match its saved checksum, so it was not moved.", f"sha256 {want} != pinned {pinned}")
+ got = _sha256_of(part, stop)
+ if got != want:
+ raise MoveError("verify", f"The copy of {src.name} did not match the original.", f"sha256 {got} != {want}")
+ os.replace(part, dst)
+ except OSError as e:
+ _quiet_remove(part)
+ if _disk_full(e):
+ raise MoveError("disk_full", "The drive ran out of space.", f"{errno.errorcode.get(e.errno, e.errno)}: {e.strerror}, {e.filename or dst}") from e
+ raise MoveError("io", f"Strata could not copy {src.name}.", f"{type(e).__name__}: {e}") from e
+ except BaseException:
+ _quiet_remove(part)
+ raise
+
+
+def _quiet_remove(path: Path) -> None:
+ try:
+ os.remove(path)
+ except OSError:
+ pass
+
+
+def _read_state(new: Path) -> dict:
+ try:
+ return json.loads((new / STATE_FILE).read_text(encoding="utf-8"))
+ except (OSError, ValueError):
+ return {}
+
+
+def interrupted(data: Path) -> dict | None:
+ """The unfinished move recorded in a data folder (``models-move.json``), for 'A move was interrupted'."""
+ st = _read_state(data)
+ return st if st.get("state") == "running" else None
+
+
+def _write_state(new: Path, **fields) -> None:
+ try:
+ tmp = new / (STATE_FILE + ".tmp")
+ tmp.write_text(json.dumps(fields, indent=1), encoding="utf-8")
+ os.replace(tmp, new / STATE_FILE)
+ except OSError:
+ pass
+
+
+def repoint_config(cfg_file: Path, old: Path, new: Path) -> bool:
+ """A run config whose model files moved from ``old`` to ``new`` names them there (each path only when its file is now there and no longer
+ at the old place). Same rule as setup.py repoint_config. Returns whether the file changed."""
+ try:
+ cfg = json.loads(cfg_file.read_text(encoding="utf-8-sig"))
+ except (OSError, ValueError):
+ return False
+
+ def fix(v):
+ if isinstance(v, list):
+ return [fix(x) for x in v]
+ if isinstance(v, dict):
+ return {k: fix(x) for k, x in v.items()}
+ if isinstance(v, str):
+ o = str(old)
+ nv, no = os.path.normcase(v), os.path.normcase(o)
+ if nv.startswith(no + os.sep) or nv.startswith(no + "/"):
+ rest = v[len(o):]
+ if re.split(r"[\\/]", rest.lstrip("\\/"))[0].lower() in ITEMS:
+ n = str(new) + rest
+ if Path(n).exists() and not Path(v).exists():
+ return n
+ return v
+
+ fixed = fix(cfg)
+ if fixed == cfg:
+ return False
+ tmp = cfg_file.with_name(cfg_file.name + ".move-tmp")
+ tmp.write_text(json.dumps(fixed, indent=2) + "\n", encoding="utf-8")
+ os.replace(tmp, cfg_file)
+ return True
+
+
+def config_files(install: Path | None) -> list[Path]:
+ """The ``strata-<model>.json`` run configs of an install folder."""
+ if not install or not Path(install).is_dir():
+ return []
+ return [p for p in Path(install).glob("strata-*.json") if not p.name.endswith(".shared-settings.json")]
+
+
+def _prune(folder: Path, stop_at: Path) -> None:
+ """Remove empty folders from ``folder`` upward, never ``stop_at`` itself."""
+ p = folder
+ while p != stop_at and _inside(p, stop_at):
+ try:
+ p.rmdir()
+ except OSError:
+ return
+ p = p.parent
+
+
+def move(old: Path, new: Path, *, on_progress: Callable[[dict], None] | None = None, cancelled: Callable[[], bool] | None = None,
+ configs: list[Path] | None = None, now: Callable[[], float] = time.time) -> dict:
+ """Move the model files from ``old`` to ``new``. Returns the summary; raises MoveError (nothing lost) or Stopped.
+
+ ``summary = {done: bool, stopped: bool, moved: [group], remaining: [group], bytes_moved, total_bytes, files_moved, same_drive}``.
+ """
+ old, new = Path(old), Path(new)
+ emit = on_progress or (lambda _e: None)
+ stop = cancelled or (lambda: False)
+ p = plan(old, new, write_test=True)
+ if p["issue"]:
+ raise MoveError(p["issue"]["code"], p["issue"]["message"], p["issue"]["details"])
+ entries = inventory(old)
+ total = sum(e.size for e in entries)
+ if not p["choices"]["move"]["possible"] and entries:
+ raise MoveError("disk_full", p["choices"]["move"]["reason"])
+ # unfinished copies of an earlier run are never trusted
+ for top in _item_dirs(new):
+ for root, _d, files in os.walk(top):
+ for name in files:
+ if name.endswith(".part"):
+ _quiet_remove(Path(root) / name)
+ same = p["same_drive"]
+ todo = {e.group: [] for e in entries}
+ for e in entries:
+ todo[e.group].append(e)
+ left = {g: len(v) for g, v in todo.items()}
+ moved_groups: list[str] = []
+ state = {"state": "running", "from": str(old), "to": str(new), "started": now(), "bytes": total}
+ _write_state(new, **state)
+ done_bytes = 0
+ files_moved = 0
+ last_emit = [0.0]
+ started = now()
+ seen_inodes: dict[tuple, Path] = {}
+ base_event = lambda **kw: {"event": "progress", "total": total, "same_drive": same, **kw}
+
+ def progress(current: str, group: str, index: int, force: bool = False):
+ t = now()
+ if not force and t - last_emit[0] < 0.25:
+ return
+ last_emit[0] = t
+ rate = done_bytes / max(t - started, 0.001)
+ emit(base_event(done=done_bytes, file=current, group=group, index=index, count=len(entries),
+ seconds_left=(int((total - done_bytes) / rate) if rate > 0 and not same else 0)))
+
+ def repoint(only_whole: bool = True):
+ for cfg in (configs or []):
+ try:
+ repoint_config(cfg, old, new)
+ except OSError:
+ pass
+
+ try:
+ for index, e in enumerate(entries, 1):
+ if stop():
+ raise Stopped()
+ src, dst = old / e.rel, new / e.rel
+ progress(e.rel, e.group, index, force=True)
+ if not src.exists():
+ done_bytes += e.size
+ left[e.group] -= 1
+ else:
+ if same:
+ dst.parent.mkdir(parents=True, exist_ok=True)
+ try:
+ if dst.exists():
+ os.remove(dst)
+ os.replace(src, dst)
+ except OSError as err:
+ raise MoveError("io", f"Strata could not move {src.name}.", f"{type(err).__name__}: {err}") from err
+ done_bytes += e.size
+ else:
+ ino = None
+ try:
+ st = src.stat()
+ ino = (st.st_dev, st.st_ino) if st.st_nlink > 1 and st.st_ino else None
+ except OSError:
+ pass
+ if ino and ino in seen_inodes and seen_inodes[ino].exists():
+ # the same file under two names (a shared pack): one copy, the second name links to it
+ try:
+ dst.parent.mkdir(parents=True, exist_ok=True)
+ if dst.exists():
+ os.remove(dst)
+ os.link(seen_inodes[ino], dst)
+ except OSError:
+ _copy_verified(src, dst, e.size, stop, lambda n: None)
+ done_bytes += e.size
+ else:
+ resumed = False
+ if dst.exists() and dst.stat().st_size == e.size:
+ # a crash between the rename and the delete: the copy may be whole; prove it before the original goes
+ resumed = _sha256_of(dst, stop) == _sha256_of(src, stop)
+ if resumed:
+ done_bytes += e.size
+ else:
+ def tick(n, group=e.group, rel=e.rel, idx=index):
+ nonlocal done_bytes
+ done_bytes += n
+ progress(rel, group, idx)
+ before = done_bytes
+ try:
+ _copy_verified(src, dst, e.size, stop, tick)
+ except BaseException:
+ done_bytes = before
+ raise
+ if ino:
+ seen_inodes[ino] = dst
+ try:
+ os.remove(src)
+ except OSError as err:
+ raise MoveError("io", f"Strata copied {src.name} but could not remove the old copy.", f"{type(err).__name__}: {err}") from err
+ _prune(src.parent, old)
+ files_moved += 1
+ left[e.group] -= 1
+ if left[e.group] == 0:
+ moved_groups.append(e.group)
+ repoint()
+ repoint()
+ state.update(state="done", finished=now())
+ _write_state(new, **state)
+ _quiet_remove(new / STATE_FILE)
+ return {"done": True, "stopped": False, "moved": moved_groups, "remaining": [], "bytes_moved": done_bytes, "total_bytes": total,
+ "files_moved": files_moved, "same_drive": same}
+ except Stopped:
+ repoint()
+ state.update(state="stopped")
+ _write_state(new, **state)
+ _quiet_remove(new / STATE_FILE) if not moved_groups and not files_moved else None
+ remaining = [g for g, n in left.items() if n > 0]
+ return {"done": False, "stopped": True, "moved": moved_groups, "remaining": remaining, "bytes_moved": done_bytes, "total_bytes": total,
+ "files_moved": files_moved, "same_drive": same}
+ except MoveError as err:
+ repoint()
+ state.update(state="failed", error=err.code)
+ _write_state(new, **state)
+ err.summary = {"moved": moved_groups, "remaining": [g for g, n in left.items() if n > 0], "bytes_moved": done_bytes, "total_bytes": total,
+ "files_moved": files_moved}
+ raise
diff --git a/tests/terminal/test_models_move.py b/tests/terminal/test_models_move.py
new file mode 100644
index 000000000..80a1408b0
--- /dev/null
+++ b/tests/terminal/test_models_move.py
@@ -0,0 +1,283 @@
+"""Moving the models to another folder (strata models move): same drive, another drive, stop, a failed check, disk full, resume."""
+from __future__ import annotations
+
+import hashlib
+import json
+import os
+import shutil
+import tempfile
+import unittest
+from pathlib import Path
+from unittest import mock
+
+from strata_control import models_move as mm
+
+
+def sha(data: bytes) -> str:
+ return hashlib.sha256(data).hexdigest()
+
+
+class MoveCase(unittest.TestCase):
+ def setUp(self):
+ self.tmp = tempfile.TemporaryDirectory(prefix="strata-move-")
+ self.addCleanup(self.tmp.cleanup)
+ self.base = Path(self.tmp.name)
+ self.old = self.base / "old"
+ self.new = self.base / "new"
+ self.files = {
+ "models/gyro-s/a.gguf": b"A" * 300_000,
+ "models/gyro-s/b.gguf": b"B" * 200_000,
+ "packs/gyro-s/dense.bin": b"D" * 100_000,
+ "models/coder/c.gguf": b"C" * 150_000,
+ "mtp/rt/experts.bin": b"M" * 50_000,
+ }
+ for rel, data in self.files.items():
+ f = self.old / rel
+ f.parent.mkdir(parents=True, exist_ok=True)
+ f.write_bytes(data)
+ # a finished download carries the pinned checksum in its mark
+ (self.old / "models/gyro-s/a.gguf.done").write_text(f"done sha256 {sha(self.files['models/gyro-s/a.gguf'])}\n")
+ (self.old / "notes.txt").write_text("not a model file") # Strata moves its own folders only
+ self.install = self.base / "install"
+ self.install.mkdir()
+ self.config = self.install / "strata-gyro-s.json"
+ self.config.write_text(json.dumps({"args": ["--native", str(self.old / "models/gyro-s/a.gguf"), "--pack", str(self.old / "packs/gyro-s")],
+ "mtp": str(self.old / "mtp/rt")}))
+ self.total = sum(len(v) for v in self.files.values())
+
+ def tree(self, root: Path) -> set[str]:
+ return {p.relative_to(root).as_posix() for p in root.rglob("*") if p.is_file() and p.name != mm.STATE_FILE}
+
+ def copy_mode(self):
+ """Another drive: the files are copied, whatever this machine's volumes are."""
+ return mock.patch.object(mm, "same_drive", return_value=False)
+
+ def run_move(self, **kw):
+ events = []
+ summary = mm.move(self.old, self.new, on_progress=events.append, configs=[self.config], **kw)
+ return summary, events
+
+
+class PlanTests(MoveCase):
+ def test_plan_numbers_and_choices(self):
+ with self.copy_mode():
+ p = mm.plan(self.old, self.new, free=lambda _p: 10**9)
+ self.assertTrue(p["ok"])
+ self.assertGreaterEqual(p["models_bytes"], self.total)
+ self.assertGreater(p["needed_bytes"], p["models_bytes"]) # the 5 % margin
+ self.assertTrue(p["choices"]["move"]["possible"])
+ self.assertEqual({m["name"] for m in p["models"]}, {"gyro-s", "coder", "mtp"})
+
+ def test_not_enough_room_switches_move_off(self):
+ with self.copy_mode():
+ p = mm.plan(self.old, self.new, free=lambda _p: 1000)
+ self.assertTrue(p["ok"])
+ self.assertFalse(p["choices"]["move"]["possible"])
+ self.assertIn("free", p["choices"]["move"]["reason"])
+ self.assertTrue(p["choices"]["keep"]["possible"] and p["choices"]["empty"]["possible"])
+
+ def test_same_drive_needs_no_room(self):
+ p = mm.plan(self.old, self.new, free=lambda _p: 0)
+ self.assertTrue(p["same_drive"])
+ self.assertTrue(p["choices"]["move"]["possible"])
+
+ def test_inside_and_same_are_refused(self):
+ self.assertEqual(mm.plan(self.old, self.old / "sub")["issue"]["code"], "inside")
+ self.assertEqual(mm.plan(self.old / "sub", self.old)["issue"]["code"], "inside")
+ self.assertEqual(mm.plan(self.old, self.old)["issue"]["code"], "same")
+
+ def test_a_folder_that_cannot_be_written_is_refused(self):
+ with mock.patch.object(mm, "_write_test", return_value="EPERM: operation not permitted, mkdir 'x'"):
+ p = mm.plan(self.old, self.new)
+ self.assertEqual(p["issue"]["code"], "not_writable")
+ self.assertIn("EPERM", p["issue"]["details"])
+
+ def test_nothing_is_changed_by_a_plan(self):
+ before = self.tree(self.old)
+ mm.plan(self.old, self.new)
+ self.assertEqual(self.tree(self.old), before)
+
+
+class MoveTests(MoveCase):
+ def check_whole(self):
+ self.assertEqual({k for k in self.tree(self.new) if not k.endswith(".done")}, set(self.files))
+ for rel, data in self.files.items():
+ self.assertEqual((self.new / rel).read_bytes(), data)
+ self.assertEqual(self.tree(self.old), {"notes.txt"}) # only what is not Strata's own is left
+ cfg = json.loads(self.config.read_text())
+ self.assertEqual(Path(cfg["args"][1]), self.new / "models/gyro-s/a.gguf")
+ self.assertEqual(Path(cfg["mtp"]), self.new / "mtp/rt")
+ self.assertFalse((self.new / mm.STATE_FILE).exists())
+
+ def test_same_drive_is_a_rename(self):
+ with mock.patch.object(shutil, "copyfileobj", side_effect=AssertionError("copied")), mock.patch.object(mm, "_copy_verified", side_effect=AssertionError("copied")):
+ summary, _events = self.run_move()
+ self.assertTrue(summary["done"] and summary["same_drive"])
+ self.check_whole()
+
+ def test_another_drive_copies_checks_and_removes(self):
+ with self.copy_mode():
+ summary, events = self.run_move()
+ self.assertTrue(summary["done"] and not summary["same_drive"])
+ self.assertEqual(summary["files_moved"], len(self.files) + 1) # + the .done mark
+ self.check_whole()
+ self.assertTrue(events and events[-1]["done"] <= self.total + 100)
+ self.assertEqual(events[0]["event"], "progress")
+
+ def test_stop_keeps_what_moved_and_removes_the_partial_copy(self):
+ calls = {"n": 0}
+
+ def stop():
+ calls["n"] += 1
+ return calls["n"] > 6 # a few chunks into the second file or so
+ with self.copy_mode():
+ summary, _events = self.run_move(cancelled=stop)
+ self.assertTrue(summary["stopped"] and not summary["done"])
+ self.assertEqual([p for p in self.new.rglob("*.part")], [])
+ # every file is in exactly one place, nothing is lost
+ for rel, data in self.files.items():
+ places = [r for r in (self.old, self.new) if (r / rel).exists()]
+ self.assertEqual(len(places), 1, rel)
+ self.assertEqual((places[0] / rel).read_bytes(), data)
+ self.assertTrue(summary["remaining"])
+ # a model that moved whole now points at the new folder; one that did not stays where it is
+ cfg = json.loads(self.config.read_text())
+ for value in (cfg["args"][1], cfg["args"][3], cfg["mtp"]):
+ self.assertTrue(Path(value).exists(), value)
+
+ def test_stop_before_anything_leaves_the_old_folder_as_it_was(self):
+ with self.copy_mode():
+ summary, _ = self.run_move(cancelled=lambda: True)
+ self.assertTrue(summary["stopped"])
+ self.assertEqual(summary["files_moved"], 0)
+ self.assertEqual({k for k in self.tree(self.old) if not k.endswith(".done")}, set(self.files) | {"notes.txt"})
+ self.assertFalse((self.new / mm.STATE_FILE).exists())
+
+ def test_a_copy_that_does_not_match_is_never_trusted(self):
+ real = mm._sha256_of
+ calls = {"n": 0}
+
+ def lies(path, stop, tick=None):
+ calls["n"] += 1
+ return "0" * 64 if str(path).endswith(".part") and calls["n"] == 1 else real(path, stop, tick)
+ with self.copy_mode(), mock.patch.object(mm, "_sha256_of", side_effect=lies):
+ with self.assertRaises(mm.MoveError) as caught:
+ self.run_move()
+ self.assertEqual(caught.exception.code, "verify")
+ self.assertEqual(list(self.new.rglob("*.part")), [])
+ self.assertEqual({k for k in self.tree(self.old) if not k.endswith(".done")}, set(self.files) | {"notes.txt"}) # the original is untouched
+
+ def test_a_file_that_differs_from_its_pinned_checksum_is_not_moved(self):
+ (self.old / "models/gyro-s/a.gguf.done").write_text("done sha256 " + "1" * 64)
+ with self.copy_mode():
+ with self.assertRaises(mm.MoveError) as caught:
+ self.run_move()
+ self.assertEqual(caught.exception.code, "verify")
+ self.assertTrue((self.old / "models/gyro-s/a.gguf").exists())
+ self.assertFalse((self.new / "models/gyro-s/a.gguf").exists())
+
+ def test_disk_full_stops_at_once_and_loses_nothing(self):
+ real = mm._copy_verified
+ seen = {"n": 0}
+
+ def full(src, dst, size, stop, tick):
+ seen["n"] += 1
+ if seen["n"] == 3:
+ part = dst.with_name(dst.name + ".part")
+ part.parent.mkdir(parents=True, exist_ok=True)
+ part.write_bytes(b"half")
+ try:
+ raise OSError(28, "No space left on device", str(dst))
+ except OSError as e:
+ mm._quiet_remove(part)
+ raise mm.MoveError("disk_full", "The drive ran out of space.", str(e)) from e
+ return real(src, dst, size, stop, tick)
+ with self.copy_mode(), mock.patch.object(mm, "_copy_verified", side_effect=full):
+ with self.assertRaises(mm.MoveError) as caught:
+ self.run_move()
+ self.assertEqual(caught.exception.code, "disk_full")
+ self.assertEqual(list(self.new.rglob("*.part")), [])
+ for rel, data in self.files.items():
+ places = [r for r in (self.old, self.new) if (r / rel).exists()]
+ self.assertEqual(len(places), 1, rel)
+ self.assertIn("remaining", caught.exception.summary)
+
+ def test_a_real_disk_full_error_is_named(self):
+ with mock.patch("builtins.open", side_effect=OSError(28, "No space left on device", "x")):
+ with self.assertRaises(mm.MoveError) as caught:
+ mm._copy_verified(self.old / "models/coder/c.gguf", self.new / "x.gguf", 1, lambda: False, lambda n: None)
+ self.assertEqual(caught.exception.code, "disk_full")
+
+ def test_a_crash_leaves_a_part_and_the_next_run_finishes(self):
+ stale = self.new / "models/gyro-s/a.gguf.part"
+ stale.parent.mkdir(parents=True)
+ stale.write_bytes(b"cut off")
+ # a file that was copied and renamed, but whose original was not removed yet
+ done = self.new / "models/coder/c.gguf"
+ done.parent.mkdir(parents=True)
+ done.write_bytes(self.files["models/coder/c.gguf"])
+ with self.copy_mode():
+ summary, _ = self.run_move()
+ self.assertTrue(summary["done"])
+ self.check_whole()
+
+ def test_an_interrupted_move_is_remembered(self):
+ with self.copy_mode():
+ with self.assertRaises(mm.MoveError):
+ with mock.patch.object(mm, "_copy_verified", side_effect=mm.MoveError("io", "x")):
+ self.run_move()
+ self.assertEqual(mm.interrupted(self.new), None) # failed is not "running"
+ mm._write_state(self.new, state="running", **{"from": str(self.old), "to": str(self.new)})
+ self.assertEqual(mm.interrupted(self.new)["from"], str(self.old))
+
+ def test_configs_are_rewritten_only_for_files_that_are_there(self):
+ self.config.write_text(json.dumps({"args": ["--native", str(self.old / "models/gone.gguf")]}))
+ with self.copy_mode():
+ self.run_move()
+ self.assertEqual(json.loads(self.config.read_text())["args"][1], str(self.old / "models/gone.gguf"))
+
+ @unittest.skipUnless(os.environ.get("STRATA_TEST_OTHER_VOLUME"), "set STRATA_TEST_OTHER_VOLUME to a folder on another volume (e.g. a subst or USB drive)")
+ def test_a_real_second_volume(self):
+ other = Path(os.environ["STRATA_TEST_OTHER_VOLUME"]) / f"strata-move-{os.getpid()}"
+ self.addCleanup(shutil.rmtree, other, True)
+ self.new = other
+ summary, _ = self.run_move()
+ self.assertTrue(summary["done"])
+ self.check_whole()
+
+
+class CommandLineTests(MoveCase):
+ def test_json_plan_and_run(self):
+ from strata_cli import cli
+ import contextlib
+ import io
+ out = io.StringIO()
+ with contextlib.redirect_stdout(out), self.copy_mode():
+ code = cli.main(["models", "move", str(self.new), "--from", str(self.old), "--plan", "--output", "json"])
+ self.assertEqual(code, 0)
+ self.assertTrue(json.loads(out.getvalue())["plan"]["choices"]["move"]["possible"])
+ out = io.StringIO()
+ with contextlib.redirect_stdout(out), self.copy_mode():
+ code = cli.main(["models", "move", str(self.new), "--from", str(self.old), "--install-dir", str(self.install), "--output", "jsonl"])
+ self.assertEqual(code, 0, out.getvalue())
+ lines = [json.loads(line) for line in out.getvalue().splitlines() if line.startswith("{")]
+ self.assertEqual(lines[-1]["event"], "result")
+ self.assertTrue(lines[-1]["done"])
+ self.assertTrue((self.new / "models/coder/c.gguf").exists())
+
+ def test_the_cancel_file_stops_the_run(self):
+ from strata_cli import cli
+ import contextlib
+ import io
+ cancel = self.base / "cancel"
+ cancel.write_text("x")
+ out = io.StringIO()
+ with contextlib.redirect_stdout(out), self.copy_mode():
+ code = cli.main(["models", "move", str(self.new), "--from", str(self.old), "--cancel-file", str(cancel), "--output", "json"])
+ self.assertEqual(code, 6)
+ self.assertTrue(json.loads(out.getvalue().splitlines()[-1])["stopped"])
+ self.assertTrue((self.old / "models/coder/c.gguf").exists())
+
+
+if __name__ == "__main__":
+ unittest.main()