ODS V3 Pre-Release: Public testing and refinement ahead of the official V3 launch. Turn your PC, Mac, or Linux box into a private AI server.
Python ai-agentsamdcomfyuidockerllama-cppllmlocal-ain8nnvidiaopen-webuiragself-hostedspeech-to-textstrix-halotext-to-speechworkflow-automation
#!/usr/bin/env python3
"""Detect and repair stale Docker Desktop bind views for ODS services.
Runs only under real WSL + Docker Desktop. Compares fresh host bind metadata
against the ORIGINAL RUNNING container's bind view via `docker exec stat`.
Recreates only services whose bind view is provably stale, using the exact
compose flags the caller already resolved. Never touches volumes, auth, or
unrelated projects. Never prints secrets.
"""
import argparse
import importlib.util
import json
import os
import re
import selectors
import shutil
import stat as statmod
import subprocess
import sys
import tarfile
import time
import uuid
from pathlib import Path
PROBE_TIMEOUT = 30
COMPOSE_TIMEOUT = 60
BACKUP_ROOT_REL = "data/wsl-bind-recovery"
BACKUP_MAX_BYTES = 1024 * 1024 * 1024
BACKUP_MAX_ENTRIES = 10000
RECOVERY_TIMEOUT = 120
_deadline = None
_install_dir = None
def log(msg):
print(f"[wsl-bind-recovery] {msg}", file=sys.stderr)
def fail(msg):
print(f"[wsl-bind-recovery] ERROR: {msg}", file=sys.stderr)
return 1
def run(argv, timeout, check=True, capture=True, binary=False):
if argv[:2] == ["docker", "compose"]:
# A recovery may run long after CLI resolution. Recheck the current
# recipe bytes before rendering or recreating any part of the stack.
policy_path = Path(__file__).with_name("compose-cache-policy.py")
try:
spec = importlib.util.spec_from_file_location("recovery_compose_policy", policy_path)
policy = importlib.util.module_from_spec(spec)
spec.loader.exec_module(policy)
policy.validate_flags(_install_dir or Path.cwd(), argv[2:])
except (OSError, ValueError, ImportError) as exc:
raise RuntimeError(f"saved Compose policy rejected recovery: {exc}") from exc
if _deadline is not None:
remaining = _deadline - time.monotonic()
if remaining <= 0:
raise RuntimeError("total recovery time budget exceeded")
timeout = min(timeout, remaining)
return subprocess.run(
argv, timeout=timeout, check=check,
stdout=subprocess.PIPE if capture else None,
stderr=subprocess.PIPE if capture else None,
text=not binary,
)
def is_wsl_docker_desktop():
try:
rel = Path("/proc/sys/kernel/osrelease").read_text().lower()
except OSError:
return False
if "microsoft" not in rel:
return False
try:
info = run(["docker", "info", "--format", "{{.OperatingSystem}}"], 15)
except (subprocess.SubprocessError, FileNotFoundError):
return False
os_name = (info.stdout or "").strip().lower()
return "docker desktop" in os_name or "dockerdesktop" in os_name
def compose_config_json(flags):
argv = ["docker", "compose", *flags, "config", "--format", "json"]
try:
out = run(argv, 30)
except subprocess.SubprocessError as exc:
raise RuntimeError(f"compose config failed: {exc}")
try:
config = json.loads(out.stdout or "{}")
except json.JSONDecodeError as exc:
raise RuntimeError(f"compose config returned invalid JSON: {exc}")
if not isinstance(config, dict) or not isinstance(config.get("services"), dict):
raise RuntimeError("compose config returned an invalid manifest")
return config
def compose_ps(flags):
argv = ["docker", "compose", *flags, "ps", "-a", "--format", "json"]
try:
out = run(argv, 30)
except subprocess.SubprocessError as exc:
raise RuntimeError(f"compose ps failed: {exc}")
text = (out.stdout or "").strip()
if not text:
return []
try:
parsed = json.loads(text)
if isinstance(parsed, list):
if any(not isinstance(row, dict) for row in parsed):
raise RuntimeError("compose ps returned an invalid row")
return parsed
if isinstance(parsed, dict):
return [parsed]
except json.JSONDecodeError:
pass
rows = []
for line in text.splitlines():
line = line.strip()
if not line:
continue
try:
rows.append(json.loads(line))
except json.JSONDecodeError:
raise RuntimeError("compose ps returned invalid JSON")
if not isinstance(rows[-1], dict):
raise RuntimeError("compose ps returned an invalid row")
return rows
def inspect_container(name):
try:
out = run(["docker", "inspect", name], 15)
except subprocess.SubprocessError:
return None
try:
data = json.loads(out.stdout)
except json.JSONDecodeError:
return None
return data[0] if isinstance(data, list) and data and isinstance(data[0], dict) else None
def active_profiles(flags):
profiles = set()
wildcard = False
env = os.environ.get("COMPOSE_PROFILES", "")
for p in env.split(","):
p = p.strip()
if not p:
continue
if p == "*":
wildcard = True
else:
profiles.add(p)
i = 0
while i < len(flags):
f = flags[i]
if f == "--profile" and i + 1 < len(flags):
val = flags[i + 1]
if val == "*":
wildcard = True
else:
profiles.add(val)
i += 2
continue
if f.startswith("--profile="):
val = f.split("=", 1)[1]
if val == "*":
wildcard = True
else:
profiles.add(val)
i += 1
return profiles, wildcard
def service_enabled(svc_cfg, profiles, wildcard=False):
svc_profiles = svc_cfg.get("profiles") or []
if not svc_profiles:
return True
if wildcard:
return True
return any(p in profiles for p in svc_profiles)
def resolve_bind_targets(config, service, profiles, wildcard=False):
services = config.get("services") or {}
svc = services.get(service)
if not svc:
raise RuntimeError(f"service {service} not in compose config")
if profiles is not None and not service_enabled(svc, profiles, wildcard):
return []
binds = []
for vol in svc.get("volumes") or []:
if not isinstance(vol, dict):
continue
vtype = vol.get("type")
if vtype != "bind":
continue
src = vol.get("source")
dst = vol.get("target")
if not src or not dst:
continue
binds.append((src, dst, bool(vol.get("read_only"))))
return binds
def host_stat(source):
try:
s = os.stat(source)
except OSError as exc:
return {"error": exc.errno}
if statmod.S_ISDIR(s.st_mode):
filetype = "dir"
elif statmod.S_ISREG(s.st_mode):
filetype = "file"
else:
filetype = "other"
return {
"device": s.st_dev,
"inode": s.st_ino,
"filetype": filetype,
}
def exec_stat(container_name, target):
"""Return {device,inode,filetype} or {error} for target inside container."""
argv = [
"docker", "exec", container_name,
"stat", "-Lc", "%d:%i:%f", "--", target,
]
try:
out = run(argv, PROBE_TIMEOUT)
except subprocess.SubprocessError as exc:
raise RuntimeError(f"docker exec stat failed for {container_name}:{target}: {exc}")
line = (out.stdout or "").strip()
if not line:
raise RuntimeError(f"empty stat output for {container_name}:{target}")
parts = line.split(":")
if len(parts) != 3:
raise RuntimeError(f"unexpected stat output for {container_name}:{target}")
try:
device = int(parts[0], 10)
inode = int(parts[1], 10)
mode = int(parts[2], 16)
except ValueError as exc:
raise RuntimeError(f"unparseable stat output for {container_name}:{target}: {exc}")
if statmod.S_ISDIR(mode):
filetype = "dir"
elif statmod.S_ISREG(mode):
filetype = "file"
else:
filetype = "other"
return {"device": device, "inode": inode, "filetype": filetype}
def prevalidate_binds(binds):
for src, _dst, _ro in binds:
host = host_stat(src)
if "error" in host:
raise RuntimeError(f"missing host source {src}")
if host["filetype"] not in ("dir", "file"):
raise RuntimeError(f"unsupported host filetype for {src}")
def bind_view_matches(container_name, src, dst, read_only, host=None, seen=None):
host = host if host is not None else host_stat(src)
seen = seen if seen is not None else exec_stat(container_name, dst)
if (host["device"], host["inode"], host["filetype"]) == (
seen["device"], seen["inode"], seen["filetype"]):
return True
if (not read_only or host["inode"] != seen["inode"] or
host["filetype"] != seen["filetype"]):
return False
# Docker Desktop exposes a Windows v9fs bind through a different mount
# namespace. Its st_dev may change while the object inode stays the same.
# Keep the strict identity check for writable and native Linux binds.
try:
host_fs = run(["stat", "-f", "-c", "%T", "--", src], PROBE_TIMEOUT).stdout.strip()
seen_fs = run(["docker", "exec", container_name, "stat", "-f", "-c",
"%T", "--", dst], PROBE_TIMEOUT).stdout.strip()
except subprocess.SubprocessError as exc:
raise RuntimeError(f"filesystem probe failed for {container_name}:{dst}: {exc}")
if host_fs != "v9fs" or seen_fs != "v9fs":
return False
container = inspect_container(container_name)
if not container or not isinstance(container.get("Mounts"), list):
raise RuntimeError(f"mount inspection failed for {container_name}:{dst}")
mounts = [m for m in container["Mounts"] if m.get("Destination") == dst]
if (len(mounts) != 1 or mounts[0].get("Type") != "bind" or
mounts[0].get("Source") != src or mounts[0].get("RW") is not False):
raise RuntimeError(f"unexpected read-only bind declaration for {container_name}:{dst}")
return True
def classify_running(container_name, binds):
prevalidate_binds(binds)
stale_target = ""
for src, dst, ro in binds:
host = host_stat(src)
seen = exec_stat(container_name, dst)
if not bind_view_matches(container_name, src, dst, ro, host, seen):
stale_target = stale_target or dst
return (True, f"stale-bind:{stale_target}") if stale_target else (False, "healthy")
def classify_stopped(container, binds):
prevalidate_binds(binds)
err = (container.get("State") or {}).get("Error") or ""
if not err:
return False, "stopped-no-evidence"
low = err.lower()
if "not a directory" not in low and "is a directory" not in low:
return False, "stopped-no-evidence"
# Require a declared target as a complete path token. A named-volume,
# executable or working-directory error elsewhere is not stale-bind proof.
for _src, dst, _ro in binds:
if re.search(r"(?<![\w/.-])" + re.escape(dst) + r"(?![\w/.-])", err):
return True, "stopped-oci-mount-error"
return False, "stopped-no-evidence"
def _bounded_tar_stream(container_name, argv, archive_path, deadline):
"""Stream tar stdout from docker exec into archive_path with byte/time caps.
Returns (bytes_written, member_count). Raises RuntimeError on any failure.
"""
capacity = os.statvfs(Path(archive_path).parent)
if capacity.f_bavail * capacity.f_frsize < BACKUP_MAX_BYTES + 16 * 1024 * 1024:
raise RuntimeError("private backup capacity insufficient; no service recreated")
sel = selectors.DefaultSelector()
proc = None
total = 0
members = 0
try:
proc = subprocess.Popen(
argv, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL,
)
sel.register(proc.stdout, selectors.EVENT_READ)
fd = os.open(archive_path, os.O_WRONLY | os.O_CREAT | os.O_EXCL | os.O_NOFOLLOW, 0o600)
with os.fdopen(fd, "wb") as fh:
while True:
remaining = deadline - time.monotonic()
if remaining <= 0:
raise RuntimeError(
f"tar stream timed out for {container_name}")
events = sel.select(timeout=min(remaining, 1.0))
if not events:
if proc.poll() is not None:
break
continue
chunk = os.read(proc.stdout.fileno(), 65536)
if not chunk:
break
total += len(chunk)
if total > BACKUP_MAX_BYTES:
raise RuntimeError(
f"tar stream exceeded byte budget for {container_name}")
fh.write(chunk)
rc = proc.wait(timeout=max(0.1, deadline - time.monotonic()))
if rc != 0:
raise RuntimeError(
f"tar stream exited {rc} for {container_name}")
except (OSError, subprocess.SubprocessError) as exc:
raise RuntimeError(f"tar stream failed for {container_name}: {exc}")
finally:
try:
sel.close()
except Exception:
pass
if proc is not None and proc.poll() is None:
try:
proc.kill()
except Exception:
pass
if proc is not None and proc.stdout is not None:
proc.stdout.close()
try:
proc.wait(timeout=5)
except Exception:
pass
# Validate member count without reading content.
try:
with tarfile.open(archive_path, "r:") as tf:
for _ in tf:
members += 1
if members > BACKUP_MAX_ENTRIES:
raise RuntimeError(
f"tar archive exceeded entry budget for {container_name}")
except tarfile.TarError as exc:
raise RuntimeError(
f"tar archive unreadable for {container_name}: {exc}")
return total, members
def backup_phantom(install_dir, container_name, binds):
"""Preserve RW phantom data from the CURRENT container view.
Only backs up binds whose container view is provably stale relative to
the host. Archives are streamed from the container's own view, never
copied from the host filesystem.
"""
parent = install_dir
for component in Path(BACKUP_ROOT_REL).parts:
parent = parent / component
try:
parent.mkdir(mode=0o700)
except FileExistsError:
pass
metadata = parent.lstat()
if not statmod.S_ISDIR(metadata.st_mode) or metadata.st_uid != os.getuid():
raise RuntimeError("unsafe backup directory ownership or type")
root = parent
if statmod.S_IMODE(root.stat().st_mode) & 0o077:
raise RuntimeError("backup directory must be owner-private")
txn = root / uuid.uuid4().hex
txn.mkdir(mode=0o700)
os.chmod(txn, 0o700)
total = 0
entries = 0
deadline = time.monotonic() + PROBE_TIMEOUT
if _deadline is not None:
deadline = min(deadline, _deadline)
for idx, (src, dst, ro) in enumerate(binds):
if ro:
continue
host = host_stat(src)
if "error" in host:
raise RuntimeError(f"missing host source {src}")
if host["filetype"] not in ("dir", "file"):
raise RuntimeError(f"unsupported host filetype for {src}")
try:
seen = exec_stat(container_name, dst)
except RuntimeError as exc:
raise RuntimeError(str(exc))
if (host["device"], host["inode"], host["filetype"]) == (
seen["device"], seen["inode"], seen["filetype"]):
continue
if seen["filetype"] not in ("dir", "file"):
raise RuntimeError(
f"unsupported container filetype for {container_name}:{dst}")
archive = txn / f"{idx:04d}.tar"
if seen["filetype"] == "dir":
argv = ["docker", "exec", container_name,
"tar", "-cf", "-", "-C", dst, "."]
else:
parent = os.path.dirname(dst.rstrip("/")) or "/"
base = os.path.basename(dst.rstrip("/"))
argv = ["docker", "exec", container_name,
"tar", "-cf", "-", "-C", parent, base]
written, members = _bounded_tar_stream(
container_name, argv, archive, deadline)
total += written
entries += members
if total > BACKUP_MAX_BYTES:
raise RuntimeError("phantom backup byte budget exceeded")
if entries > BACKUP_MAX_ENTRIES:
raise RuntimeError("phantom backup entry budget exceeded")
log(f"preserved phantom {dst} -> {archive}")
return total, entries
def recreate(flags, service):
argv = [
"docker", "compose", *flags, "up", "-d", "--no-deps",
"--no-build", "--pull", "never", "--force-recreate", service,
]
run(argv, COMPOSE_TIMEOUT, check=True)
def verify(flags, service, expected_binds):
try:
rows = compose_ps(flags)
except RuntimeError as exc:
return False, f"compose-ps-failed:{exc}"
matched = False
for row in rows:
if row.get("Service") != service:
continue
name = row.get("Name")
if not name:
return False, "container-missing"
matched = True
container = inspect_container(name)
if not container:
return False, "inspect-failed"
state = (container.get("State") or {}).get("Status", "")
if (container.get("State") or {}).get("Paused") or state == "paused":
return False, "paused"
if state != "running":
return False, f"state={state}"
try:
prevalidate_binds(expected_binds)
for src, dst, ro in expected_binds:
host = host_stat(src)
seen = exec_stat(name, dst)
if not bind_view_matches(name, src, dst, ro, host, seen):
return False, f"stale-bind:{dst}"
except RuntimeError as exc:
return False, f"verify-error:{exc}"
return (True, "ok") if matched else (False, "container-missing")
def main(argv=None):
global _deadline, _install_dir
_deadline = None
_install_dir = None
parser = argparse.ArgumentParser()
parser.add_argument("--install-dir", required=True)
parser.add_argument("--service", default="")
parser.add_argument("--check", action="store_true")
parser.add_argument("flags", nargs=argparse.REMAINDER)
args = parser.parse_args(argv)
install_dir = Path(args.install_dir).resolve()
if not install_dir.is_dir():
return fail(f"install dir not found: {install_dir}")
_install_dir = install_dir
flags = args.flags[1:] if args.flags and args.flags[0] == "--" else args.flags
if not flags:
return fail("no compose flags supplied")
if not is_wsl_docker_desktop():
log("not WSL+DockerDesktop; skipping")
return 0
_deadline = time.monotonic() + RECOVERY_TIMEOUT
if shutil.which("docker") is None:
return fail("docker not found")
try:
config = compose_config_json(flags)
except RuntimeError as exc:
return fail(str(exc))
project = config.get("name") or ""
if not project:
return fail("compose config missing project name")
profiles, wildcard = active_profiles(flags)
requested = args.service.strip()
try:
rows = compose_ps(flags)
except RuntimeError as exc:
return fail(str(exc))
candidates = []
for row in rows:
svc = row.get("Service") or ""
# Compose ps includes project orphans after an extension is removed
# or disabled. Only the caller's selected manifest authorizes recovery.
if svc not in config["services"]:
continue
if requested and svc != requested:
continue
name = row.get("Name")
if not name:
continue
container = inspect_container(name)
if not container:
return fail(f"inspect failed for container {name}")
labels = (container.get("Config") or {}).get("Labels") or {}
if labels.get("com.docker.compose.project") != project:
continue
if labels.get("com.docker.compose.service") != svc:
continue
candidates.append((svc, name, container))
if not candidates:
log("no matching ODS containers; nothing to do")
return 0
# Prevalidate ALL bind sources across ALL selected candidates before any
# classification or mutation. Explicit --service bypasses profile gating
# (matching Compose itself); bulk start skips disabled profiles.
prepared = []
for svc, name, container in candidates:
try:
if requested:
binds = resolve_bind_targets(config, svc, None)
else:
binds = resolve_bind_targets(config, svc, profiles, wildcard)
except RuntimeError as exc:
return fail(str(exc))
if not binds:
continue
try:
prevalidate_binds(binds)
except RuntimeError as exc:
return fail(str(exc))
prepared.append((svc, name, container, binds))
stale_services = []
for svc, name, container, binds in prepared:
state = (container.get("State") or {}).get("Status", "")
paused = bool((container.get("State") or {}).get("Paused"))
if paused or state == "paused":
return fail(
f"service {svc} ({name}) is paused; refusing to mutate")
try:
if state == "running":
stale, reason = classify_running(name, binds)
elif state in ("exited", "created", "dead"):
stale, reason = classify_stopped(container, binds)
else:
stale, reason = False, f"state={state}"
except RuntimeError as exc:
return fail(str(exc))
if stale:
log(f"service {svc} ({name}) stale: {reason}")
stale_services.append((svc, name, binds, state))
else:
log(f"service {svc} ({name}) {reason}")
if not stale_services:
log("all bind views healthy")
return 0
if args.check:
log(f"check mode: {len(stale_services)} stale service(s)")
return 2
backup_bytes = backup_entries = 0
for svc, name, binds, state in stale_services:
if state != "running":
continue
try:
size, count = backup_phantom(install_dir, name, binds)
backup_bytes += size
backup_entries += count
if backup_bytes > BACKUP_MAX_BYTES or backup_entries > BACKUP_MAX_ENTRIES:
raise RuntimeError("combined backup budget exceeded")
except (RuntimeError, OSError) as exc:
return fail(str(exc))
# Compose recreates the entire service, including its replicas. Preserve
# every original stale view above, then recreate and verify each service once.
service_binds = {svc: binds for svc, _name, binds, _state in stale_services}
for svc in service_binds:
log(f"recreating {svc}")
try:
recreate(flags, svc)
except (subprocess.SubprocessError, RuntimeError) as exc:
return fail(f"recreate failed for {svc}: {exc}")
for svc, binds in service_binds.items():
ok, reason = verify(flags, svc, binds)
if not ok:
return fail(f"post-recreate verification failed for {svc}: {reason}")
log(f"verified {svc}")
return 0
if __name__ == "__main__":
sys.exit(main())