Self-improving AI Employees for everyone
Python ai-agentsai-employeeai-employeesassistantautonomous-agentsco-employeescompanygrok-botinstinctloop-agentsmuseown-your-datapersonalself-hosted
fix(machina): Windows subprocess resolution, timestamps, graceful shutdown
trohitg81cf8356f8cf
diff --git a/machina/colors.py b/machina/colors.pyindex 3db209a9..511e26ed 100644--- a/machina/colors.py+++ b/machina/colors.py@@ -1,4 +1,12 @@-"""Rich console + per-service color rotation (Honcho-style)."""+"""Rich console + per-service color rotation (Honcho-style).++Latency-analysis hook: ``emit`` routes through ``Console.log`` instead of+``Console.print``, which prepends a built-in timestamp column. The console+is configured with ``log_time_format=[%H:%M:%S.%f]`` for ms precision and+``log_path=False`` to drop the file:line caller info that rich adds by+default. ``console.print`` callers (banner / step headers) remain timestamp-+less so the launch banner stays readable.+"""from __future__ import annotations@@ -7,8 +15,9 @@ from rich.console import Console# Single shared console — single-writer aggregator avoids interleaving-# concurrent stream output (VS Code's pty host pattern).-console = Console()+# concurrent stream output (VS Code's pty host pattern). ``log_*`` settings+# only affect ``console.log`` calls; ``console.print`` is unaffected.+console = Console(log_time_format="[%H:%M:%S.%f]", log_path=False)# Honcho's rotation, lifted: skip very-dark (black) and very-bright (white)@@ -25,13 +34,14 @@ def next_color() -> str:def emit(name: str, color: str, line: str, *, stream: str = "stdout") -> None:- """Print one line tagged with the service name + color.+ """Print one line tagged with the service name + color, timestamped.- Renders as ``cyan │ <message>`` so prefix width is uniform- across services.+ Renders as ``[14:23:45.123] cyan | <message>``. The timestamp+ column is added by ``console.log``; prefix width stays uniform so+ side-by-side latency comparison across services lines up visually."""width = 12prefix = f"[{color}]{name:<{width}}[/{color}]"style = "" if stream == "stdout" else "[dim]"suffix = "" if stream == "stdout" else "[/dim]"- console.print(f"{prefix}{style} | {line}{suffix}", highlight=False)+ console.log(f"{prefix}{style} | {line}{suffix}", highlight=False)diff --git a/machina/commands/build.py b/machina/commands/build.pyindex a51c3cf5..d8ec3356 100644--- a/machina/commands/build.py+++ b/machina/commands/build.py@@ -72,22 +72,22 @@ def _ensure_uv(python_cmd: str) -> str:def _ensure_temporal() -> None:- """``temporal-server`` is a global npm CLI; install if missing (non-fatal)."""- version = capture(["temporal-server", "--version"])+ """Ensure the ``temporal`` CLI (from npm package ``temporal-server``) is on PATH."""+ version = capture(["temporal", "--version"])if version:- console.print(f" temporal-server: {version}")+ console.print(f" temporal: {version}")return- console.print(" temporal-server: not found, installing globally...")+ console.print(" temporal: not found, installing globally...")rc = run(["npm", "install", "-g", "temporal-server"], check=False)if rc != 0:console.print(- " [yellow]Warning: temporal-server install failed. "+ " [yellow]Warning: temporal install failed. ""Distributed execution unavailable.[/]")return- version = capture(["temporal-server", "--version"])+ version = capture(["temporal", "--version"])if version:- console.print(f" temporal-server: {version}")+ console.print(f" temporal: {version}")# ---------------------------------------------------------------- build@@ -135,25 +135,28 @@ def build_command() -> None:env_path = root / ".env"template_path = root / ".env.template"+ # Step markers go through ``console.log`` so each [N/5] line is+ # timestamped — diff between consecutive timestamps is the wall-clock+ # cost of that step, no manual instrumentation needed.if not env_path.exists() and template_path.exists():shutil.copy2(template_path, env_path)- console.print("[0/5] Created .env from template")+ console.log("[0/5] Created .env from template")if not is_postinstall:- console.print("[1/5] Installing dependencies...")+ console.log("[1/5] Installing dependencies...")run(["pnpm", "install"], cwd=root)else:- console.print("[1/5] Dependencies already installed by package manager")+ console.log("[1/5] Dependencies already installed by package manager")- console.print("[2/5] Building client...")+ console.log("[2/5] Building client...")run(["pnpm", "--filter", "react-flow-client", "run", "build"], cwd=root)- console.print("[3/5] Installing Python dependencies...")+ console.log("[3/5] Installing Python dependencies...")if not (server_dir / ".venv").exists():run(["uv", "venv"], cwd=server_dir)run(["uv", "sync"], cwd=server_dir)- console.print("[4/5] Installing Playwright browser...")+ console.log("[4/5] Installing Playwright browser...")rc = run(["playwright", "install", "chromium"], cwd=server_dir, check=False)if rc != 0:console.print(@@ -161,7 +164,7 @@ def build_command() -> None:"JS-rendered scraping unavailable.[/]")- console.print("[5/5] Verifying edgymeow binary...")+ console.log("[5/5] Verifying edgymeow binary...")bin_name = "edgymeow-server.exe" if sys.platform == "win32" else "edgymeow-server"edgymeow_bin = root / "node_modules" / "edgymeow" / "bin" / bin_nameif edgymeow_bin.exists():@@ -172,4 +175,4 @@ def build_command() -> None:"Set WHATSAPP_RUNTIME_ENABLED=false to disable.[/]")- console.print("\n[green]Build complete.[/]")+ console.log("[green]Build complete.[/]")diff --git a/machina/commands/dev.py b/machina/commands/dev.pyindex 78e34361..90c4fcb8 100644--- a/machina/commands/dev.py+++ b/machina/commands/dev.py@@ -83,7 +83,7 @@ def _build_specs(root: Path, cfg, *, daemon: bool, use_vite: bool) -> list[Serviserver_spec,ServiceSpec(name="temporal",- argv=["temporal-server", "api"],+ argv=["temporal", "api"],cwd=root,ready_port=cfg.temporal_port,),@@ -106,9 +106,10 @@ def dev_command(console.print(f"Mode: {'Daemon (uvicorn)' if daemon else 'Development (uvicorn)'}")console.print(f"Ports: {', '.join(str(p) for p in cfg.all_ports)}")- console.print("Freeing ports...")+ console.log("Freeing ports...")for port in cfg.all_ports:kill_port(port)+ console.log("Ports ready")_clear_vite_cache(root)diff --git a/machina/commands/start.py b/machina/commands/start.pyindex d12270e6..81bb69ab 100644--- a/machina/commands/start.py+++ b/machina/commands/start.py@@ -26,6 +26,7 @@ from machina.config import load_configfrom machina.platform_ import IS_WINDOWS, IS_WSL, platform_name, project_rootfrom machina.buildenv import validate_build, venv_pythonfrom machina.ports import kill_port+from machina.run import which_argvfrom machina.supervisor import Manager, RestartPolicy, ServiceSpec@@ -69,10 +70,10 @@ def _sqlalchemy_preflight(root: Path) -> None:def _temporal_running() -> bool:- """Check whether ``temporal-server`` is already up via its CLI."""+ """Check whether the temporal server is already up via the ``temporal`` CLI."""try:out = subprocess.run(- ["temporal-server", "status"],+ which_argv(["temporal", "status"]),capture_output=True,text=True,timeout=5,@@ -123,7 +124,7 @@ def _build_specs(root: Path, cfg, *, temporal_running: bool) -> list[ServiceSpecspecs.append(ServiceSpec(name="temporal",- argv=["temporal-server", "api"],+ argv=["temporal", "api"],cwd=root,ready_port=cfg.temporal_port,)@@ -143,10 +144,10 @@ def start_command() -> None:if temporal_running:console.print("[dim]Temporal already running, skipping[/]")- console.print("Freeing ports...")+ console.log("Freeing ports...")for port in cfg.all_ports:kill_port(port)- console.print("Ports ready")+ console.log("Ports ready")version = _read_version(root)console.print()diff --git a/machina/run.py b/machina/run.pyindex 603cac5b..e322f1dd 100644--- a/machina/run.py+++ b/machina/run.py@@ -17,6 +17,7 @@ swallowed stderr, etc.).from __future__ import annotations+import shutilimport subprocessfrom pathlib import Path@@ -25,6 +26,20 @@ import typerfrom machina.colors import console+def which_argv(argv: list[str]) -> list[str]:+ """Resolve argv[0] via PATH (PATHEXT-aware on Windows for .cmd/.bat).++ ``subprocess.run`` / ``anyio.open_process`` with ``shell=False`` do not+ consult PATHEXT, so on Windows ``["npm", ...]`` raises FileNotFoundError+ because the launcher is ``npm.cmd``. ``shutil.which`` mirrors what the+ shell would resolve. Shared by ``run`` / ``capture`` and the supervisor+ so all subprocess entry points behave identically.+ """+ if not argv:+ return argv+ return [shutil.which(argv[0]) or argv[0], *argv[1:]]++def run(argv: list[str],*,@@ -32,7 +47,14 @@ def run(check: bool = True,) -> int:"""Inherit-stdio run; raises :class:`typer.Exit` on non-zero when ``check``."""- proc = subprocess.run(argv, cwd=str(cwd) if cwd else None)+ resolved = which_argv(argv)+ try:+ proc = subprocess.run(resolved, cwd=str(cwd) if cwd else None)+ except FileNotFoundError:+ console.print(f"[red]Command not found:[/] {argv[0] if argv else ''}")+ if check:+ raise typer.Exit(code=127)+ return 127if check and proc.returncode != 0:console.print(f"[red]Command failed:[/] {' '.join(argv)}")raise typer.Exit(code=proc.returncode)@@ -47,7 +69,7 @@ def capture(argv: list[str], *, cwd: Path | str | None = None) -> str | None:"""try:result = subprocess.run(- argv,+ which_argv(argv),cwd=str(cwd) if cwd else None,capture_output=True,text=True,diff --git a/machina/supervisor.py b/machina/supervisor.pyindex f65406c7..895cfbf4 100644--- a/machina/supervisor.py+++ b/machina/supervisor.py@@ -31,6 +31,7 @@ from typing import Iterable, Optionalimport anyiofrom machina.colors import emit, next_color+from machina.run import which_argvfrom machina.tcp import wait_for_tcp_portfrom machina.tree import add_to_job, kill_tree, new_session_kwargs@@ -67,26 +68,6 @@ def _full_env(spec_env: dict[str, str]) -> dict[str, str]:}-async def _terminate_then_kill(- proc: anyio.abc.Process, *, grace: float-) -> None:- """SIGTERM (POSIX) / TerminateProcess (Win) -> wait grace -> tree-kill."""- if proc.returncode is not None:- return- try:- proc.terminate()- except ProcessLookupError:- return- with anyio.move_on_after(grace):- await proc.wait()- if proc.returncode is None:- kill_tree(proc.pid)- try:- await proc.wait()- except Exception:- pass--class Manager:"""Supervise multiple ``ServiceSpec`` instances in one TaskGroup."""@@ -94,6 +75,12 @@ class Manager:self._specs: list[ServiceSpec] = []self._fatal: list[str] = [] # names of services that crash-loopedself._active = 0+ # name -> (Process, color). Populated by _spawn_once for the+ # lifetime of each child. Used by _wait_for_signal so a second+ # Ctrl-C can force-kill PIDs synchronously without waiting on the+ # cancel-scope unwind.+ self._procs: dict[str, tuple[anyio.abc.Process, str]] = {}+ self._shutting_down = Falsedef add(self, spec: ServiceSpec) -> None:if spec.color is None:@@ -119,6 +106,13 @@ class Manager:for exc in eg.exceptions:emit("manager", "red", f"unexpected error: {exc}", stream="stderr")unhandled = True+ finally:+ # Runs even on ``CancelledError`` (which is ``BaseException``+ # in py311+ and so escapes ``except* Exception``). Without the+ # finally, the shutdown-complete marker is silently skipped on+ # the Ctrl-C path -- the user's actual exit case.+ if self._shutting_down:+ emit("manager", "green", "shutdown complete")if unhandled:return 1return 1 if self._fatal else 0@@ -137,12 +131,20 @@ class Manager:cancel_scope.cancel()async def _wait_for_signal(self, cancel_scope: anyio.CancelScope) -> None:- """Cancel the TaskGroup on Ctrl-C / SIGTERM (cross-platform).+ """Two-stage Ctrl-C / SIGTERM handler.++ First signal: emit a "shutting down" notice and cancel the task+ group so each ``_spawn_once`` unwinds via :meth:`_stop_proc`+ (SIGTERM -> grace -> tree-kill).++ Second signal: skip the grace period entirely -- iterate the+ ``self._procs`` registry and tree-kill every still-running PID+ synchronously. This matches the typical "Ctrl-C, then Ctrl-C+ again to force" UX from systemd / VS Code / docker compose.- Windows can only deliver ``SIGINT`` and ``SIGBREAK`` to a Python- process; passing ``SIGTERM`` would raise ``NotImplementedError``- from anyio. We probe the platform and subscribe to whichever- subset is available.+ Windows only delivers ``SIGINT`` / ``SIGBREAK`` to a Python+ process; passing ``SIGTERM`` raises ``NotImplementedError``+ from anyio, so we subscribe to whichever subset is available."""if sys.platform == "win32":signals = (signal.SIGINT, getattr(signal, "SIGBREAK", signal.SIGINT))@@ -151,14 +153,28 @@ class Manager:try:with anyio.open_signal_receiver(*signals) as receiver:async for sig in receiver:- emit(- "manager",- "yellow",- f"received {sig.name}, shutting down",- stream="stderr",- )- cancel_scope.cancel()- return+ if not self._shutting_down:+ self._shutting_down = True+ emit(+ "manager",+ "yellow",+ f"received {sig.name} -- shutting down "+ "(Ctrl-C again to force-kill)",+ stream="stderr",+ )+ cancel_scope.cancel()+ else:+ emit(+ "manager",+ "red",+ f"received {sig.name} again -- force-killing",+ stream="stderr",+ )+ for name, (proc, color) in list(self._procs.items()):+ if proc.returncode is None:+ kill_tree(proc.pid)+ emit(name, color, "force-killed", stream="stderr")+ returnexcept NotImplementedError:# Some environments (eg. asyncio on Windows without# ProactorEventLoop, or signal handling from a non-main@@ -167,6 +183,46 @@ class Manager:# the cancel scope when they finish.await anyio.sleep_forever()+ async def _stop_proc(self, proc: anyio.abc.Process, spec: ServiceSpec) -> None:+ """SIGTERM (POSIX) / TerminateProcess (Win) -> wait grace -> tree-kill.++ Wrapped in ``CancelScope(shield=True)`` -- this is the anyio-+ documented idiom for cleanup that must run to completion during+ an outer cancellation. Without the shield, ``_stop_proc`` is+ invoked from inside an already-cancelled scope (the supervisor+ signal handler cancelled the task group), so the very first+ ``await proc.wait()`` re-raises ``CancelledError`` and the grace+ period never elapses.+ """+ if proc.returncode is not None:+ return+ color = spec.color or "white"+ emit(spec.name, color, "stopping", stream="stderr")+ with anyio.CancelScope(shield=True):+ try:+ proc.terminate()+ except ProcessLookupError:+ return+ with anyio.move_on_after(spec.terminate_grace_seconds):+ await proc.wait()+ if proc.returncode is None:+ emit(+ spec.name,+ "red",+ f"did not exit in {spec.terminate_grace_seconds:.0f}s, "+ "force-killing process tree",+ stream="stderr",+ )+ kill_tree(proc.pid)+ with anyio.move_on_after(2.0):+ await proc.wait()+ emit(+ spec.name,+ color,+ f"stopped (exit {proc.returncode})",+ stream="stderr",+ )+async def _supervise(self, spec: ServiceSpec, cancel_scope: anyio.CancelScope) -> None:@@ -212,32 +268,45 @@ class Manager:backoff_seconds = min(backoff_seconds * 2, 30.0)async def _spawn_once(self, spec: ServiceSpec) -> int:- """Spawn the service, drain its output, return exit code."""+ """Spawn the service, drain its output, return exit code.++ ``async with`` on the ``anyio.Process`` runs ``aclose`` on exit,+ which closes stdin/stdout/stderr and awaits the child. Without+ this, asyncio's Windows ProactorEventLoop leaves the underlying+ ``BaseSubprocessTransport`` un-closed and ``__del__`` later logs+ ``ResourceWarning: unclosed transport`` / ``ValueError: I/O+ operation on closed pipe`` at GC time.+ """env = _full_env(spec.env)spawn_kwargs = new_session_kwargs()- proc = await anyio.open_process(- spec.argv,+ async with await anyio.open_process(+ which_argv(spec.argv),cwd=str(spec.cwd) if spec.cwd else None,env=env,stdout=subprocess.PIPE,stderr=subprocess.PIPE,**spawn_kwargs,- )- if sys.platform == "win32":- add_to_job(proc.pid)- emit(spec.name, spec.color or "white", f"started pid={proc.pid}")-- try:- async with anyio.create_task_group() as tg:- tg.start_soon(self._drain, proc.stdout, spec, "stdout")- tg.start_soon(self._drain, proc.stderr, spec, "stderr")- if spec.ready_port:- tg.start_soon(self._announce_ready, spec)- exit_code = await proc.wait()- tg.cancel_scope.cancel()- except anyio.get_cancelled_exc_class():- await _terminate_then_kill(proc, grace=spec.terminate_grace_seconds)- raise+ ) as proc:+ if sys.platform == "win32":+ add_to_job(proc.pid)+ color = spec.color or "white"+ emit(spec.name, color, f"started pid={proc.pid}")+ # Register for force-kill access on second Ctrl-C; deregister on+ # exit so a restarted-then-died service doesn't leave a stale entry.+ self._procs[spec.name] = (proc, color)+ try:+ async with anyio.create_task_group() as tg:+ tg.start_soon(self._drain, proc.stdout, spec, "stdout")+ tg.start_soon(self._drain, proc.stderr, spec, "stderr")+ if spec.ready_port:+ tg.start_soon(self._announce_ready, spec)+ exit_code = await proc.wait()+ tg.cancel_scope.cancel()+ except anyio.get_cancelled_exc_class():+ await self._stop_proc(proc, spec)+ raise+ finally:+ self._procs.pop(spec.name, None)return exit_codeasync def _drain(