Skip to content

Self-improving AI Employees for everyone

fix(machina): Windows subprocess resolution, timestamps, graceful shutdown

trohitg81cf8356f8cf
diff --git a/machina/colors.py b/machina/colors.py
index 3db209a9..511e26ed 100644
--- a/machina/colors.py
+++ b/machina/colors.py
@@ -1,4 +1,12 @@
-"""Rich console + per-service color rotation (Honcho-style)."""
+"""Rich console + per-service color rotation (Honcho-style).
+
+Latency-analysis hook: ``emit`` routes through ``Console.log`` instead of
+``Console.print``, which prepends a built-in timestamp column. The console
+is configured with ``log_time_format=[%H:%M:%S.%f]`` for ms precision and
+``log_path=False`` to drop the file:line caller info that rich adds by
+default. ``console.print`` callers (banner / step headers) remain timestamp-
+less so the launch banner stays readable.
+"""
from __future__ import annotations
@@ -7,8 +15,9 @@ from rich.console import Console
# Single shared console — single-writer aggregator avoids interleaving
-# concurrent stream output (VS Code's pty host pattern).
-console = Console()
+# concurrent stream output (VS Code's pty host pattern). ``log_*`` settings
+# only affect ``console.log`` calls; ``console.print`` is unaffected.
+console = Console(log_time_format="[%H:%M:%S.%f]", log_path=False)
# Honcho's rotation, lifted: skip very-dark (black) and very-bright (white)
@@ -25,13 +34,14 @@ def next_color() -> str:
def emit(name: str, color: str, line: str, *, stream: str = "stdout") -> None:
- """Print one line tagged with the service name + color.
+ """Print one line tagged with the service name + color, timestamped.
- Renders as ``cyan │ <message>`` so prefix width is uniform
- across services.
+ Renders as ``[14:23:45.123] cyan | <message>``. The timestamp
+ column is added by ``console.log``; prefix width stays uniform so
+ side-by-side latency comparison across services lines up visually.
"""
width = 12
prefix = f"[{color}]{name:<{width}}[/{color}]"
style = "" if stream == "stdout" else "[dim]"
suffix = "" if stream == "stdout" else "[/dim]"
- console.print(f"{prefix}{style} | {line}{suffix}", highlight=False)
+ console.log(f"{prefix}{style} | {line}{suffix}", highlight=False)
diff --git a/machina/commands/build.py b/machina/commands/build.py
index a51c3cf5..d8ec3356 100644
--- a/machina/commands/build.py
+++ b/machina/commands/build.py
@@ -72,22 +72,22 @@ def _ensure_uv(python_cmd: str) -> str:
def _ensure_temporal() -> None:
- """``temporal-server`` is a global npm CLI; install if missing (non-fatal)."""
- version = capture(["temporal-server", "--version"])
+ """Ensure the ``temporal`` CLI (from npm package ``temporal-server``) is on PATH."""
+ version = capture(["temporal", "--version"])
if version:
- console.print(f" temporal-server: {version}")
+ console.print(f" temporal: {version}")
return
- console.print(" temporal-server: not found, installing globally...")
+ console.print(" temporal: not found, installing globally...")
rc = run(["npm", "install", "-g", "temporal-server"], check=False)
if rc != 0:
console.print(
- " [yellow]Warning: temporal-server install failed. "
+ " [yellow]Warning: temporal install failed. "
"Distributed execution unavailable.[/]"
)
return
- version = capture(["temporal-server", "--version"])
+ version = capture(["temporal", "--version"])
if version:
- console.print(f" temporal-server: {version}")
+ console.print(f" temporal: {version}")
# ---------------------------------------------------------------- build
@@ -135,25 +135,28 @@ def build_command() -> None:
env_path = root / ".env"
template_path = root / ".env.template"
+ # Step markers go through ``console.log`` so each [N/5] line is
+ # timestamped — diff between consecutive timestamps is the wall-clock
+ # cost of that step, no manual instrumentation needed.
if not env_path.exists() and template_path.exists():
shutil.copy2(template_path, env_path)
- console.print("[0/5] Created .env from template")
+ console.log("[0/5] Created .env from template")
if not is_postinstall:
- console.print("[1/5] Installing dependencies...")
+ console.log("[1/5] Installing dependencies...")
run(["pnpm", "install"], cwd=root)
else:
- console.print("[1/5] Dependencies already installed by package manager")
+ console.log("[1/5] Dependencies already installed by package manager")
- console.print("[2/5] Building client...")
+ console.log("[2/5] Building client...")
run(["pnpm", "--filter", "react-flow-client", "run", "build"], cwd=root)
- console.print("[3/5] Installing Python dependencies...")
+ console.log("[3/5] Installing Python dependencies...")
if not (server_dir / ".venv").exists():
run(["uv", "venv"], cwd=server_dir)
run(["uv", "sync"], cwd=server_dir)
- console.print("[4/5] Installing Playwright browser...")
+ console.log("[4/5] Installing Playwright browser...")
rc = run(["playwright", "install", "chromium"], cwd=server_dir, check=False)
if rc != 0:
console.print(
@@ -161,7 +164,7 @@ def build_command() -> None:
"JS-rendered scraping unavailable.[/]"
)
- console.print("[5/5] Verifying edgymeow binary...")
+ console.log("[5/5] Verifying edgymeow binary...")
bin_name = "edgymeow-server.exe" if sys.platform == "win32" else "edgymeow-server"
edgymeow_bin = root / "node_modules" / "edgymeow" / "bin" / bin_name
if edgymeow_bin.exists():
@@ -172,4 +175,4 @@ def build_command() -> None:
"Set WHATSAPP_RUNTIME_ENABLED=false to disable.[/]"
)
- console.print("\n[green]Build complete.[/]")
+ console.log("[green]Build complete.[/]")
diff --git a/machina/commands/dev.py b/machina/commands/dev.py
index 78e34361..90c4fcb8 100644
--- a/machina/commands/dev.py
+++ b/machina/commands/dev.py
@@ -83,7 +83,7 @@ def _build_specs(root: Path, cfg, *, daemon: bool, use_vite: bool) -> list[Servi
server_spec,
ServiceSpec(
name="temporal",
- argv=["temporal-server", "api"],
+ argv=["temporal", "api"],
cwd=root,
ready_port=cfg.temporal_port,
),
@@ -106,9 +106,10 @@ def dev_command(
console.print(f"Mode: {'Daemon (uvicorn)' if daemon else 'Development (uvicorn)'}")
console.print(f"Ports: {', '.join(str(p) for p in cfg.all_ports)}")
- console.print("Freeing ports...")
+ console.log("Freeing ports...")
for port in cfg.all_ports:
kill_port(port)
+ console.log("Ports ready")
_clear_vite_cache(root)
diff --git a/machina/commands/start.py b/machina/commands/start.py
index d12270e6..81bb69ab 100644
--- a/machina/commands/start.py
+++ b/machina/commands/start.py
@@ -26,6 +26,7 @@ from machina.config import load_config
from machina.platform_ import IS_WINDOWS, IS_WSL, platform_name, project_root
from machina.buildenv import validate_build, venv_python
from machina.ports import kill_port
+from machina.run import which_argv
from machina.supervisor import Manager, RestartPolicy, ServiceSpec
@@ -69,10 +70,10 @@ def _sqlalchemy_preflight(root: Path) -> None:
def _temporal_running() -> bool:
- """Check whether ``temporal-server`` is already up via its CLI."""
+ """Check whether the temporal server is already up via the ``temporal`` CLI."""
try:
out = subprocess.run(
- ["temporal-server", "status"],
+ which_argv(["temporal", "status"]),
capture_output=True,
text=True,
timeout=5,
@@ -123,7 +124,7 @@ def _build_specs(root: Path, cfg, *, temporal_running: bool) -> list[ServiceSpec
specs.append(
ServiceSpec(
name="temporal",
- argv=["temporal-server", "api"],
+ argv=["temporal", "api"],
cwd=root,
ready_port=cfg.temporal_port,
)
@@ -143,10 +144,10 @@ def start_command() -> None:
if temporal_running:
console.print("[dim]Temporal already running, skipping[/]")
- console.print("Freeing ports...")
+ console.log("Freeing ports...")
for port in cfg.all_ports:
kill_port(port)
- console.print("Ports ready")
+ console.log("Ports ready")
version = _read_version(root)
console.print()
diff --git a/machina/run.py b/machina/run.py
index 603cac5b..e322f1dd 100644
--- a/machina/run.py
+++ b/machina/run.py
@@ -17,6 +17,7 @@ swallowed stderr, etc.).
from __future__ import annotations
+import shutil
import subprocess
from pathlib import Path
@@ -25,6 +26,20 @@ import typer
from machina.colors import console
+def which_argv(argv: list[str]) -> list[str]:
+ """Resolve argv[0] via PATH (PATHEXT-aware on Windows for .cmd/.bat).
+
+ ``subprocess.run`` / ``anyio.open_process`` with ``shell=False`` do not
+ consult PATHEXT, so on Windows ``["npm", ...]`` raises FileNotFoundError
+ because the launcher is ``npm.cmd``. ``shutil.which`` mirrors what the
+ shell would resolve. Shared by ``run`` / ``capture`` and the supervisor
+ so all subprocess entry points behave identically.
+ """
+ if not argv:
+ return argv
+ return [shutil.which(argv[0]) or argv[0], *argv[1:]]
+
+
def run(
argv: list[str],
*,
@@ -32,7 +47,14 @@ def run(
check: bool = True,
) -> int:
"""Inherit-stdio run; raises :class:`typer.Exit` on non-zero when ``check``."""
- proc = subprocess.run(argv, cwd=str(cwd) if cwd else None)
+ resolved = which_argv(argv)
+ try:
+ proc = subprocess.run(resolved, cwd=str(cwd) if cwd else None)
+ except FileNotFoundError:
+ console.print(f"[red]Command not found:[/] {argv[0] if argv else ''}")
+ if check:
+ raise typer.Exit(code=127)
+ return 127
if check and proc.returncode != 0:
console.print(f"[red]Command failed:[/] {' '.join(argv)}")
raise typer.Exit(code=proc.returncode)
@@ -47,7 +69,7 @@ def capture(argv: list[str], *, cwd: Path | str | None = None) -> str | None:
"""
try:
result = subprocess.run(
- argv,
+ which_argv(argv),
cwd=str(cwd) if cwd else None,
capture_output=True,
text=True,
diff --git a/machina/supervisor.py b/machina/supervisor.py
index f65406c7..895cfbf4 100644
--- a/machina/supervisor.py
+++ b/machina/supervisor.py
@@ -31,6 +31,7 @@ from typing import Iterable, Optional
import anyio
from machina.colors import emit, next_color
+from machina.run import which_argv
from machina.tcp import wait_for_tcp_port
from machina.tree import add_to_job, kill_tree, new_session_kwargs
@@ -67,26 +68,6 @@ def _full_env(spec_env: dict[str, str]) -> dict[str, str]:
}
-async def _terminate_then_kill(
- proc: anyio.abc.Process, *, grace: float
-) -> None:
- """SIGTERM (POSIX) / TerminateProcess (Win) -> wait grace -> tree-kill."""
- if proc.returncode is not None:
- return
- try:
- proc.terminate()
- except ProcessLookupError:
- return
- with anyio.move_on_after(grace):
- await proc.wait()
- if proc.returncode is None:
- kill_tree(proc.pid)
- try:
- await proc.wait()
- except Exception:
- pass
-
-
class Manager:
"""Supervise multiple ``ServiceSpec`` instances in one TaskGroup."""
@@ -94,6 +75,12 @@ class Manager:
self._specs: list[ServiceSpec] = []
self._fatal: list[str] = [] # names of services that crash-looped
self._active = 0
+ # name -> (Process, color). Populated by _spawn_once for the
+ # lifetime of each child. Used by _wait_for_signal so a second
+ # Ctrl-C can force-kill PIDs synchronously without waiting on the
+ # cancel-scope unwind.
+ self._procs: dict[str, tuple[anyio.abc.Process, str]] = {}
+ self._shutting_down = False
def add(self, spec: ServiceSpec) -> None:
if spec.color is None:
@@ -119,6 +106,13 @@ class Manager:
for exc in eg.exceptions:
emit("manager", "red", f"unexpected error: {exc}", stream="stderr")
unhandled = True
+ finally:
+ # Runs even on ``CancelledError`` (which is ``BaseException``
+ # in py311+ and so escapes ``except* Exception``). Without the
+ # finally, the shutdown-complete marker is silently skipped on
+ # the Ctrl-C path -- the user's actual exit case.
+ if self._shutting_down:
+ emit("manager", "green", "shutdown complete")
if unhandled:
return 1
return 1 if self._fatal else 0
@@ -137,12 +131,20 @@ class Manager:
cancel_scope.cancel()
async def _wait_for_signal(self, cancel_scope: anyio.CancelScope) -> None:
- """Cancel the TaskGroup on Ctrl-C / SIGTERM (cross-platform).
+ """Two-stage Ctrl-C / SIGTERM handler.
+
+ First signal: emit a "shutting down" notice and cancel the task
+ group so each ``_spawn_once`` unwinds via :meth:`_stop_proc`
+ (SIGTERM -> grace -> tree-kill).
+
+ Second signal: skip the grace period entirely -- iterate the
+ ``self._procs`` registry and tree-kill every still-running PID
+ synchronously. This matches the typical "Ctrl-C, then Ctrl-C
+ again to force" UX from systemd / VS Code / docker compose.
- Windows can only deliver ``SIGINT`` and ``SIGBREAK`` to a Python
- process; passing ``SIGTERM`` would raise ``NotImplementedError``
- from anyio. We probe the platform and subscribe to whichever
- subset is available.
+ Windows only delivers ``SIGINT`` / ``SIGBREAK`` to a Python
+ process; passing ``SIGTERM`` raises ``NotImplementedError``
+ from anyio, so we subscribe to whichever subset is available.
"""
if sys.platform == "win32":
signals = (signal.SIGINT, getattr(signal, "SIGBREAK", signal.SIGINT))
@@ -151,14 +153,28 @@ class Manager:
try:
with anyio.open_signal_receiver(*signals) as receiver:
async for sig in receiver:
- emit(
- "manager",
- "yellow",
- f"received {sig.name}, shutting down",
- stream="stderr",
- )
- cancel_scope.cancel()
- return
+ if not self._shutting_down:
+ self._shutting_down = True
+ emit(
+ "manager",
+ "yellow",
+ f"received {sig.name} -- shutting down "
+ "(Ctrl-C again to force-kill)",
+ stream="stderr",
+ )
+ cancel_scope.cancel()
+ else:
+ emit(
+ "manager",
+ "red",
+ f"received {sig.name} again -- force-killing",
+ stream="stderr",
+ )
+ for name, (proc, color) in list(self._procs.items()):
+ if proc.returncode is None:
+ kill_tree(proc.pid)
+ emit(name, color, "force-killed", stream="stderr")
+ return
except NotImplementedError:
# Some environments (eg. asyncio on Windows without
# ProactorEventLoop, or signal handling from a non-main
@@ -167,6 +183,46 @@ class Manager:
# the cancel scope when they finish.
await anyio.sleep_forever()
+ async def _stop_proc(self, proc: anyio.abc.Process, spec: ServiceSpec) -> None:
+ """SIGTERM (POSIX) / TerminateProcess (Win) -> wait grace -> tree-kill.
+
+ Wrapped in ``CancelScope(shield=True)`` -- this is the anyio-
+ documented idiom for cleanup that must run to completion during
+ an outer cancellation. Without the shield, ``_stop_proc`` is
+ invoked from inside an already-cancelled scope (the supervisor
+ signal handler cancelled the task group), so the very first
+ ``await proc.wait()`` re-raises ``CancelledError`` and the grace
+ period never elapses.
+ """
+ if proc.returncode is not None:
+ return
+ color = spec.color or "white"
+ emit(spec.name, color, "stopping", stream="stderr")
+ with anyio.CancelScope(shield=True):
+ try:
+ proc.terminate()
+ except ProcessLookupError:
+ return
+ with anyio.move_on_after(spec.terminate_grace_seconds):
+ await proc.wait()
+ if proc.returncode is None:
+ emit(
+ spec.name,
+ "red",
+ f"did not exit in {spec.terminate_grace_seconds:.0f}s, "
+ "force-killing process tree",
+ stream="stderr",
+ )
+ kill_tree(proc.pid)
+ with anyio.move_on_after(2.0):
+ await proc.wait()
+ emit(
+ spec.name,
+ color,
+ f"stopped (exit {proc.returncode})",
+ stream="stderr",
+ )
+
async def _supervise(
self, spec: ServiceSpec, cancel_scope: anyio.CancelScope
) -> None:
@@ -212,32 +268,45 @@ class Manager:
backoff_seconds = min(backoff_seconds * 2, 30.0)
async def _spawn_once(self, spec: ServiceSpec) -> int:
- """Spawn the service, drain its output, return exit code."""
+ """Spawn the service, drain its output, return exit code.
+
+ ``async with`` on the ``anyio.Process`` runs ``aclose`` on exit,
+ which closes stdin/stdout/stderr and awaits the child. Without
+ this, asyncio's Windows ProactorEventLoop leaves the underlying
+ ``BaseSubprocessTransport`` un-closed and ``__del__`` later logs
+ ``ResourceWarning: unclosed transport`` / ``ValueError: I/O
+ operation on closed pipe`` at GC time.
+ """
env = _full_env(spec.env)
spawn_kwargs = new_session_kwargs()
- proc = await anyio.open_process(
- spec.argv,
+ async with await anyio.open_process(
+ which_argv(spec.argv),
cwd=str(spec.cwd) if spec.cwd else None,
env=env,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
**spawn_kwargs,
- )
- if sys.platform == "win32":
- add_to_job(proc.pid)
- emit(spec.name, spec.color or "white", f"started pid={proc.pid}")
-
- try:
- async with anyio.create_task_group() as tg:
- tg.start_soon(self._drain, proc.stdout, spec, "stdout")
- tg.start_soon(self._drain, proc.stderr, spec, "stderr")
- if spec.ready_port:
- tg.start_soon(self._announce_ready, spec)
- exit_code = await proc.wait()
- tg.cancel_scope.cancel()
- except anyio.get_cancelled_exc_class():
- await _terminate_then_kill(proc, grace=spec.terminate_grace_seconds)
- raise
+ ) as proc:
+ if sys.platform == "win32":
+ add_to_job(proc.pid)
+ color = spec.color or "white"
+ emit(spec.name, color, f"started pid={proc.pid}")
+ # Register for force-kill access on second Ctrl-C; deregister on
+ # exit so a restarted-then-died service doesn't leave a stale entry.
+ self._procs[spec.name] = (proc, color)
+ try:
+ async with anyio.create_task_group() as tg:
+ tg.start_soon(self._drain, proc.stdout, spec, "stdout")
+ tg.start_soon(self._drain, proc.stderr, spec, "stderr")
+ if spec.ready_port:
+ tg.start_soon(self._announce_ready, spec)
+ exit_code = await proc.wait()
+ tg.cancel_scope.cancel()
+ except anyio.get_cancelled_exc_class():
+ await self._stop_proc(proc, spec)
+ raise
+ finally:
+ self._procs.pop(spec.name, None)
return exit_code
async def _drain(