Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
"""
ExtensionProcess: manages a generator running in an isolated subprocess.
Each extension runs in its own venv via runner.py.
Communication is done via newline-delimited JSON on stdin/stdout.
Interface is intentionally compatible with direct BaseGenerator usage
so GeneratorRegistry can treat both transparently.
"""
import base64
import json
import os
import platform
import queue
import re
import subprocess
import sys
import threading
import uuid
from pathlib import Path
from typing import Callable, Optional
_RUNNER_PATH = Path(__file__).parent.parent / "runner.py"
_MISSING_MODULE_RE = re.compile(r"No module named ['\"]([^'\"]+)['\"]")
_AUTO_REPAIR_PACKAGE_MAP = {
"PIL": "Pillow",
}
def _venv_python(ext_dir: Path) -> Path:
"""Returns the path to the venv's Python executable."""
if platform.system() == "Windows":
return ext_dir / "venv" / "Scripts" / "python.exe"
return ext_dir / "venv" / "bin" / "python"
class ExtensionProcess:
"""
Wraps an extension subprocess. Presents the same interface as a
direct generator (load / unload / generate / is_loaded / params_schema).
"""
def __init__(self, ext_dir: Path, manifest: dict) -> None:
self.ext_dir = ext_dir
self.manifest = manifest
self.model_dir = None # set by registry after init
self.outputs_dir = None # set by registry after init
self.shared_model_dirs: dict[str, Path] = {}
self._proc: Optional[subprocess.Popen] = None
self._queue: queue.Queue = queue.Queue()
self._lock: threading.Lock = threading.Lock()
self._loaded: bool = False
# Mirrors BaseGenerator attributes used by the registry
self.hf_repo = manifest.get("hf_repo", "")
self.hf_skip_prefixes = manifest.get("hf_skip_prefixes", [])
self.download_check = manifest.get("download_check", "")
self._params_schema = manifest.get("params_schema", [])
# Public metadata
self.MODEL_ID = manifest.get("id", "")
self.DISPLAY_NAME = manifest.get("name", "")
self.VRAM_GB = manifest.get("vram_gb", 0)
# ------------------------------------------------------------------ #
# Subprocess lifecycle
# ------------------------------------------------------------------ #
def _build_env(self) -> dict:
from services.generator_registry import MODELS_DIR, WORKSPACE_DIR
env = os.environ.copy()
env["EXTENSION_DIR"] = str(self.ext_dir)
env["MODELS_DIR"] = str(MODELS_DIR)
env["WORKSPACE_DIR"] = str(WORKSPACE_DIR)
env["THREEWS_API_DIR"] = str(Path(__file__).parent.parent)
# Lets extensions call back into the three.ws Forge API (e.g. /llm/chat for the shared LLM).
env.setdefault("THREEWS_API_URL", "http://127.0.0.1:8765")
# Extensions written against the upstream Modly template read these names.
env["MODLY_API_DIR"] = env["THREEWS_API_DIR"]
env.setdefault("MODLY_API_URL", env["THREEWS_API_URL"])
# Force the worker's Python stdio to UTF-8 so it matches the UTF-8
# pipe readers below regardless of the OS locale (cp1252/cp932).
env["PYTHONUTF8"] = "1"
env["PYTHONIOENCODING"] = "utf-8"
if sys.platform == "darwin":
env.setdefault("NUMBA_DISABLE_JIT", "1")
# Must be set before the subprocess's first `import torch`: PyTorch
# reads this once to decide whether MPS ops with no Metal kernel
# (e.g. 3D pooling) fall back to CPU or raise NotImplementedError.
# Setting it inside generator.py is too late, since generator.py
# itself imports torch before calling select_device().
env.setdefault("PYTORCH_ENABLE_MPS_FALLBACK", "1")
# Keep capability identity separate from storage identity. MODEL_DIR
# retains its node-private meaning; shared roots are passed explicitly.
if self.model_dir is not None:
env["MODEL_DIR"] = str(self.model_dir)
env["MODEL_ID"] = self.MODEL_ID
env["MODEL_NODE_ID"] = self.manifest.get(
"node_id", self.MODEL_ID.split("/", 1)[-1]
)
env["SHARED_MODEL_DIRS"] = json.dumps(
{group_id: str(path) for group_id, path in self.shared_model_dirs.items()}
)
# Extension venvs are based on python-embed which ships without a CA bundle.
# Only set SSL_CERT_FILE if not already provided (preserves corporate/custom certs).
if "SSL_CERT_FILE" not in env:
try:
import certifi
env["SSL_CERT_FILE"] = certifi.where()
except ImportError:
pass
return env
def _start(self) -> None:
"""Launch the subprocess and wait for the 'ready' signal."""
python = _venv_python(self.ext_dir)
if not python.exists():
raise RuntimeError(
f"[{self.MODEL_ID}] venv not found at {python}. "
"Run the extension's setup.py first."
)
for attempt in range(3):
# Use a fresh queue per subprocess lifetime so late messages from an
# older reader thread cannot poison startup for the new process.
run_queue: queue.Queue = queue.Queue()
self._queue = run_queue
self._proc = subprocess.Popen(
[str(python), str(_RUNNER_PATH)],
stdin=subprocess.PIPE,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
# Without these, text=True decodes with the locale codec: cp1252
# on Windows. tqdm draws its partial blocks with U+258D/U+258F,
# whose UTF-8 bytes (0x8d/0x8f) are undefined there: the decode
# raises, _stderr_loop dies, nobody drains the pipe, and the
# child blocks forever on write once it fills. errors="replace"
# keeps a stray non-UTF-8 byte from resurrecting that failure.
encoding="utf-8",
errors="replace",
bufsize=1,
env=self._build_env(),
)
# Background thread: read stdout → queue
reader = threading.Thread(target=self._read_loop, args=(self._proc, run_queue), daemon=True)
reader.start()
# Background thread: forward stderr to our stderr
stderr_fwd = threading.Thread(target=self._stderr_loop, args=(self._proc,), daemon=True)
stderr_fwd.start()
# Wait for ready: runner sends params_schema in this message
msg = self._recv(timeout=None)
if msg.get("type") == "ready":
# Override params_schema with what the generator class actually declares
if msg.get("params_schema"):
self._params_schema = msg["params_schema"]
print(f"[ExtensionProcess] {self.MODEL_ID} subprocess started (pid {self._proc.pid})")
return
self._proc.kill()
self._proc.wait()
missing_module = self._extract_missing_module(msg)
package_name = self._resolve_auto_repair_package(missing_module) if missing_module else None
if package_name and attempt < 2:
self._install_missing_package(python, missing_module, package_name)
continue
raise RuntimeError(f"[{self.MODEL_ID}] Expected 'ready', got: {msg}")
def _extract_missing_module(self, msg: dict) -> Optional[str]:
"""Returns missing import name from a runner error payload, if present."""
blob = f"{msg.get('message', '')}\n{msg.get('traceback', '')}"
match = _MISSING_MODULE_RE.search(blob)
return match.group(1) if match else None
def _resolve_auto_repair_package(self, module_name: str) -> Optional[str]:
"""
Maps a missing import name to a pip package for safe auto-repair.
Important: do not guess package names for arbitrary missing modules,
because that can install wrong packages and break environments.
"""
if module_name in _AUTO_REPAIR_PACKAGE_MAP:
return _AUTO_REPAIR_PACKAGE_MAP[module_name]
root = module_name.split(".")[0]
return _AUTO_REPAIR_PACKAGE_MAP.get(root)
def _install_missing_package(self, python: Path, module_name: str, package_name: str) -> None:
"""Best-effort auto-repair for a known missing import in extension venv."""
print(
f"[ExtensionProcess] {self.MODEL_ID} missing module '{module_name}' "
f"-> installing '{package_name}'"
)
try:
subprocess.run(
[str(python), "-m", "pip", "install", package_name],
check=True,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
text=True,
encoding="utf-8",
errors="replace",
)
except subprocess.CalledProcessError as exc:
details = (exc.stderr or exc.stdout or "").strip()
raise RuntimeError(
f"[{self.MODEL_ID}] Auto-repair failed while installing '{package_name}' "
f"for missing module '{module_name}'.\n{details[-2000:]}"
) from exc
def _read_loop(self, proc: subprocess.Popen, msg_queue: queue.Queue) -> None:
"""Continuously reads stdout and pushes parsed JSON to the queue."""
try:
for line in proc.stdout:
line = line.strip()
if line:
try:
msg_queue.put(json.loads(line))
except json.JSONDecodeError:
print(f"[{self.MODEL_ID}] {line}", file=sys.stderr)
finally:
msg_queue.put(None) # sentinel: process is done
def _stderr_loop(self, proc: subprocess.Popen) -> None:
"""Forward subprocess stderr to the main process stderr, emitting
one line every time we see EITHER '\\n' or '\\r'. tqdm writes live
progress updates with '\\r' only, so a newline-only iterator would
buffer every tick until the loop exits with '\\n', which is why
the HUD's log pane went dark during multi-minute volume decode.
No per-line extension-id prefix: the HUD log pane is a single
truncated line, and eating 20 characters with "[forge-hy3d2-mac] "
hides the tail of the tqdm bar the user actually wants to read.
"""
stream = proc.stderr
if stream is None:
return
buf = []
while True:
ch = stream.read(1)
if not ch:
if buf:
print(''.join(buf), file=sys.stderr, flush=True)
return
if ch in ("\r", "\n"):
if buf:
print(''.join(buf), file=sys.stderr, flush=True)
buf = []
else:
buf.append(ch)
def _send(self, msg: dict) -> None:
with self._lock:
self._proc.stdin.write(json.dumps(msg) + "\n")
self._proc.stdin.flush()
def _recv(self, timeout: float | None = 120.0) -> dict:
try:
msg = self._queue.get(timeout=timeout)
except queue.Empty:
raise TimeoutError(f"[{self.MODEL_ID}] No response from subprocess after {timeout}s")
if msg is None:
raise RuntimeError(f"[{self.MODEL_ID}] Subprocess died unexpectedly")
return msg
def _ensure_started(self) -> None:
if self._proc is None or self._proc.poll() is not None:
self._start()
# ------------------------------------------------------------------ #
# BaseGenerator-compatible interface
# ------------------------------------------------------------------ #
def is_downloaded(self) -> bool:
if self.download_check:
return (self.model_dir / self.download_check).exists()
return self.model_dir.exists() and any(self.model_dir.iterdir())
def is_loaded(self) -> bool:
return self._loaded and self._proc is not None and self._proc.poll() is None
def load(self) -> None:
self._ensure_started()
self._send({"action": "load"})
msg = self._recv(timeout=None) # model load can be arbitrarily slow
if msg.get('type') in ['loaded', 'ready']:
self._loaded = True
elif msg.get("type") == "error":
raise RuntimeError(msg.get("traceback") or msg.get("message"))
else:
raise RuntimeError(f"[{self.MODEL_ID}] Unexpected response to load: {msg}")
def unload(self) -> None:
if self._proc and self._proc.poll() is None:
try:
self._send({"action": "unload"})
self._recv(timeout=30.0)
except Exception:
pass
self._loaded = False
def generate(
self,
image_bytes: bytes,
params: dict,
progress_cb: Optional[Callable[[int, str], None]] = None,
cancel_event: Optional[threading.Event] = None,
) -> Path:
return self._generate_request(
{"image_b64": base64.b64encode(image_bytes).decode()},
params, progress_cb, cancel_event,
)
def _receive_generation(
self,
req_id: str,
progress_cb: Optional[Callable[[int, str], None]],
cancel_event: Optional[threading.Event],
) -> Path:
from services.generators.base import GenerationCancelled
# Grace period after sending a cooperative cancel before hard-killing
# the subprocess. Long enough to let generators that check cancel_event
# between steps shut down cleanly, short enough that the user isn't
# left staring at a stuck UI when the subprocess is blocked inside a
# native call (octree decode, marching cubes, etc.) that ignores stdin.
CANCEL_GRACE_SECONDS = 3.0
cancel_sent_at: Optional[float] = None
while True:
# Check for cancellation
if cancel_event and cancel_event.is_set():
if cancel_sent_at is None:
# First observation of the cancel: ask the subprocess to stop.
try:
self._send({"action": "cancel", "id": req_id})
except Exception:
pass
import time
cancel_sent_at = time.monotonic()
else:
import time
if time.monotonic() - cancel_sent_at >= CANCEL_GRACE_SECONDS:
# Grace period expired: the subprocess is not
# responding (almost certainly stuck in native code).
# Hard-kill it and drop our state so the next
# generation forces a fresh load.
try:
if self._proc and self._proc.poll() is None:
self._proc.kill()
self._proc.wait(timeout=5.0)
except Exception:
pass
self._loaded = False
self._proc = None
print(
f"[ExtensionProcess] {self.MODEL_ID} subprocess killed "
f"after {CANCEL_GRACE_SECONDS}s grace; model will reload on next run",
file=sys.stderr,
)
raise GenerationCancelled()
# Poll queue with short timeout so we can re-check cancel_event
try:
msg = self._queue.get(timeout=0.5)
except queue.Empty:
continue
if msg is None:
raise RuntimeError(f"[{self.MODEL_ID}] Subprocess died during generation")
t = msg.get("type")
if t == "progress":
if progress_cb:
progress_cb(msg.get("pct", 0), msg.get("step", ""))
elif t == "done":
return Path(msg["output_path"])
elif t == "error":
# A failed generation can leave the worker without a model: a
# lazy texture-setup failure frees the shape pipeline and then
# raises. The worker reports its post-failure state, so drop
# our cached flag and let GeneratorRegistry.get_active() reload
# before the next run rather than reusing a broken worker.
if msg.get("loaded") is False:
self._loaded = False
raise RuntimeError(msg.get("traceback") or msg.get("message", "Unknown error"))
elif t == "cancelled":
raise GenerationCancelled()
elif t == "log":
print(f"[{self.MODEL_ID}] {msg.get('message', '')}", file=sys.stderr)
def generate_artifact(
self,
input_kind: str,
artifact_path: Path,
params: dict,
progress_cb: Optional[Callable[[int, str], None]] = None,
cancel_event: Optional[threading.Event] = None,
) -> Path:
"""Send a typed artifact envelope to the isolated runner."""
from services.artifact_input import TypedArtifactInput, revalidate_artifact_input
from services.generator_registry import WORKSPACE_DIR
validated = revalidate_artifact_input(
WORKSPACE_DIR, TypedArtifactInput(kind=input_kind, path=artifact_path)
)
return self._generate_request(
{"input": {"kind": validated.kind, "path": str(validated.path)}},
params, progress_cb, cancel_event,
)
def _generate_request(
self,
input_payload: dict,
params: dict,
progress_cb: Optional[Callable[[int, str], None]],
cancel_event: Optional[threading.Event],
) -> Path:
req_id = str(uuid.uuid4())
self._send({
"action": "generate", "id": req_id, "model_id": self.MODEL_ID,
**input_payload, "params": params,
"outputs_dir": str(self.outputs_dir) if self.outputs_dir else None,
})
return self._receive_generation(req_id, progress_cb, cancel_event)
def params_schema(self) -> list:
return self._params_schema
def stop(self) -> None:
"""Hard-stop the subprocess.
Used by Free Memory / unload_all. Cooperative shutdown was the wrong
semantics here: torch.mps.empty_cache() does not reliably release
wired Metal pages, so only process exit actually returns the memory
to the OS. We SIGKILL, reap the zombie, and drop our refs so the
next load() starts a fresh subprocess.
"""
proc = self._proc
self._loaded = False
if proc and proc.poll() is None:
try:
proc.kill()
proc.wait(timeout=5)
except Exception as exc:
# Keep the process reference so callers can verify that the
# worker may still be alive and refuse to mutate its venv.
self._proc = proc
raise RuntimeError(
f"[{self.MODEL_ID}] Could not stop extension subprocess"
) from exc
if proc.poll() is None:
self._proc = proc
raise RuntimeError(
f"[{self.MODEL_ID}] Extension subprocess is still running"
)
self._proc = None
self._drain_queue()
def _drain_queue(self) -> None:
while not self._queue.empty():
try:
self._queue.get_nowait()
except queue.Empty:
break