Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

13.3 KB
Raw Blame
"""
three.ws Forge Extension Runner, generic subprocess entry point.

Runs inside the extension's own venv. Loaded by ExtensionProcess via:
    {venv_python} {runner_path}

Environment variables (set by ExtensionProcess):
    EXTENSION_DIR   absolute path to the extension directory
    MODELS_DIR      where model weights are stored
    WORKSPACE_DIR   where generated files are saved
    THREEWS_API_DIR   path to three.ws Forge's api/ dir (so generator.py can import
                      from services.generators.base)

Protocol: newline-delimited JSON on stdin/stdout.
Stderr is captured separately by ExtensionProcess for logging.
"""
import sys
import json
import os
import traceback
import base64
import threading
import importlib.util
from pathlib import Path

# ------------------------------------------------------------------ #
# Env
# ------------------------------------------------------------------ #

EXT_DIR       = Path(os.environ["EXTENSION_DIR"])
MODELS_DIR    = Path(os.environ.get("MODELS_DIR",    Path.home() / ".three-ws-forge" / "models"))
WORKSPACE_DIR = Path(os.environ.get("WORKSPACE_DIR", Path.home() / ".three-ws-forge" / "workspace"))
THREEWS_API_DIR = os.environ.get("THREEWS_API_DIR") or os.environ.get("MODLY_API_DIR", "")
# MODEL_DIR is set by ExtensionProcess to match its own model_dir (composite node id path).
# Falls back to MODELS_DIR/manifest_id for standalone/legacy use.
_MODEL_DIR_OVERRIDE = os.environ.get("MODEL_DIR", "")
_MODEL_ID_OVERRIDE = os.environ.get("MODEL_ID", "")
_MODEL_NODE_ID_OVERRIDE = os.environ.get("MODEL_NODE_ID", "")
try:
    _SHARED_MODEL_DIRS = {
        str(group_id): Path(path)
        for group_id, path in json.loads(os.environ.get("SHARED_MODEL_DIRS", "{}")).items()
    }
except (AttributeError, TypeError, ValueError, json.JSONDecodeError):
    _SHARED_MODEL_DIRS = {}

# Inject three.ws Forge's api/ so generator.py can do:
#   from services.generators.base import BaseGenerator, ...
if THREEWS_API_DIR and THREEWS_API_DIR not in sys.path:
    sys.path.insert(0, THREEWS_API_DIR)

# Inject ext dir so generator.py can import local vendor modules
if str(EXT_DIR) not in sys.path:
    sys.path.insert(0, str(EXT_DIR))

# Match the UTF-8 pipe readers on the parent side even when the OS locale
# is a legacy codepage (cp1252/cp932 on Windows). Without this, any Unicode
# print() from a generator (e.g. tqdm output) crashes the worker.
from services.stdio_utf8 import ensure_utf8_stdio
ensure_utf8_stdio()


# ------------------------------------------------------------------ #
# Protocol helpers
# ------------------------------------------------------------------ #

def send(msg: dict) -> None:
    sys.stdout.write(json.dumps(msg) + "\n")
    sys.stdout.flush()


def recv():
    """Yields parsed JSON messages from stdin, one per line."""
    for raw in sys.stdin:
        raw = raw.strip()
        if raw:
            try:
                yield json.loads(raw)
            except json.JSONDecodeError as exc:
                send({"type": "log", "level": "error",
                      "message": f"Runner: invalid JSON on stdin: {exc}"})


# ------------------------------------------------------------------ #
# Generator loader
# ------------------------------------------------------------------ #

def load_generator(manifest: dict):
    """Dynamically load the generator class from generator.py."""
    spec = importlib.util.spec_from_file_location(
        "generator", EXT_DIR / "generator.py"
    )
    mod = importlib.util.module_from_spec(spec)
    sys.modules["generator"] = mod
    spec.loader.exec_module(mod)
    return getattr(mod, manifest["generator_class"])


def _select_node(
    manifest: dict, model_dir_override: str, node_id_override: str = ""
) -> dict:
    nodes = manifest.get("nodes") or []
    if nodes and node_id_override:
        return next((n for n in nodes if n.get("id") == node_id_override), nodes[0])
    if nodes and model_dir_override:
        node_id = Path(model_dir_override).name
        return next((n for n in nodes if n.get("id") == node_id), nodes[0])
    if nodes:
        return nodes[0]
    return {}


def _resolve_ready_schema(GenClass, node: dict, manifest: dict) -> list:
    try:
        return GenClass.params_schema()
    except Exception:
        return node.get("params_schema") or manifest.get("params_schema", [])


def _generator_is_loaded(gen) -> bool:
    """
    Best-effort read of the generator's loaded state.

    Never raises: this is used on the error path, where a generator left in a
    half-initialised state can make is_loaded() itself blow up. An unreadable
    state is reported as "not loaded" so the caller reloads rather than reusing
    a worker that may be broken.
    """
    try:
        return bool(gen.is_loaded())
    except Exception:
        return False


def _ensure_model_loaded(gen) -> bool:
    """
    Guarantees the model is in memory before inference. Returns True if a
    reload was needed.

    Texture pipelines are built lazily on first use, and extensions typically
    free the shape pipeline first to make room for them. When that setup fails
    (a missing `xatlas` being the common case) the generator is left with
    _model = None, yet the worker process stays alive and both ExtensionProcess
    and GeneratorRegistry still consider it loaded, so load() is never called
    again and every later generate() dies with

        TypeError: 'NoneType' object is not callable

    until the process is killed by hand. Re-loading here mirrors what
    GeneratorRegistry.get_active() already does host-side, which makes a failed
    setup recoverable on the next attempt instead of permanently poisoning the
    worker.
    """
    if _generator_is_loaded(gen):
        return False
    gen.load()
    return True


def _apply_manifest_metadata(gen, manifest: dict, node: dict) -> None:
    gen.hf_repo = node.get("hf_repo") or manifest.get("hf_repo", "")
    gen.hf_skip_prefixes = node.get("hf_skip_prefixes") or manifest.get("hf_skip_prefixes", [])
    gen.download_check = node.get("download_check") or manifest.get("download_check", "")
    gen._params_schema = node.get("params_schema") or manifest.get("params_schema", [])


def decode_model_input(msg: dict):
    """Decode legacy image bytes or revalidate a typed artifact in the worker."""
    if "input" not in msg:
        return base64.b64decode(msg["image_b64"])
    value = msg["input"]
    if not isinstance(value, dict) or set(value) != {"kind", "path"}:
        raise ValueError("Typed artifact input must contain exactly kind and path")
    from services.artifact_input import TypedArtifactInput, revalidate_artifact_input
    typed = TypedArtifactInput(kind=value.get("kind"), path=Path(value.get("path", "")))
    return revalidate_artifact_input(WORKSPACE_DIR, typed)


def validate_requested_model(msg: dict, manifest: dict, node: dict) -> None:
    """Reject requests routed to a worker for a different manifest node."""
    requested = msg.get("model_id")
    if requested is None:  # Backward compatibility with already-running legacy hosts.
        return
    expected = manifest["id"]
    if node.get("id"):
        expected = f"{expected}/{node['id']}"
    if requested != expected:
        raise ValueError(
            f"Generation request model '{requested}' does not match worker model '{expected}'"
        )


# ------------------------------------------------------------------ #
# Main loop
# ------------------------------------------------------------------ #

def main() -> None:
    manifest = json.loads((EXT_DIR / "manifest.json").read_text(encoding="utf-8"))
    model_id = _MODEL_ID_OVERRIDE or manifest["id"]

    try:
        GenClass = load_generator(manifest)
    except Exception:
        send({"type": "error", "id": None,
              "message": "Failed to load generator class",
              "traceback": traceback.format_exc()})
        return

    # Support both flat manifest (legacy) and nodes[] format. The host passes an
    # explicit node id; MODEL_DIR name inference remains only as a legacy fallback.
    node = _select_node(manifest, _MODEL_DIR_OVERRIDE, _MODEL_NODE_ID_OVERRIDE)

    # Announce readiness and send params_schema so ExtensionProcess
    # can serve it without needing to query the subprocess later.
    # We try to get it from the generator class (may be a classmethod),
    # falling back to the selected node, then to the top-level manifest.
    send({"type": "ready", "params_schema": _resolve_ready_schema(GenClass, node, manifest)})

    # Use MODEL_DIR env var (set by ExtensionProcess) when available so the
    # generator uses the exact same path that is_downloaded() checks against.
    # Falls back to MODELS_DIR/manifest_id for legacy / standalone use.
    model_dir = Path(_MODEL_DIR_OVERRIDE) if _MODEL_DIR_OVERRIDE else MODELS_DIR / model_id
    gen = GenClass(model_dir, WORKSPACE_DIR)
    gen.MODEL_ID = model_id
    gen.MODEL_NODE_ID = node.get("id", "")
    gen.shared_model_dirs = dict(_SHARED_MODEL_DIRS)
    _apply_manifest_metadata(gen, manifest, node)

    # Active cancel events keyed by request id
    _cancel: dict[str, threading.Event] = {}

    for msg in recv():
        action = msg.get("action")
        rid    = msg.get("id")

        try:
            # ---- load ------------------------------------------------
            if action == "load":
                gen.load()
                send({"type": "loaded"})

            # ---- generate --------------------------------------------
            elif action == "generate":
                validate_requested_model(msg, manifest, node)
                cancel_evt = threading.Event()
                _cancel[rid] = cancel_evt
                model_input  = decode_model_input(msg)
                params       = msg.get("params", {})
                if hasattr(model_input, "kind"):
                    if not isinstance(params, dict):
                        raise ValueError("Model params must be an object")
                    from services.artifact_input import RESERVED_ARTIFACT_PARAMS
                    params = {key: value for key, value in params.items()
                              if key not in RESERVED_ARTIFACT_PARAMS}
                    params["scene_manifest_path"] = str(model_input.path)
                if msg.get("outputs_dir"):
                    gen.outputs_dir = Path(msg["outputs_dir"])
                    gen.outputs_dir.mkdir(parents=True, exist_ok=True)

                def progress_cb(pct: int, step: str = "") -> None:
                    send({"type": "progress", "id": rid, "pct": pct, "step": step})

                try:
                    if _ensure_model_loaded(gen):
                        send({"type": "log", "level": "warning",
                              "message": ("Model was not loaded (earlier setup failure?); "
                                          "reloaded before generating.")})
                    if hasattr(model_input, "kind"):
                        output_path = gen.generate_artifact(
                            model_input.kind, model_input.path, params, progress_cb, cancel_evt
                        )
                    else:
                        output_path = gen.generate(model_input, params, progress_cb, cancel_evt)
                    if node.get("output") == "scene":
                        from services.scene_input import validate_scene_input
                        resolved_output = Path(output_path).resolve(strict=True)
                        try:
                            relative_output = resolved_output.relative_to(WORKSPACE_DIR.resolve(strict=True))
                        except (OSError, ValueError) as exc:
                            raise ValueError("Generated scene output is outside the workspace") from exc
                        output_path = validate_scene_input(WORKSPACE_DIR, relative_output.as_posix())
                    send({"type": "done", "id": rid, "output_path": str(output_path)})
                except Exception as exc:
                    # Detect GenerationCancelled by name to avoid import issues
                    if type(exc).__name__ == "GenerationCancelled":
                        send({"type": "cancelled", "id": rid})
                    else:
                        # Report whether the model survived the failure so the
                        # host can drop its cached "loaded" flag and reload
                        # instead of reusing a worker whose _model is None.
                        send({"type": "error", "id": rid,
                              "message": str(exc),
                              "traceback": traceback.format_exc(),
                              "loaded": _generator_is_loaded(gen)})
                finally:
                    _cancel.pop(rid, None)

            # ---- cancel ----------------------------------------------
            elif action == "cancel":
                evt = _cancel.get(rid)
                if evt:
                    evt.set()

            # ---- unload ----------------------------------------------
            elif action == "unload":
                gen.unload()
                send({"type": "unloaded"})

            # ---- shutdown --------------------------------------------
            elif action == "shutdown":
                gen.unload()
                break

        except Exception:
            send({"type": "error", "id": rid,
                  "message": "Unexpected runner error",
                  "traceback": traceback.format_exc()})


if __name__ == "__main__":
    main()