Skip to content

armgit / Soup

Public Imported from GitHub
0

Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.

53.7 KB
Raw Blame
"""FastAPI application for Soup Web UI."""

import contextlib
import ipaddress
import json as json_mod
import logging
import os
import secrets
import subprocess
import sys
import tempfile
import threading
import time
from collections import OrderedDict
from pathlib import Path
from types import MappingProxyType
from typing import Mapping, Optional
from urllib.parse import urlencode

from pydantic import BaseModel as PydanticBaseModel
from pydantic import Field

logger = logging.getLogger(__name__)

STATIC_DIR = Path(__file__).parent / "static"

# Max file read size to prevent memory exhaustion
_MAX_INSPECT_LIMIT = 500

# #1191: live session ids per app; the oldest is dropped past this.
_MAX_UI_SESSIONS = 64

# #939: cap the body before FastAPI parses it, sized per route (chat/send
# forwards upstream; inspect only ever needs a path and an int). #897 extends
# the same mechanism to every YAML-bearing Web UI route.
_MAX_CHAT_SEND_BODY_BYTES = 1024 * 1024
_MAX_DATA_INSPECT_BODY_BYTES = 8 * 1024
_MAX_YAML_REQUEST_BYTES = 1024 * 1024
_BODY_SIZE_LIMITS = {
    "/api/chat/send": _MAX_CHAT_SEND_BODY_BYTES,
    "/api/data/inspect": _MAX_DATA_INSPECT_BODY_BYTES,
    "/api/config/validate": _MAX_YAML_REQUEST_BYTES,
    "/api/train/start": _MAX_YAML_REQUEST_BYTES,
    "/api/config/from-form": _MAX_YAML_REQUEST_BYTES,
}


class _RequestBodySizeLimitMiddleware:
    """Reject an oversized POST body before route/model parsing runs.

    Checks ``Content-Length`` first, then streams and counts the actual body
    bytes so a missing or understated header cannot bypass the cap. Per-route
    limits keep the mechanism shared while allowing small JSON requests and
    YAML-bearing configuration requests to use different ceilings.
    """

    def __init__(self, app, limits: Mapping[str, int] = _BODY_SIZE_LIMITS) -> None:
        self._app = app
        self._limits = limits

    async def _reject(self, scope, receive, send) -> None:
        from starlette.responses import JSONResponse

        response = JSONResponse(
            status_code=413,
            content={"detail": "Request body too large"},
        )
        await response(scope, receive, send)

    async def __call__(self, scope, receive, send) -> None:
        max_body_size = None
        if scope.get("type") == "http" and scope.get("method") == "POST":
            max_body_size = self._limits.get(scope.get("path"))
        if max_body_size is None:
            await self._app(scope, receive, send)
            return

        for key, value in scope.get("headers", ()):
            if key.lower() != b"content-length":
                continue
            try:
                if int(value) > max_body_size:
                    await self._reject(scope, receive, send)
                    return
            except ValueError:
                # Do not trust a malformed header; the receive wrapper below
                # still enforces the actual byte count.
                pass

        received = 0
        buffered_messages = []
        while True:
            message = await receive()
            buffered_messages.append(message)
            if message.get("type") == "http.request":
                received += len(message.get("body", b""))
                if received > max_body_size:
                    await self._reject(scope, receive, send)
                    return
                if message.get("more_body", False):
                    continue
            break

        message_index = 0

        async def receive_buffered():
            nonlocal message_index
            if message_index < len(buffered_messages):
                message = buffered_messages[message_index]
                message_index += 1
                return message
            return await receive()

        await self._app(scope, receive_buffered, send)


_CHART_JS_URL = "https://cdn.jsdelivr.net/npm/[email protected]/dist/chart.umd.min.js"

CONTENT_SECURITY_POLICY = "; ".join((
    "default-src 'self'",
    f"script-src 'self' {_CHART_JS_URL}",
    # Rendered markup and Chart.js set inline style attributes.
    "style-src 'self' 'unsafe-inline'",
    "img-src 'self' data:",
    "connect-src 'self'",
    "object-src 'none'",
    "base-uri 'none'",
    "frame-ancestors 'none'",
    "form-action 'self'",
))

SECURITY_HEADERS: Mapping[str, str] = MappingProxyType({
    "Content-Security-Policy": CONTENT_SECURITY_POLICY,
    "X-Content-Type-Options": "nosniff",
    "Referrer-Policy": "no-referrer",
    "X-Frame-Options": "DENY",
})

# FastAPI's interactive docs (served on a loopback bind only) bootstrap with an
# inline script and CDN assets, so the page policy above would leave them blank.
_POLICY_EXEMPT_PATHS = frozenset({"/docs", "/docs/oauth2-redirect", "/redoc"})


class _SecurityHeadersMiddleware:
    """Add the response headers in ``SECURITY_HEADERS`` to every HTTP response.

    Pure ASGI: it rewrites only the ``http.response.start`` message and forwards
    every body message as it arrives, so streamed responses (the training log
    event stream) are not buffered. A header the endpoint already set is kept.
    """

    def __init__(
        self,
        app,
        headers: Mapping[str, str] = SECURITY_HEADERS,
        policy_exempt_paths: frozenset = frozenset(),
    ) -> None:
        self._app = app
        self._headers = tuple(
            (name.lower().encode("latin-1"), value.encode("latin-1"))
            for name, value in headers.items()
        )
        self._policy_exempt_paths = policy_exempt_paths

    async def __call__(self, scope, receive, send) -> None:
        if scope.get("type") != "http":
            await self._app(scope, receive, send)
            return

        exempt = scope.get("path") in self._policy_exempt_paths

        async def send_with_headers(message) -> None:
            if message.get("type") == "http.response.start":
                exempt_policy = exempt and message.get("status") == 200
                extra = tuple(
                    (name, value)
                    for name, value in self._headers
                    if not (exempt_policy and name == b"content-security-policy")
                )
                existing = list(message.get("headers", ()))
                present = {name.lower() for name, _ in existing}
                added = [(n, v) for n, v in extra if n not in present]
                message = {**message, "headers": existing + added}
            await send(message)

        await self._app(scope, receive, send_with_headers)


def _host_key(host: str) -> str:
    """Normalize IP literals and case-insensitive hostnames for comparison."""
    try:
        return str(ipaddress.ip_address(host))
    except ValueError:
        return host.lower()


def _host_header_allowed(value: str, allowed_hosts: frozenset[tuple[str, int]]) -> bool:
    """Match a single HTTP Host header against the served host and port pairs."""
    if not value:
        return False
    if value.startswith("["):
        host, sep, rest = value[1:].partition("]")
        if not sep:
            return False
        try:
            ipaddress.IPv6Address(host)
        except ValueError:
            return False
        if rest:
            if not rest.startswith(":"):
                return False
            port_s = rest[1:]
        else:
            port_s = "80"
    else:
        host, sep, port_s = value.rpartition(":")
        if not sep:
            host, port_s = value, "80"
        elif ":" in host:
            return False
    if not port_s.isascii() or not port_s.isdigit():
        return False
    # Strip leading zeroes before converting; an arbitrarily long port cannot
    # overflow Python's integer-string conversion limit.
    port_s = port_s.lstrip("0") or "0"
    if len(port_s) > 5:
        return False
    return (_host_key(host), int(port_s)) in allowed_hosts


class _HostAllowlistMiddleware:
    """Validate loopback HTTP hosts before routes, auth and body reads."""

    def __init__(self, app, allowed_hosts: frozenset[tuple[str, int]]) -> None:
        self._app = app
        self._allowed_hosts = allowed_hosts

    async def __call__(self, scope, receive, send) -> None:
        # The UI has no WebSocket routes; do not allow a future one to bypass
        # this HTTP-only gate. Lifespan messages still pass through.
        if scope.get("type") == "websocket":
            await send({"type": "websocket.close", "code": 1008})
            return
        if scope.get("type") != "http":
            await self._app(scope, receive, send)
            return
        hosts = [value for name, value in scope.get("headers", ()) if name.lower() == b"host"]
        if len(hosts) != 1 or not _host_header_allowed(
            hosts[0].decode("latin-1"), self._allowed_hosts
        ):
            from starlette.responses import JSONResponse

            response = JSONResponse({"detail": "Host not allowed"}, status_code=403)
            await response(scope, receive, send)
            return
        await self._app(scope, receive, send)


class TrainRequest(PydanticBaseModel):
    """Request body for starting a training run."""
    config_yaml: str


class TrainStatus(PydanticBaseModel):
    """Current training process status."""
    running: bool
    pid: Optional[int] = None
    config_path: Optional[str] = None


class DataInspectRequest(PydanticBaseModel):
    """Request body for data inspection."""
    path: str
    limit: int = Field(default=50, ge=1, le=_MAX_INSPECT_LIMIT)


class TrainLogBuffer:
    """Thread-safe bounded ring buffer for subprocess output lines."""

    def __init__(self, maxlen: int = 10000):
        self._maxlen = maxlen
        self._lines: list[tuple[int, str]] = []
        self._lock = threading.Lock()
        self._new_line_cond = threading.Condition(self._lock)
        self._done = False
        self._total_emitted = 0

    def append(self, text: str) -> None:
        with self._new_line_cond:
            idx = self._total_emitted
            self._total_emitted += 1
            self._lines.append((idx, text))
            if len(self._lines) > self._maxlen:
                self._lines.pop(0)
            self._new_line_cond.notify_all()

    def mark_done(self) -> None:
        with self._new_line_cond:
            self._done = True
            self._new_line_cond.notify_all()

    def is_done(self) -> bool:
        with self._lock:
            return self._done

    def get_lines_from(self, start_idx: int) -> list[tuple[int, str]]:
        with self._lock:
            return [item for item in self._lines if item[0] >= start_idx]

    def wait_for_lines_or_done(
        self, next_idx: int, timeout: float = 0.5
    ) -> tuple[list[tuple[int, str]], bool]:
        """Wait until lines >= next_idx are available or process is marked done."""
        with self._new_line_cond:
            available = [item for item in self._lines if item[0] >= next_idx]
            if available or self._done:
                return available, self._done
            self._new_line_cond.wait(timeout=timeout)
            available = [item for item in self._lines if item[0] >= next_idx]
            return available, self._done


def _remove_temp_config(path: Optional[str]) -> None:
    if not path:
        return
    try:
        os.unlink(path)
    except FileNotFoundError:
        pass
    except OSError as exc:
        logger.debug("Could not remove temp config %s: %s", path, exc)


def _wait_and_remove_temp_config(
    proc: Optional[subprocess.Popen], config_path: Optional[str]
) -> None:
    if not config_path:
        return
    try:
        wait_fn = getattr(proc, "wait", None)
        if callable(wait_fn):
            wait_fn()
    except Exception as exc:
        logger.debug("Waiting for process %s failed: %s", proc, exc)
        return
    _remove_temp_config(config_path)


def _drain_stdout_worker(
    proc: subprocess.Popen,
    log_buffer: TrainLogBuffer,
    config_path: Optional[str] = None,
) -> None:
    """Continuously read stdout of proc until EOF and write to log_buffer."""
    stdout = getattr(proc, "stdout", None)
    if stdout is None:
        log_buffer.mark_done()
        _wait_and_remove_temp_config(proc, config_path)
        return

    try:
        while True:
            raw_line = stdout.readline()
            if not raw_line or raw_line == b"":
                break
            if isinstance(raw_line, bytes):
                text = raw_line.decode("utf-8", errors="replace").rstrip("\n\r")
            elif isinstance(raw_line, str):
                text = raw_line.rstrip("\n\r")
            else:
                break
            log_buffer.append(text)
    except (ValueError, OSError) as exc:
        logger.debug("Drain worker exception: %s", exc)
    finally:
        try:
            stdout.close()
        except Exception:
            pass
        log_buffer.mark_done()
        _wait_and_remove_temp_config(proc, config_path)


def _resolve_train_argv(config_path: str) -> list[str]:
    """Construct command-line arguments for launching training subprocess."""
    return [sys.executable, "-m", "soup_cli", "train", "--config", config_path, "--yes"]


# Global state for training process
_train_process: Optional[subprocess.Popen] = None
_train_config_path: Optional[str] = None
_train_log_buffer: Optional[TrainLogBuffer] = None
_train_drain_thread: Optional[threading.Thread] = None
_train_lock = threading.Lock()

# Auth token generated at startup — printed to console for the user.
# Reads/writes go through `_auth_token_lock` so token rotation never
# leaves a window where some requests see the old value and some the new.
_auth_token: str = secrets.token_urlsafe(32)
_auth_token_lock = threading.Lock()


def get_auth_token() -> str:
    """Return the current auth token (for printing at startup)."""
    with _auth_token_lock:
        return _auth_token


def set_auth_token(token: str) -> None:
    """Replace the process-wide auth token (used by `soup ui --auth-token`).

    Validates via `utils.qr_url.validate_token` so a malformed override
    can't bypass the urlsafe-base64 shape check.
    """
    from soup_cli.utils.qr_url import validate_token

    validated = validate_token(token)
    global _auth_token
    with _auth_token_lock:
        _auth_token = validated


# Ephemeral single-use tickets for SSE endpoints (v0.74.x #687)
# Valid for 30 seconds, single use only.
_tickets: dict[str, float] = {}
_tickets_lock = threading.Lock()


def _cleanup_expired_tickets() -> None:
    now = time.time()
    expired = [t for t, exp in _tickets.items() if exp < now]
    for t in expired:
        _tickets.pop(t, None)


def create_auth_ticket() -> str:
    """Create a short-lived (30s) single-use ticket for SSE connection."""
    ticket = secrets.token_urlsafe(32)
    now = time.time()
    with _tickets_lock:
        _cleanup_expired_tickets()
        _tickets[ticket] = now + 30.0
    return ticket


def consume_auth_ticket(ticket: str) -> bool:
    """Consume a single-use ticket if valid and unexpired."""
    if not ticket:
        return False
    now = time.time()
    with _tickets_lock:
        _cleanup_expired_tickets()
        exp = _tickets.pop(ticket, None)
        if exp is not None and now <= exp:
            return True
    return False


def _bearer_matches(header_value: str) -> bool:
    """Verify Bearer token against configured token in constant time.

    Encodes both values to UTF-8 bytes before comparing, because
    secrets.compare_digest raises TypeError if passed non-ASCII str arguments
    (e.g. when Starlette decodes header bytes >= 0x80 using Latin-1).
    """
    with _auth_token_lock:
        expected = f"Bearer {_auth_token}".encode("utf-8")
    return secrets.compare_digest(header_value.encode("utf-8"), expected)


def _is_loopback(host: str) -> bool:
    """Return True if host is a loopback address or localhost."""
    if host.lower() == "localhost":
        return True
    try:
        return ipaddress.ip_address(host.strip("[]")).is_loopback
    except ValueError:
        return False


def create_app(host: str = "127.0.0.1", port: int = 7860):
    """Create the Soup Web UI FastAPI application."""
    if not _is_loopback(host):
        token = get_auth_token()
        valid = False
        if token:
            try:
                from soup_cli.utils.qr_url import validate_token

                validate_token(token)
                valid = True
            except (TypeError, ValueError):
                valid = False
        if not valid:
            raise ValueError(
                f"Binding non-loopback host '{host}' requires a valid authentication token."
            )

    from fastapi import Depends, FastAPI, HTTPException, Query, Request
    from fastapi.middleware.cors import CORSMiddleware
    from fastapi.responses import HTMLResponse, RedirectResponse
    from fastapi.staticfiles import StaticFiles

    # #731: FastAPI's interactive docs describe every route, parameter and
    # schema, and served none of it behind a token -- so a `soup ui --public`
    # bind let anyone on the LAN enumerate the whole API surface. Gating them
    # behind `_verify_token` does not work: `/docs` is a browser navigation and
    # Swagger cannot attach a Bearer header to it (the #687 constraint), so
    # gating would break the page for the developer while `/openapi.json` stayed
    # readable by curl. Passing `None` removes the routes outright -- there is
    # no handler left to reach -- and loopback keeps the convenience.
    _docs_enabled = _is_loopback(host)
    app = FastAPI(
        title="Soup Web UI",
        version="1.0.0",
        openapi_url="/openapi.json" if _docs_enabled else None,
        docs_url="/docs" if _docs_enabled else None,
        redoc_url="/redoc" if _docs_enabled else None,
        # Derived from `swagger_ui_oauth2_redirect_url`, not from `docs_url`:
        # leaving it at its default keeps `/docs/oauth2-redirect` serving even
        # once `/docs` is gone.
        swagger_ui_oauth2_redirect_url=(
            "/docs/oauth2-redirect" if _docs_enabled else None
        ),
    )

    # Install before CORS so a 413 from the size cap still carries the same
    # CORS headers as a normal endpoint response.
    app.add_middleware(_RequestBodySizeLimitMiddleware)

    # Restrict CORS to the origin we actually serve. When `host == "0.0.0.0"`
    # the literal `http://0.0.0.0:<port>` is never a browser origin, so we
    # allow loopback origins AND the same-LAN regex shape. The Bearer
    # token is the actual security gate on mutating endpoints.
    if host == "0.0.0.0":
        app.add_middleware(
            CORSMiddleware,
            allow_origin_regex=(
                r"^https?://("
                r"localhost|127\.0\.0\.1|"
                r"10\.\d+\.\d+\.\d+|"
                r"192\.168\.\d+\.\d+|"
                r"172\.(?:1[6-9]|2[0-9]|3[01])\.\d+\.\d+"
                r")(:\d+)?$"
            ),
            allow_methods=["GET", "POST", "DELETE"],
            allow_headers=["Authorization", "Content-Type"],
        )
    else:
        allowed_origin = f"http://{host}:{port}"
        app.add_middleware(
            CORSMiddleware,
            allow_origins=[allowed_origin],
            allow_methods=["GET", "POST", "DELETE"],
            allow_headers=["Authorization", "Content-Type"],
        )

    # On loopback binds, validate the bound name and aliases before routes,
    # authentication and body reads. Public binds keep the Bearer gate.
    if _is_loopback(host):
        allowed = frozenset(
            (_host_key(name.strip("[]")), port)
            for name in ("localhost", "127.0.0.1", "::1", host)
        )
        app.add_middleware(_HostAllowlistMiddleware, allowed_hosts=allowed)

    # Added last, so refusals, body-size errors and CORS preflights carry the
    # security headers as well as normal endpoint responses.
    app.add_middleware(
        _SecurityHeadersMiddleware,
        policy_exempt_paths=_POLICY_EXEMPT_PATHS if _docs_enabled else frozenset(),
    )

    # #1191: `/?token=` swaps the token for an HttpOnly session cookie, so a
    # reload keeps the session without page script ever reading a credential.
    # The ids live only in this app's memory: a restart ends every session.
    # Loopback binds only -- `--public` over LAN HTTP keeps the closure flow.
    # The port is in the name because cookies are not isolated by port.
    _session_cookie = f"soup_ui_session_{port}"
    _session_ids: "OrderedDict[str, None]" = OrderedDict()
    _session_lock = threading.Lock()
    _session_enabled = _is_loopback(host)

    def _new_session() -> str:
        session_id = secrets.token_urlsafe(32)
        with _session_lock:
            _session_ids[session_id] = None
            while len(_session_ids) > _MAX_UI_SESSIONS:
                _session_ids.popitem(last=False)
        return session_id

    def _session_matches(request: Request) -> bool:
        """True for a live session cookie; 403 on a cross-origin state change.

        SameSite=Strict does not help here: every port of 127.0.0.1 is one site,
        so a page on another local port gets the cookie attached. Browsers always
        send `Origin` on a fetch POST/DELETE, so anything but GET/HEAD must name
        exactly this server -- scheme, host and port. `check_local_request`
        cannot do this: it compares hostnames without ports on purpose.
        """
        # Defence in depth: `_new_session` only runs behind the same check in
        # `index()`, so a non-loopback app never holds an id to match.
        if not _session_enabled:
            return False
        value = request.cookies.get(_session_cookie, "").encode("utf-8")
        if not value:
            return False
        with _session_lock:
            live = list(_session_ids)
        # Constant-time against every live id, like `_bearer_matches`.
        matched = False
        for session_id in live:
            matched |= secrets.compare_digest(value, session_id.encode("utf-8"))
        if not matched:
            return False
        if request.method not in ("GET", "HEAD"):
            origin = request.headers.get("Origin", "").lower()
            expected = f"{request.url.scheme}://{request.headers.get('Host', '')}".lower()
            if not origin or origin != expected:
                raise HTTPException(status_code=403, detail="Origin not allowed")
        return True

    def _verify_token(request: Request):
        """Verify Bearer token (or the #1191 session cookie) on API endpoints."""
        auth = request.headers.get("Authorization", "")
        # Constant-time compare — a plain != leaks the token byte-by-byte via
        # response timing when `soup ui --public` is exposed on a LAN.
        if _bearer_matches(auth) or _session_matches(request):
            return
        raise HTTPException(status_code=401, detail="Unauthorized")

    def _verify_token_or_ticket(request: Request):
        """Verify Bearer token, session cookie, or a single-use SSE ticket."""
        auth = request.headers.get("Authorization", "")
        if auth and _bearer_matches(auth):
            return

        if _session_matches(request):
            return

        ticket = request.query_params.get("ticket", "")
        if ticket and consume_auth_ticket(ticket):
            return

        raise HTTPException(status_code=401, detail="Unauthorized")

    # --- Auth ticket exchange for SSE ---

    @app.post("/api/auth/ticket", dependencies=[Depends(_verify_token)])
    def issue_auth_ticket():
        """Exchange Bearer token for a short-lived (30s) single-use SSE ticket."""
        return {"ticket": create_auth_ticket()}

    # --- Static files ---

    @app.get("/", response_class=HTMLResponse)
    def index(request: Request):
        token = request.query_params.get("token")
        # A wrong token sets nothing: the page loads as before and its prompt
        # recovers. On a non-loopback bind the page's script still needs
        # `?token=` from the URL, so there is no redirect either.
        if _session_enabled and token and _bearer_matches(f"Bearer {token}"):
            query = urlencode(
                [(k, v) for k, v in request.query_params.multi_items() if k != "token"]
            )
            response = RedirectResponse(
                request.url.path + (f"?{query}" if query else ""), status_code=303
            )
            # No Max-Age/Expires: it ends with the browser session. No Secure:
            # `soup ui` serves plain HTTP.
            response.set_cookie(
                _session_cookie, _new_session(), httponly=True, samesite="strict", path="/"
            )
            return response
        index_path = STATIC_DIR / "index.html"
        return HTMLResponse(content=index_path.read_text(encoding="utf-8"))

    app.mount("/static", StaticFiles(directory=str(STATIC_DIR)), name="static")

    # --- Runs API ---

    @app.get("/api/runs", dependencies=[Depends(_verify_token)])
    def list_runs(limit: int = Query(default=50, ge=1, le=500)):
        from soup_cli.experiment.tracker import ExperimentTracker

        tracker = ExperimentTracker()
        try:
            runs = tracker.list_runs(limit=limit)
            return {"runs": runs}
        finally:
            tracker.close()

    @app.get("/api/runs/compare", dependencies=[Depends(_verify_token)])
    def compare_runs(ids: str = Query(default="")):
        """Compare metrics for multiple runs."""
        from soup_cli.experiment.tracker import ExperimentTracker

        if not ids or not ids.strip():
            raise HTTPException(status_code=400, detail="ids parameter required")

        run_ids = [rid.strip() for rid in ids.split(",") if rid.strip()]
        if len(run_ids) > 5:
            raise HTTPException(
                status_code=400, detail="Maximum 5 runs per comparison"
            )
        if not run_ids:
            raise HTTPException(status_code=400, detail="ids parameter required")

        tracker = ExperimentTracker()
        try:
            result = []
            for rid in run_ids:
                run_info = tracker.get_run(rid)
                metrics = tracker.get_metrics(rid)
                config = {}
                if run_info and run_info.get("config_json"):
                    try:
                        config = json_mod.loads(run_info["config_json"])
                    except (ValueError, TypeError):
                        pass
                result.append({
                    "run_id": rid,
                    "config": config,
                    "metrics": metrics,
                })
            return {"runs": result}
        finally:
            tracker.close()

    @app.get("/api/runs/{run_id}", dependencies=[Depends(_verify_token)])
    def get_run(run_id: str):
        from soup_cli.experiment.tracker import ExperimentTracker

        tracker = ExperimentTracker()
        try:
            run = tracker.get_run(run_id)
            if not run:
                raise HTTPException(status_code=404, detail="Run not found")
            return run
        finally:
            tracker.close()

    @app.get("/api/runs/{run_id}/metrics", dependencies=[Depends(_verify_token)])
    def get_run_metrics(run_id: str):
        from soup_cli.experiment.tracker import ExperimentTracker

        tracker = ExperimentTracker()
        try:
            run = tracker.get_run(run_id)
            if not run:
                raise HTTPException(status_code=404, detail="Run not found")
            metrics = tracker.get_metrics(run_id)
            return {"run_id": run_id, "metrics": metrics}
        finally:
            tracker.close()

    @app.delete("/api/runs/{run_id}", dependencies=[Depends(_verify_token)])
    def delete_run(run_id: str):
        from soup_cli.experiment.tracker import ExperimentTracker

        tracker = ExperimentTracker()
        try:
            deleted = tracker.delete_run(run_id)
            if not deleted:
                raise HTTPException(status_code=404, detail="Run not found")
            return {"deleted": True, "run_id": run_id}
        finally:
            tracker.close()

    @app.get("/api/runs/{run_id}/eval", dependencies=[Depends(_verify_token)])
    def get_run_eval(run_id: str):
        from soup_cli.experiment.tracker import ExperimentTracker

        tracker = ExperimentTracker()
        try:
            results = tracker.get_eval_results(run_id=run_id)
            return {"run_id": run_id, "eval_results": results}
        finally:
            tracker.close()

    # --- GPU / System Info ---

    @app.get("/api/system", dependencies=[Depends(_verify_token)])
    def system_info():
        from soup_cli import __version__
        from soup_cli.utils.gpu import detect_device, get_gpu_info

        device, device_name = detect_device()
        gpu_info = get_gpu_info()
        return {
            "version": __version__,
            "device": device,
            "device_name": device_name,
            "gpu_info": gpu_info,
            "python_version": sys.version.split()[0],
        }

    # --- Templates ---

    @app.get("/api/templates", dependencies=[Depends(_verify_token)])
    def list_templates():
        from soup_cli.config.schema import TEMPLATES

        return {"templates": {name: yaml_str for name, yaml_str in TEMPLATES.items()}}

    # --- Config Validation ---

    @app.post("/api/config/validate", dependencies=[Depends(_verify_token)])
    def validate_config(body: dict):
        from soup_cli.config.loader import load_config_from_string

        yaml_str = body.get("yaml", "")
        if not yaml_str:
            raise HTTPException(status_code=400, detail="Empty config")
        try:
            config = load_config_from_string(yaml_str)
            return {"valid": True, "config": config.model_dump()}
        except Exception as exc:
            return {"valid": False, "error": str(exc)}

    # --- Training ---

    @app.post("/api/train/start", dependencies=[Depends(_verify_token)])
    def start_training(req: TrainRequest):
        global _train_process, _train_config_path

        with _train_lock:
            if _train_process and _train_process.poll() is None:
                raise HTTPException(
                    status_code=409, detail="Training already in progress"
                )

            # Validate config before writing to disk
            from soup_cli.config.loader import load_config_from_string
            from soup_cli.utils.terminal import strip_control

            try:
                load_config_from_string(req.config_yaml)
            except ValueError as exc:
                # The loader's own message names the field and the suggestion;
                # an unknown key now refuses here (#879), so this is where a
                # Web UI user learns which key. Rendered through escapeHtml().
                # The log line goes to the server's terminal, so the key's
                # control characters are stripped there.
                logger.warning("Invalid training config: %s", strip_control(exc))
                raise HTTPException(
                    status_code=400, detail=f"Invalid training configuration: {exc}"
                )
            except Exception as exc:
                logger.warning("Invalid training config: %s", strip_control(exc))
                raise HTTPException(
                    status_code=400, detail="Invalid training configuration"
                )

            # Securely-created temp file. A FIXED name in the shared temp dir
            # let a local attacker pre-place a symlink there and redirect this
            # write; mkstemp creates a fresh O_EXCL file (no symlink following,
            # unpredictable name).
            fd, config_path = tempfile.mkstemp(
                prefix="soup_ui_config_", suffix=".yaml"
            )
            try:
                try:
                    fh = os.fdopen(fd, "w", encoding="utf-8")
                except BaseException:
                    with contextlib.suppress(OSError):
                        # os.fdopen can occasionally close the descriptor itself before raising;
                        # practically unreachable with the fixed "w"/utf-8 arguments here.
                        os.close(fd)
                    raise
                with fh:
                    fh.write(req.config_yaml)
                proc = subprocess.Popen(
                    _resolve_train_argv(config_path),
                    stdout=subprocess.PIPE,
                    stderr=subprocess.STDOUT,
                )
            except Exception:
                _remove_temp_config(config_path)
                raise

            _train_config_path = config_path
            _train_process = proc
            global _train_log_buffer, _train_drain_thread
            _train_log_buffer = TrainLogBuffer(maxlen=10000)
            _train_drain_thread = threading.Thread(
                target=_drain_stdout_worker,
                args=(_train_process, _train_log_buffer, config_path),
                daemon=True,
                name="soup_train_drain",
            )
            _train_drain_thread.start()
            return {"started": True, "pid": _train_process.pid}

    @app.get("/api/train/status", dependencies=[Depends(_verify_token)])
    def train_status():
        global _train_process
        with _train_lock:
            if _train_process is None:
                return TrainStatus(running=False)
            poll = _train_process.poll()
            if poll is None:
                return TrainStatus(
                    running=True,
                    pid=_train_process.pid,
                    config_path=_train_config_path,
                )
            return TrainStatus(running=False, pid=_train_process.pid)

    @app.post("/api/train/stop", dependencies=[Depends(_verify_token)])
    def stop_training():
        global _train_process
        with _train_lock:
            if _train_process and _train_process.poll() is None:
                _train_process.terminate()
                return {"stopped": True}
            return {"stopped": False, "detail": "No training in progress"}

    # --- Data Inspection ---

    @app.post("/api/data/inspect", dependencies=[Depends(_verify_token)])
    def inspect_data(req: DataInspectRequest):
        from soup_cli.data.loader import load_raw_data
        from soup_cli.utils.paths import is_under_cwd

        # The path lives inside the body, so it can't be checked before the
        # body is parsed. _BODY_SIZE_LIMITS caps that read at 8 KiB instead.

        # Path traversal protection. Use realpath + commonpath containment
        # (is_under_cwd) — the old str.startswith check let a sibling like
        # ".../project-secrets" pass as under ".../project".
        try:
            resolved = Path(req.path).resolve()
        except (ValueError, OSError):
            raise HTTPException(status_code=400, detail="Invalid path")

        if not is_under_cwd(req.path):
            raise HTTPException(
                status_code=403, detail="Access denied: path outside working directory"
            )

        if not resolved.exists():
            raise HTTPException(status_code=404, detail="File not found")

        try:
            raw_data = load_raw_data(resolved)
        except Exception as exc:
            logger.warning("Data inspect error: %s", exc)
            raise HTTPException(status_code=400, detail="Failed to load data file")

        total = len(raw_data)
        sample = raw_data[: req.limit]

        # Detect format. detect_format samples a bounded prefix itself (#1424),
        # and a file that mixes shapes is refused, so a ValueError here is a
        # reportable outcome for the explorer rather than a 500: this is where
        # someone lands after soup train refused the same file.
        from soup_cli.data.formats import detect_format

        format_error = None
        try:
            fmt = detect_format(raw_data) if raw_data else "unknown"
        except ValueError as exc:
            fmt, format_error = "unknown", str(exc)

        # Basic stats
        keys = set()
        for entry in sample:
            if isinstance(entry, dict):
                keys.update(entry.keys())

        return {
            "path": str(resolved),
            "total": total,
            "format": fmt,
            "format_error": format_error,
            "keys": sorted(keys),
            "sample": sample,
        }

    # --- Training Live Monitor (SSE) ---

    @app.get("/api/train/logs", dependencies=[Depends(_verify_token_or_ticket)])
    def stream_training_logs(request: Request):
        """SSE endpoint streaming training log lines in real time."""
        from fastapi.responses import StreamingResponse

        last_event_id = request.headers.get("Last-Event-ID")
        skip_count = 0
        if last_event_id and last_event_id.isdigit():
            skip_count = int(last_event_id) + 1

        def _generate_log_events():
            with _train_lock:
                buf = _train_log_buffer
            if buf is None:
                yield "event: done\ndata: {}\n\n"
                return

            current_idx = skip_count
            while True:
                lines, is_done = buf.wait_for_lines_or_done(current_idx, timeout=0.5)
                for idx, text in lines:
                    data = json_mod.dumps({"line": text, "id": idx})
                    yield f"id: {idx}\ndata: {data}\n\n"
                    current_idx = idx + 1

                if is_done:
                    # Drain any remaining lines buffered before completion
                    remaining = buf.get_lines_from(current_idx)
                    for idx, text in remaining:
                        data = json_mod.dumps({"line": text, "id": idx})
                        yield f"id: {idx}\ndata: {data}\n\n"
                        current_idx = idx + 1
                    break

            yield "event: done\ndata: {}\n\n"

        return StreamingResponse(
            _generate_log_events(),
            media_type="text/event-stream",
            headers={
                "Cache-Control": "no-cache",
                "X-Accel-Buffering": "no",
            },
        )

    @app.get("/api/train/metrics/live", dependencies=[Depends(_verify_token_or_ticket)])
    def stream_live_metrics(
        request: Request,
        run_id: Optional[str] = Query(default=None),
    ):
        """SSE endpoint streaming new metrics as they're logged."""
        from fastapi.responses import StreamingResponse

        def _generate_metrics_events():
            from soup_cli.experiment.tracker import ExperimentTracker

            with _train_lock:
                proc = _train_process
            if proc is None and run_id is None:
                yield "event: done\ndata: {}\n\n"
                return

            last_step = -1
            max_polls = 3  # For tests: limit poll cycles when process done
            polls_since_new = 0

            while True:
                tracker = ExperimentTracker()
                try:
                    if run_id:
                        metrics = tracker.get_metrics(run_id)
                    else:
                        yield "event: done\ndata: {}\n\n"
                        return
                finally:
                    tracker.close()

                new_metrics = [
                    m for m in metrics if m.get("step", 0) > last_step
                ]
                if new_metrics:
                    for m_row in new_metrics:
                        data = json_mod.dumps(m_row, default=str)
                        yield f"data: {data}\n\n"
                    last_step = max(
                        m.get("step", 0) for m in new_metrics
                    )
                    polls_since_new = 0
                else:
                    polls_since_new += 1

                # Check if training is still running
                with _train_lock:
                    proc = _train_process
                if proc is None or proc.poll() is not None:
                    if polls_since_new >= 1:
                        yield "event: done\ndata: {}\n\n"
                        return

                # Yield heartbeat
                yield ":heartbeat\n\n"

                if polls_since_new >= max_polls:
                    yield "event: done\ndata: {}\n\n"
                    return

                time.sleep(0.1)  # Short poll for tests

        return StreamingResponse(
            _generate_metrics_events(),
            media_type="text/event-stream",
            headers={
                "Cache-Control": "no-cache",
                "X-Accel-Buffering": "no",
            },
        )

    @app.get("/api/train/progress", dependencies=[Depends(_verify_token)])
    def train_progress(
        run_id: Optional[str] = Query(default=None),
    ):
        """Return current training progress snapshot."""
        # Read the shared process handle under the lock, like every sibling
        # endpoint (start/status/stop) — avoids a torn read racing a concurrent
        # start/stop.
        with _train_lock:
            proc = _train_process
        is_running = proc is not None and proc.poll() is None

        if not is_running and run_id is None:
            return {"running": False, "current_step": 0, "run_id": None}

        if run_id:
            from soup_cli.experiment.tracker import ExperimentTracker

            tracker = ExperimentTracker()
            try:
                metrics = tracker.get_metrics(run_id)
                current_step = metrics[-1]["step"] if metrics else 0
            finally:
                tracker.close()

            return {
                "running": is_running,
                "current_step": current_step,
                "run_id": run_id,
            }

        return {"running": is_running, "current_step": 0, "run_id": None}

    # --- Config Builder ---

    @app.get("/api/config/schema", dependencies=[Depends(_verify_token)])
    def config_schema():
        """Return config schema as JSON for form generation."""
        from soup_cli.config.schema import (
            DataConfig,
            LoraConfig,
            SoupConfig,
            TrainingConfig,
        )

        def _extract_field_info(model_cls):
            """Extract field metadata from a Pydantic model."""
            result = {}
            for name, field_info in model_cls.model_fields.items():
                info = {"type": "string", "required": field_info.is_required()}

                # Get default value
                if field_info.default is not None:
                    info["default"] = field_info.default

                # Get type annotation
                annotation = field_info.annotation
                if annotation is not None:
                    ann_str = str(annotation)
                    if "int" in ann_str:
                        info["type"] = "integer"
                    elif "float" in ann_str:
                        info["type"] = "number"
                    elif "bool" in ann_str:
                        info["type"] = "boolean"

                    # Check for Literal (enum) types
                    args = getattr(annotation, "__args__", None)
                    if args:
                        # Filter out NoneType for Optional[Literal[...]]
                        non_none = [a for a in args if a is not type(None)]
                        if non_none and all(isinstance(a, str) for a in non_none):
                            info["type"] = "enum"
                            info["options"] = list(non_none)

                # Get constraints from metadata
                for meta in (field_info.metadata or []):
                    if hasattr(meta, "ge"):
                        info["ge"] = meta.ge
                    if hasattr(meta, "le"):
                        info["le"] = meta.le

                result[name] = info
            return result

        schema = _extract_field_info(SoupConfig)
        schema["data"] = _extract_field_info(DataConfig)
        schema["training"] = _extract_field_info(TrainingConfig)
        schema["training"]["lora"] = _extract_field_info(LoraConfig)
        return schema

    @app.get("/api/recipes", dependencies=[Depends(_verify_token)])
    def list_recipes():
        """Return recipe catalog as JSON."""
        from soup_cli.recipes.catalog import RECIPES

        recipes_list = []
        for name, meta in RECIPES.items():
            recipes_list.append({
                "name": name,
                "model": meta.model,
                "task": meta.task,
                "description": meta.description,
                "tags": list(meta.tags) if hasattr(meta, "tags") else [],
                "yaml": meta.yaml_str,
            })
        return {"recipes": recipes_list}

    @app.post("/api/config/from-form", dependencies=[Depends(_verify_token)])
    def form_to_yaml(body: dict):
        """Convert form field values to validated YAML string."""
        import yaml

        from soup_cli.config.loader import load_config_from_string
        from soup_cli.utils.terminal import strip_control

        # Build YAML from form values
        config_dict = {}
        for key, val in body.items():
            if val is not None and val != "" and val != {}:
                config_dict[key] = val

        try:
            yaml_str = yaml.dump(
                config_dict, default_flow_style=False, sort_keys=False
            )
            # Validate
            load_config_from_string(yaml_str)
            return {"yaml": yaml_str}
        except ValueError as exc:
            logger.warning("Config form validation error: %s", strip_control(exc))
            return {"error": f"Invalid configuration: {exc}"}
        except TypeError as exc:
            logger.warning("Config form validation error: %s", strip_control(exc))
            return {"error": "Invalid configuration"}

    # --- Chat Proxy ---

    class ChatMessage(PydanticBaseModel):
        """A single chat message."""
        role: str
        content: str

    class ChatRequest(PydanticBaseModel):
        """Request body for chat send."""
        messages: list[ChatMessage]
        endpoint: str
        temperature: float = Field(default=0.7, ge=0.0, le=2.0)
        max_tokens: int = Field(default=512, ge=1, le=16384)
        top_p: float = Field(default=0.9, ge=0.0, le=1.0)
        adapter: Optional[str] = None

    @app.post("/api/chat/send", dependencies=[Depends(_verify_token)])
    def chat_send(req: ChatRequest):
        """SSE proxy endpoint streaming chat completions."""
        from urllib.parse import urlparse

        from fastapi.responses import StreamingResponse

        from soup_cli.utils.net_guard import UNSPECIFIED_HOST_HINT, refuse_private_ip_literal

        # Validate messages
        if not req.messages:
            raise HTTPException(status_code=400, detail="messages cannot be empty")

        # SSRF protection: localhost-only HTTP, HTTPS for remote, and no
        # private / link-local / reserved IP literal on either scheme.
        # 0.0.0.0 is the bind-any wildcard, not a loopback address.
        try:
            parsed = urlparse(req.endpoint)
        except ValueError:
            # e.g. an unbalanced or non-IPv6 bracketed host: a bad request,
            # not a server error.
            raise HTTPException(status_code=400, detail="endpoint is not a valid URL") from None
        if parsed.scheme == "http":
            import ipaddress as _ipaddr

            host = parsed.hostname or ""
            is_local = host == "localhost"
            if not is_local:
                try:
                    addr = _ipaddr.ip_address(host)
                    is_local = addr.is_loopback
                except ValueError:
                    is_local = False
            if not is_local and host == "0.0.0.0":
                raise HTTPException(
                    status_code=400, detail=f"endpoint {UNSPECIFIED_HOST_HINT}"
                )
            if not is_local:
                raise HTTPException(
                    status_code=400,
                    detail="HTTP only allowed for localhost endpoints",
                )
        elif parsed.scheme != "https":
            raise HTTPException(
                status_code=400,
                detail="Only HTTP (localhost) or HTTPS endpoints allowed",
            )
        try:
            refuse_private_ip_literal(parsed.hostname, label="endpoint")
        except ValueError as exc:
            # The message is fixed text (the label is a constant); the
            # endpoint itself is never echoed back.
            raise HTTPException(status_code=400, detail=str(exc)) from None

        # Validate bounds
        if req.max_tokens > 16384:
            raise HTTPException(
                status_code=400, detail="max_tokens exceeds 16384 cap"
            )
        if req.temperature < 0.0 or req.temperature > 2.0:
            raise HTTPException(
                status_code=400, detail="temperature must be 0.0-2.0"
            )
        if req.top_p < 0.0 or req.top_p > 1.0:
            raise HTTPException(
                status_code=400, detail="top_p must be 0.0-1.0"
            )

        def _stream_chat():
            import httpx

            url = req.endpoint.rstrip("/") + "/v1/chat/completions"
            payload = {
                "messages": [m.model_dump() for m in req.messages],
                "max_tokens": req.max_tokens,
                "temperature": req.temperature,
                "top_p": req.top_p,
                "stream": True,
            }
            if req.adapter:
                payload["model"] = req.adapter

            try:
                with httpx.stream(
                    "POST", url,
                    json=payload,
                    headers={"Content-Type": "application/json"},
                    timeout=120.0,
                ) as resp:
                    for line in resp.iter_lines():
                        if line.startswith("data: "):
                            data_str = line[6:]
                            if data_str.strip() == "[DONE]":
                                yield "data: {\"done\": true}\n\n"
                                return
                            try:
                                parsed_data = json_mod.loads(data_str)
                                delta = (
                                    parsed_data.get("choices", [{}])[0]
                                    .get("delta", {})
                                    .get("content", "")
                                )
                                if delta:
                                    out = json_mod.dumps({"delta": delta})
                                    yield f"data: {out}\n\n"
                            except (ValueError, IndexError, KeyError):
                                pass
                yield "data: {\"done\": true}\n\n"
            except Exception as exc:
                logger.warning("Chat proxy error: %s", exc)
                err_msg = json_mod.dumps(
                    {"error": "Connection failed"}
                )
                yield f"data: {err_msg}\n\n"

        return StreamingResponse(
            _stream_chat(),
            media_type="text/event-stream",
            headers={
                "Cache-Control": "no-cache",
                "X-Accel-Buffering": "no",
            },
        )

    # --- v0.53.9 #94: SSE training-event stream ---

    @app.get("/api/train/stream", dependencies=[Depends(_verify_token_or_ticket)])
    async def stream_train_events():
        """SSE endpoint streaming `TrainEvent` payloads as JSON frames.

        Per-subscriber cursor — multiple concurrent listeners each receive
        every event (no destructive drain). Uses `asyncio.sleep` so the
        uvicorn async loop is not blocked under default workers.
        """
        import asyncio

        from fastapi.responses import StreamingResponse

        from soup_cli.utils.sse_train_stream import TrainEvent, format_sse_frame
        from soup_cli.utils.train_event_buffer import get_global_buffer

        buffer = get_global_buffer()

        async def _gen():
            # Start from cursor 0 — new subscribers receive a bounded
            # catch-up of retained events (deque maxlen=1000) before
            # streaming fresh ones. Concurrent subscribers are independent.
            cursor = 0
            max_ticks = 200  # cap to keep test runs bounded; ~20s at 100ms
            empty_ticks = 0
            for _ in range(max_ticks):
                events, cursor = buffer.snapshot_since(cursor)
                if events:
                    empty_ticks = 0
                    for event in events:
                        yield format_sse_frame(event)
                else:
                    empty_ticks += 1
                    yield ":heartbeat\n\n"
                with _train_lock:
                    proc = _train_process
                if proc is None or proc.poll() is not None:
                    if empty_ticks >= 1:
                        done = TrainEvent(type="status", message="done")
                        yield format_sse_frame(done)
                        return
                await asyncio.sleep(0.1)
            done = TrainEvent(type="status", message="timeout")
            yield format_sse_frame(done)

        return StreamingResponse(
            _gen(),
            media_type="text/event-stream",
            headers={
                "Cache-Control": "no-cache",
                "X-Accel-Buffering": "no",
            },
        )

    # --- v0.53.9 #100: Tool-call observation panel ---

    @app.get("/api/tool-outputs", dependencies=[Depends(_verify_token)])
    def list_tool_outputs(
        limit: int = Query(default=100, ge=1, le=1000),
    ):
        """Return the most recent tool-call records as JSON.

        Records are pushed by the SFT trainer's tool-calling callback
        into the process-wide `ToolOutputsBuffer`. Read-only; safe for
        cross-origin polling.
        """
        from soup_cli.utils.tool_outputs import get_global_tool_buffer

        records = get_global_tool_buffer().snapshot(limit=limit)
        return {
            "count": len(records),
            "records": [
                {
                    "name": r.name,
                    "started_ts": r.started_ts,
                    "duration_ms": r.duration_ms,
                    "success": r.success,
                    "output_preview": r.output_preview,
                    "error": r.error,
                }
                for r in records
            ],
        }

    # --- Health ---

    @app.get("/api/health")
    def health():
        return {"status": "ok"}

    return app