Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

11.1 KB
Raw Blame
"""Per-run execution: fetch the data, build the envelope, deliver the callback.

For the current Journey 1/2 scope every node is a pure-HogQL display node, so a
run is a capped fetch through the data plane — the ipykernel is not involved.

A run fetches up to `cache_limit` rows in one ClickHouse query and keeps them in
an in-memory per-run cache; `/page` requests within the cache are local slices
(no ClickHouse work, no held backend workers). Only paging beyond the cache — or
after a kernel restart emptied it — re-queries the data plane with LIMIT/OFFSET.
This is the capped, memory-resident precursor of the file-backed result store in
`result_store.py`.
"""

import json
import time
import logging
import threading
import urllib.error
import urllib.request
from collections import OrderedDict
from typing import Any

from . import envelope

logger = logging.getLogger(__name__)

_CALLBACK_TIMEOUT_SECONDS = 15
_DEFAULT_PAGE_LIMIT = 50
_DEFAULT_CACHE_LIMIT = 300
_MAX_CACHED_RESULTS = 20

# run_id -> {columns, types, rows (json-safe), complete}; complete means the cache
# holds the query's entire result, not just the first cache_limit rows.
_result_cache: OrderedDict[str, dict[str, Any]] = OrderedDict()
_result_cache_lock = threading.Lock()

# run_id -> {event, kernel}, registered for the lifetime of the run's execute_run call.
# /interrupt sets the event (cancels queued runs and data-plane waits) and SIGINTs the
# kernel only when the run is a kernel node actually executing a cell.
_cancel_states: dict[str, dict[str, Any]] = {}
_cancel_states_lock = threading.Lock()


def request_interrupt(run_id: str) -> bool:
    """Handle an /interrupt for `run_id`; return whether the run was known (still in flight)."""
    with _cancel_states_lock:
        state = _cancel_states.get(run_id)
    if state is None:
        return False
    state["event"].set()
    if state["kernel"]:
        # Deferred so a pure-HogQL interrupt never imports jupyter_client (heavy, sandbox-only);
        # for a kernel-node run the module is already loaded, since the run itself imported it.
        from . import executor  # noqa: PLC0415 — keeps jupyter_client off the server import path

        executor.get_executor().interrupt_for_run(run_id)
    return True


def complete_code(payload: dict[str, Any]) -> dict[str, Any]:
    """Handle a /complete: completions from the live kernel's namespace."""
    from . import executor  # noqa: PLC0415 — keeps jupyter_client off the server import path

    return executor.get_executor().complete(str(payload.get("code") or ""), int(payload.get("cursor_pos") or 0))


def inspect_code(payload: dict[str, Any]) -> dict[str, Any]:
    """Handle an /inspect: the signature and docstring of the name under the cursor."""
    from . import executor  # noqa: PLC0415 — keeps jupyter_client off the server import path

    return executor.get_executor().inspect(
        str(payload.get("code") or ""),
        int(payload.get("cursor_pos") or 0),
        int(payload.get("detail_level") or 0),
    )


def execute_run(payload: dict[str, Any]) -> None:
    """Entry point for a /run request, invoked on a background thread.

    Routing rule: a kernel node — python or duckdb —
    runs in the ipykernel (materialize inputs, run code); a pure-HogQL display node
    stays a capped data-plane fetch and never touches the kernel.
    """
    run_id = str(payload.get("run_id") or "")
    node = payload.get("node") or {}
    is_kernel_node = node.get("type") in ("python", "duckdb")
    cancel_event = threading.Event()
    with _cancel_states_lock:
        # A retried /run for a run still in flight must not clobber its cancel state (the
        # interrupt would target the retry, and the first thread's exit would unregister it).
        # The in-flight thread owns the run and delivers its callback; drop the duplicate.
        if run_id in _cancel_states:
            logger.warning("nb_kernel ignoring duplicate /run for in-flight run %s", run_id)
            return
        _cancel_states[run_id] = {"event": cancel_event, "kernel": is_kernel_node}
    started = time.monotonic()
    try:
        # Interrupt-caused failures are labeled at the source (DataPlaneInterrupted, a
        # SIGINT'd cell), so a real failure that merely raced the cancel keeps its error —
        # and a completed run stays ok: the result is real.
        if is_kernel_node:
            result = _run_kernel_node(payload, cancel_event)
        else:
            result = _build_envelope(payload, cancel_event)
    finally:
        with _cancel_states_lock:
            _cancel_states.pop(run_id, None)
    # The whole sandbox-side run — dominated by the phases below it but also covering
    # kernel boot and queueing behind another run's cell. Backed out into metrics by the
    # backend callback (sql_v2_metrics.py).
    timings = result.setdefault("timings", {})
    timings["sandbox_total_s"] = round(time.monotonic() - started, 3)
    _post_callback(payload["callback_url"], payload["callback_token"], result)


def _run_kernel_node(payload: dict[str, Any], cancel_event: threading.Event) -> dict[str, Any]:
    # Deferred so the server startup path never imports jupyter_client (heavy, sandbox-only).
    from . import executor  # noqa: PLC0415 — keeps jupyter_client off the server import path

    try:
        return executor.get_executor().run_kernel_node(payload, cancel_event)
    except Exception as exc:  # noqa: BLE001 — a run must always produce a callback envelope
        logger.exception("nb_kernel kernel node run failed")
        return envelope.from_error(f"Node failed in the sandbox: {exc}")


def _fetch_capped_page(
    payload: dict[str, Any], limit: int, offset: int, cancel_event: threading.Event | None = None
) -> dict[str, Any]:
    """Fetch limit+1 rows through the data plane; the extra row only signals has_more."""
    # Deferred so a broken pyarrow install degrades to a per-run error envelope
    # instead of preventing the server from starting.
    from . import data_plane  # noqa: PLC0415

    columns, rows, types = data_plane.fetch_query_page(
        payload["data_plane_url"],
        payload["data_plane_token"],
        payload["code"],
        limit=limit + 1,
        offset=offset,
        cancel_event=cancel_event,
    )
    has_more = len(rows) > limit
    return {"columns": columns, "rows": rows[:limit], "types": types, "has_more": has_more}


def _cache_result(
    run_id: str, columns: list[str], types: list[list[str]], rows: list[list[Any]], complete: bool
) -> None:
    with _result_cache_lock:
        _result_cache[run_id] = {"columns": columns, "types": types, "rows": rows, "complete": complete}
        _result_cache.move_to_end(run_id)
        while len(_result_cache) > _MAX_CACHED_RESULTS:
            _result_cache.popitem(last=False)


def _cached_page(run_id: str, offset: int, limit: int) -> dict[str, Any] | None:
    """Slice a page out of the cached result, or None if the cache can't serve it."""
    with _result_cache_lock:
        entry = _result_cache.get(run_id)
        if entry is None:
            return None
        _result_cache.move_to_end(run_id)
    rows = entry["rows"]
    end = offset + limit
    # A page that runs past an incomplete cache needs rows we don't have — re-query.
    if not entry["complete"] and end > len(rows):
        return None
    return {
        "columns": entry["columns"],
        "types": entry["types"],
        "rows": rows[offset:end],
        "has_more": end < len(rows) or not entry["complete"],
    }


def _build_envelope(payload: dict[str, Any], cancel_event: threading.Event | None = None) -> dict[str, Any]:
    from . import data_plane  # noqa: PLC0415 — see _fetch_capped_page

    page_limit = int(payload.get("page_limit") or _DEFAULT_PAGE_LIMIT)
    cache_limit = max(int(payload.get("cache_limit") or _DEFAULT_CACHE_LIMIT), page_limit)
    # The data-plane wait for the display fetch — the same phase a kernel node's input
    # materialization reports, so one metric covers both lanes. Attached on error paths
    # too: a run that died waiting on the data plane must still report where its time went.
    timings: dict[str, float] = {}
    try:
        fetch_started = time.monotonic()
        try:
            result = _fetch_capped_page(payload, limit=cache_limit, offset=0, cancel_event=cancel_event)
        finally:
            timings["input_wait_s"] = round(time.monotonic() - fetch_started, 3)
        rows = envelope.json_safe_rows(result["rows"])
        run_id = str(payload.get("run_id") or "")
        if run_id:
            _cache_result(run_id, result["columns"], result["types"], rows, complete=not result["has_more"])
        built = envelope.from_columns_and_rows(
            result["columns"],
            result["rows"][:page_limit],
            result["types"],
            has_more=result["has_more"] or len(rows) > page_limit,
        )
    except data_plane.DataPlaneInterrupted:
        built = envelope.as_interrupted(envelope.from_error(envelope.INTERRUPTED_MESSAGE))
    except data_plane.DataPlaneError as exc:
        built = envelope.from_error(str(exc))
    except Exception as exc:  # noqa: BLE001 — any failure must still produce a callback
        built = envelope.from_error(f"Run failed in the sandbox: {exc}")
    if timings:
        built["timings"] = timings
    return built


def fetch_page(payload: dict[str, Any]) -> dict[str, Any]:
    """Serve a /page request: a result-frame slice, a cached local slice, or a bounded re-query.

    A payload carrying `result_id` is a kernel-node result — it pages exclusively from the
    on-disk result frame (its code is not a HogQL query, so there is no data-plane fallback).
    Raises DataPlaneError/ResultStoreError for user-facing errors; the server maps them to 400.
    """
    offset = int(payload.get("offset") or 0)
    limit = int(payload.get("limit") or _DEFAULT_PAGE_LIMIT)

    result_id = payload.get("result_id")
    if result_id:
        # Deferred so a broken pyarrow install degrades per-request, not at server startup.
        from . import result_store  # noqa: PLC0415

        return result_store.read_page(str(result_id), offset, limit)

    cached = _cached_page(str(payload.get("run_id") or ""), offset, limit)
    if cached is not None:
        return cached

    page = _fetch_capped_page(payload, limit=limit, offset=offset)
    return {
        "columns": page["columns"],
        "types": page["types"],
        "rows": envelope.json_safe_rows(page["rows"]),
        "has_more": page["has_more"],
    }


def _post_callback(callback_url: str, callback_token: str, result: dict[str, Any]) -> None:
    request = urllib.request.Request(
        callback_url,
        data=json.dumps({"envelope": result}).encode(),
        headers={"Authorization": f"Bearer {callback_token}", "Content-Type": "application/json"},
        method="POST",
    )
    try:
        # url is the backend's own callback endpoint from the signed run payload, never user-controlled.
        # nosemgrep: python.lang.security.audit.dynamic-urllib-use-detected.dynamic-urllib-use-detected
        urllib.request.urlopen(request, timeout=_CALLBACK_TIMEOUT_SECONDS)
    except Exception:  # noqa: BLE001 — best-effort; the backend watchdog covers a lost callback
        logger.exception("nb_kernel callback delivery failed")