Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

7.7 KB
Raw Blame
"""Token-authed sandbox -> backend callback for SQLV2 runs (Journey 1 slice).

Mirrors PostHog Desktop's agent-proxy callback: a plain function view (no team
scoping, no session), authed by a Bearer token the run endpoint minted. Wired in
posthog/urls.py at internal/notebooks/runs/<run_id>/result/.
"""

import json

from django.core import signing
from django.db.models import Q
from django.http import JsonResponse
from django.utils import timezone

import structlog
from drf_spectacular.utils import OpenApiResponse, extend_schema

from products.notebooks.backend.models import KernelRuntime, NotebookNodeRun
from products.notebooks.backend.sql_v2 import verify_callback_token
from products.notebooks.backend.sql_v2_concurrency import release_run_slots
from products.notebooks.backend.sql_v2_metrics import outcome_for_status, record_node_run_terminal
from products.notebooks.backend.sql_v2_serializers import NotebookSQLV2CallbackRequestSerializer

logger = structlog.get_logger(__name__)

# The envelope lands whole in a Postgres row. The kernel already caps streams, media, and
# preview cells well below this; anything bigger is a misbehaving (or hostile) sandbox, so
# reject rather than store. Sized above the kernel's worst case (~4 MB media + streams).
MAX_ENVELOPE_BYTES = 8_000_000


@extend_schema(
    tags=["notebooks"],
    request=NotebookSQLV2CallbackRequestSerializer,
    responses={
        200: OpenApiResponse(description="Result stored"),
        401: OpenApiResponse(description="Missing or invalid callback token"),
        403: OpenApiResponse(description="Token does not match the run"),
        404: OpenApiResponse(description="Run not found"),
    },
    summary="SQLV2 run result callback",
    description=(
        "Internal endpoint the notebook sandbox POSTs its result envelope to after a SQLV2 run. "
        "Authenticated with the signed callback token minted by the run endpoint (no session). "
        "Idempotent: re-delivery of the same run_id upserts the same row."
    ),
)
def notebook_sql_v2_callback(request, run_id: str) -> JsonResponse:
    if request.method != "POST":
        return JsonResponse({"error": "Method not allowed"}, status=405)

    authorization = request.headers.get("Authorization", "")
    if not authorization.startswith("Bearer "):
        return JsonResponse({"error": "Missing authorization bearer token"}, status=401)
    token = authorization[len("Bearer ") :].strip()
    if not token:
        return JsonResponse({"error": "Missing authorization bearer token"}, status=401)

    try:
        token_run_id, team_id = verify_callback_token(token)
    except signing.BadSignature:
        return JsonResponse({"error": "Invalid callback token"}, status=401)

    if token_run_id != run_id:
        return JsonResponse({"error": "Token does not match run"}, status=403)

    if len(request.body) > MAX_ENVELOPE_BYTES:
        logger.warning("sql_v2_callback_envelope_too_large", run_id=run_id, size=len(request.body))
        return JsonResponse({"error": "Envelope too large"}, status=400)

    try:
        body = json.loads(request.body)
    except (json.JSONDecodeError, ValueError):
        return JsonResponse({"error": "Invalid JSON body"}, status=400)

    serializer = NotebookSQLV2CallbackRequestSerializer(data=body)
    if not serializer.is_valid():
        return JsonResponse({"error": "Invalid request body", "detail": serializer.errors}, status=400)

    # Store the raw JSON envelope (JSON-native types) — the serializer's validated_data
    # would coerce result_id to a uuid.UUID, which the JSONField can't serialize.
    envelope = body["envelope"]
    # The snapshot is kernel state, read only off KernelRuntime. Keeping a copy on every run
    # row would park a duplicate catalog forever on the table that grows fastest (one row per
    # cell execution), so it rides the envelope to get here and is then dropped.
    frames = envelope.pop("frames", None)

    try:
        # select_related: the frame snapshot below scopes to the run's user and the slot release
        # reads the notebook's short id, so fetch both up front rather than lazy-loading each on
        # attribute access.
        run = NotebookNodeRun.objects.for_team(team_id).select_related("user", "notebook").get(id=run_id)
    except NotebookNodeRun.DoesNotExist:
        return JsonResponse({"error": "Run not found"}, status=404)

    status_by_envelope = {
        "ok": NotebookNodeRun.Status.DONE,
        "interrupted": NotebookNodeRun.Status.INTERRUPTED,
    }
    run.status = status_by_envelope.get(envelope.get("status"), NotebookNodeRun.Status.FAILED)
    run.envelope = envelope
    run.result_id = envelope.get("result_id")
    run.error = envelope.get("error")
    # Claim the RUNNING -> terminal transition with a status-guarded UPDATE so exactly one
    # delivery wins it — concurrent re-deliveries must not each report the run's metrics.
    # A loser still writes the row (the deliberate upsert: a late real result may overwrite
    # an interrupt placeholder), it just doesn't report.
    won_transition = bool(
        NotebookNodeRun.objects.for_team(team_id)
        .filter(id=run.id, status=NotebookNodeRun.Status.RUNNING)
        .update(
            status=run.status,
            envelope=run.envelope,
            result_id=run.result_id,
            error=run.error,
            updated_at=timezone.now(),
        )
    )
    if not won_transition:
        run.save(update_fields=["status", "envelope", "result_id", "error", "updated_at"])

    _store_frame_snapshot(run, frames, team_id)

    # The kernel lane's release site. This callback claims its own transition rather than going
    # through finish_node_run, so it has to hand the slot back itself or the notebook stays
    # blocked until the TTL.
    release_run_slots(team_id, run.notebook.short_id, str(run.id))

    if won_transition:
        record_node_run_terminal(run, outcome_for_status(run.status))

    return JsonResponse({"ok": True})


def _store_frame_snapshot(run: NotebookNodeRun, frames: list | None, team_id: int) -> None:
    """File the run's DuckDB catalog snapshot against the kernel that produced it (Journey 7).

    `frames` is absent for a hogql run (it never enters the kernel) and for a kernel whose
    catalog read failed. Both mean "leave the stored snapshot alone" — a hogql run changes no
    local state, and a failed read knows nothing. Only an actual empty list means "the kernel
    has nothing", so absent must never be coerced to empty here.
    """
    if frames is None or not run.kernel_runtime_id:
        return
    # Scoped to the dispatch-time kernel rather than "the notebook's current kernel": if the
    # kernel was replaced mid-run, this snapshot describes the dead one and must not overwrite
    # the live one. Team AND user because a KernelRuntime is scoped to both — kernels are per
    # user, so a notebook's collaborators each have their own, and a snapshot must never land
    # on someone else's row. Both come from the run, which was itself looked up team-scoped.
    #
    # The created_at guard makes the write last-run-wins rather than last-callback-wins. The
    # kernel runs cells one at a time in arrival order, but callbacks land on separate web
    # workers and can arrive out of order, so without it a slow older callback would overwrite
    # a newer catalog and leave deleted frames on show until the next run. Done as a
    # conditional UPDATE rather than read-then-write so concurrent callbacks can't interleave.
    (
        KernelRuntime.objects.filter(id=run.kernel_runtime_id, team_id=team_id, user=run.user)
        .filter(Q(frames_run_created_at__isnull=True) | Q(frames_run_created_at__lt=run.created_at))
        .update(frames=frames, frames_run_created_at=run.created_at)
    )