:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
"""Helpers for the revamped-notebooks SQLV2 run flow (Journey 1).
The backend dispatches a run to the in-sandbox kernel-server with a single HTTP
POST (mirroring PostHog Desktop's agent-server). The kernel-server fetches the
node's capped result page from the data-plane endpoint (real ClickHouse data via
HogQL) and POSTs the envelope back to the token-authed callback endpoint. The
control plane (write_file / execute) is used only to deploy and launch the
kernel-server package — never per run.
The callback and data-plane tokens are stateless signed tokens for the slice;
hardening swaps them for the RS256 sandbox event-ingest JWTs used by PostHog Desktop.
"""
import hmac
import time
import uuid
import hashlib
from typing import Literal
from django.conf import settings
from django.core import signing
import requests
import structlog
import posthoganalytics
from posthog.dataclasses import frozen
from posthog.models.user import User
from products.notebooks.backend.kernel_package import (
BAKED_PACKAGE_ROOT,
BAKED_VERSION_PATH,
SANDBOX_PACKAGE_NAME,
kernel_package_bytes_and_hash,
)
from products.notebooks.backend.models import KernelRuntime, Notebook, NotebookNodeRun
from products.tasks.backend.facade.sandbox import SandboxBase, get_sandbox_class_for_backend
logger = structlog.get_logger(__name__)
REVAMPED_PY_NOTEBOOKS_FLAG = "revamped-py-notebooks"
NOTEBOOKS_FRAME_STORE_FLAG = "notebooks-frame-store"
NOTEBOOKS_FRAME_STORE_CH_WRITES_FLAG = "notebooks-frame-store-ch-writes"
# How a run's result rows reached whoever consumes them. The data plane resolves this at
# the moment it picks a transport and reports it in the 202 accept body; the sandbox echoes
# it back in the envelope without interpreting it, and `sql_v2_metrics` turns it into a
# metric label. Because the value crosses the sandbox boundary, where user code can forge
# an envelope, `sql_v2_metrics.DELIVERY_LABEL_VALUES` is the allowlist that bounds label
# cardinality; add any new mode to both places.
DELIVERY_DIRECT = "direct" # direct lane: ClickHouse to Redis to the UI, no sandbox involved
DELIVERY_INLINE = "inline" # sandbox fetch over the inline (Redis) transport, clamped at the async row ceiling
DELIVERY_OBJECT_RELAY = "object_relay" # a Temporal worker streams ClickHouse rows to object storage
DELIVERY_OBJECT_CH_WRITES = "object_ch_writes" # ClickHouse writes the object itself via INSERT INTO FUNCTION s3
DELIVERY_NONE = "none" # kernel run that read no ClickHouse result, so no transport was involved
DELIVERY_MIXED = "mixed" # one run materialized several inputs and they did not agree on a transport
_CALLBACK_TOKEN_SALT = "notebooks.sql_v2.callback"
_DATA_PLANE_TOKEN_SALT = "notebooks.sql_v2.data_plane"
# Both tokens are minted at dispatch and used until the run ends: the data-plane token for the
# heartbeat, the callback token for the result. They must outlive the kernel's execute cap
# (_EXECUTE_TIMEOUT_SECONDS in sandbox/kernel/executor.py, 6 hours) plus the input fetches.
_CALLBACK_TOKEN_MAX_AGE_SECONDS = 7 * 60 * 60
_DATA_PLANE_TOKEN_MAX_AGE_SECONDS = 7 * 60 * 60
# Rows in the display page the run envelope carries to the UI.
DISPLAY_PAGE_LIMIT = 50
# Rows the kernel fetches and caches per run. Pages within the cache are local slices
# (no ClickHouse re-query); paging beyond it falls back to a LIMIT/OFFSET re-query.
RESULT_CACHE_ROWS = 300
# The container port the sandbox already exposes (mapped to a host port at create
# time). Mirrors docker_sandbox.AGENT_SERVER_PORT (47821) and
# modal_sandbox.AGENT_SERVER_PORT (8080) — the kernel-server binds it inside the sandbox.
_CONTAINER_PORT_BY_BACKEND: dict[str, int] = {
KernelRuntime.Backend.DOCKER: 47821,
KernelRuntime.Backend.MODAL: 8080,
}
_PACKAGE_ROOT = "/tmp/nb_kernel_pkg"
_TARBALL_PATH = "/tmp/nb_kernel.tar.gz"
_SECRET_PATH = "/tmp/nb_sql_v2_secret"
_SERVER_LOG_PATH = "/tmp/nb_kernel_server.log"
_SERVER_PID_PATH = "/tmp/nb_kernel_server.pid"
_SERVER_READY_TIMEOUT_SECONDS = 15
_RUN_POST_TIMEOUT_SECONDS = 10
# A page fetch holds a web worker for the whole kernel -> data plane -> CH round trip.
_PAGE_POST_TIMEOUT_SECONDS = 60
# An interrupt only sets a cancel event and sends a SIGINT in the sandbox: near-instant.
_INTERRUPT_POST_TIMEOUT_SECONDS = 5
# Just past the sandbox's own 2s introspection wait, so a busy kernel answers empty, not a timeout.
_INTROSPECTION_POST_TIMEOUT_SECONDS = 4
# Safety-net TTL for the per-user page-fetch lock: sized just past the POST timeout so a
# worker killed before its finally-release can't wedge the user's paging for long.
PAGE_LOCK_TTL_SECONDS = _PAGE_POST_TIMEOUT_SECONDS + 10
_COMMAND_TOKEN_TTL_SECONDS = 300
def sql_v2_page_lock_key(team_id: int, user_id: int | None) -> str:
"""One in-flight page fetch per user: each holds a web worker for up to the POST timeout."""
return f"sqlv2_page_inflight:{team_id}:{user_id if user_id is not None else 'anon'}"
class SQLV2KernelNotRunning(Exception):
"""Raised when a run is dispatched but the notebook has no running sandbox."""
class SQLV2PageError(Exception):
"""A page fetch the kernel rejected or failed; message is user-facing."""
def _flag_enabled_for(flag: str, user: User | None) -> bool:
if user is None or not user.distinct_id:
return False
kwargs: dict = {"only_evaluate_locally": False, "send_feature_flag_events": False}
org = getattr(user, "organization", None)
if org is not None:
org_id = str(org.id)
kwargs["groups"] = {"organization": org_id}
kwargs["group_properties"] = {"organization": {"id": org_id}}
return bool(posthoganalytics.feature_enabled(flag, user.distinct_id, **kwargs))
def is_sql_v2_enabled(user: User | None) -> bool:
return _flag_enabled_for(REVAMPED_PY_NOTEBOOKS_FLAG, user)
def is_frame_store_enabled(user: User | None) -> bool:
"""Whether this user's whole-frame materializations may use the object-storage path.
Narrower than `is_sql_v2_enabled`: it gates only the transport, so a user without it
still runs SQLV2 nodes, just over the inline transport and its 50k row clamp. Both this
and the deployment's `frame_store.is_enabled()` must hold, so the flag can widen the
rollout no further than the environment is provisioned for.
"""
return _flag_enabled_for(NOTEBOOKS_FRAME_STORE_FLAG, user)
def is_frame_store_ch_writes_enabled(user: User | None) -> bool:
"""Whether this user's materializations take the ClickHouse-side write path.
One switch for the whole new flow: it moves the query onto the offline pool as the
dedicated `notebooks` user, and hands the object write to ClickHouse. Sits inside
`is_frame_store_enabled`, which has to be on before any of this is reached.
Resolved in the web process and carried on the job, because the Temporal activity that
acts on it has no request user to evaluate a flag against.
"""
return _flag_enabled_for(NOTEBOOKS_FRAME_STORE_CH_WRITES_FLAG, user)
def mint_callback_token(run_id: str, team_id: int) -> str:
return signing.dumps({"run_id": run_id, "team_id": team_id}, salt=_CALLBACK_TOKEN_SALT)
def verify_callback_token(token: str) -> tuple[str, int]:
"""Return (run_id, team_id) from a valid token, else raise signing.BadSignature."""
data = signing.loads(token, salt=_CALLBACK_TOKEN_SALT, max_age=_CALLBACK_TOKEN_MAX_AGE_SECONDS)
return str(data["run_id"]), int(data["team_id"])
def kernel_server_secret(runtime_id: str) -> str:
"""Per-kernel command-auth secret, derived from Django's signing key.
Only the backend can derive it (holds SECRET_KEY); a copy is written into the
sandbox at bootstrap so the kernel-server can verify command tokens. Nothing to
persist. This is the HMAC analogue of Code's RS256 connection-JWT keypair;
hardening to RS256 would let the sandbox hold only a public key.
"""
return hmac.new(
settings.SECRET_KEY.encode(),
f"nb-sql-v2-kernel:{runtime_id}".encode(),
hashlib.sha256,
).hexdigest()
def mint_command_token(secret: str, run_id: str, ttl_seconds: int = _COMMAND_TOKEN_TTL_SECONDS) -> str:
"""Sign a short-lived, run-scoped command token; `kernel.auth` verifies it in the sandbox."""
exp = int(time.time()) + ttl_seconds
signature = hmac.new(secret.encode(), f"{run_id}.{exp}".encode(), hashlib.sha256).hexdigest()
return f"{run_id}.{exp}.{signature}"
def _backend_base_url() -> str:
# The sandbox reaches the host backend here. SANDBOX_API_URL overrides (e.g. ngrok for Modal
# from local dev). In dev the kernel runs in local Docker, where the host is
# host.docker.internal (SITE_URL would be an unreachable localhost); in prod the kernel runs
# remotely and must use the public SITE_URL.
if settings.SANDBOX_API_URL:
base = settings.SANDBOX_API_URL
elif settings.DEBUG:
base = "http://host.docker.internal:8000"
else:
base = settings.SITE_URL
return base.rstrip("/")
def build_callback_url(run_id: str) -> str:
return f"{_backend_base_url()}/internal/notebooks/runs/{run_id}/result/"
def build_data_plane_url() -> str:
return f"{_backend_base_url()}/internal/notebooks/data_plane/query/"
def build_heartbeat_url() -> str:
return f"{_backend_base_url()}/api/notebooks/sandbox/heartbeat/"
def mint_data_plane_token(notebook_short_id: str, team_id: int, user_id: int | None, run_id: str | None = None) -> str:
"""Sign a data-plane token for the sandbox.
`run_id` names the run the fetch belongs to, so a data-plane query counts as observable
progress on it (see `touch_run_progress`). It rides the signed token rather than the
request body because the sandbox must not be able to name a different run than the one
the backend dispatched to it.
"""
return signing.dumps(
{"notebook_short_id": notebook_short_id, "team_id": team_id, "user_id": user_id, "run_id": run_id},
salt=_DATA_PLANE_TOKEN_SALT,
)
@frozen
class DataPlaneClaims:
notebook_short_id: str
team_id: int
user_id: int | None
# Absent on a token minted before the run-progress claim existed, so it stays optional:
# such a token still fetches data, it just cannot advance its run's watchdog clock.
run_id: str | None = None
def verify_data_plane_token(token: str) -> DataPlaneClaims:
"""Return the claims from a valid token, else raise signing.BadSignature."""
data = signing.loads(token, salt=_DATA_PLANE_TOKEN_SALT, max_age=_DATA_PLANE_TOKEN_MAX_AGE_SECONDS)
user_id = data.get("user_id")
run_id = data.get("run_id")
return DataPlaneClaims(
notebook_short_id=str(data["notebook_short_id"]),
team_id=int(data["team_id"]),
user_id=int(user_id) if user_id is not None else None,
run_id=str(run_id) if run_id else None,
)
def _find_running_runtime(notebook: Notebook, user: User | None) -> KernelRuntime | None:
runtime = (
KernelRuntime.objects.filter(
team_id=notebook.team_id,
notebook_short_id=notebook.short_id,
user=user if isinstance(user, User) else None,
)
.order_by("-last_used_at")
.first()
)
if runtime is None or not runtime.sandbox_id:
return None
if runtime.status not in (KernelRuntime.Status.RUNNING, KernelRuntime.Status.STARTING):
return None
return runtime
_COMMAND_TOKEN_HEADER = "X-Command-Token"
def _sandbox_auth_headers(connect_token: str | None, command_token: str | None = None) -> dict[str, str]:
"""Auth headers for a request into the sandbox.
The Modal connect token rides `Authorization: Bearer` — Modal's edge accepts it there
as well as in the query string, and headers stay out of proxy/access logs while URLs
do not. With that slot taken, the kernel's command token travels in its own header.
"""
headers: dict[str, str] = {}
if connect_token:
headers["Authorization"] = f"Bearer {connect_token}"
if command_token:
headers[_COMMAND_TOKEN_HEADER] = command_token
return headers
def _server_version(server_url: str, connect_token: str | None) -> str | None:
"""The deployed package hash the running server reports, or None if unreachable."""
try:
response = requests.get(
f"{server_url.rstrip('/')}/health",
headers=_sandbox_auth_headers(connect_token),
timeout=2,
)
if response.status_code != 200:
return None
return str(response.json().get("version") or "")
except (requests.RequestException, ValueError):
return None
def _wait_for_server_ready(server_url: str, connect_token: str | None, expected_version: str) -> None:
deadline = time.monotonic() + _SERVER_READY_TIMEOUT_SECONDS
while time.monotonic() < deadline:
if _server_version(server_url, connect_token) == expected_version:
return
time.sleep(0.3)
raise RuntimeError("SQLV2 kernel-server did not become ready")
# Stop a previous server via its PID file — never pkill by our own name: the
# pattern would match this very launch command's shell and kill it mid-deploy.
# The pkill lines only clear pre-package servers (distinct names, safe to
# match) from sandboxes that predate the PID file; drop them once those age out.
_STOP_PREVIOUS_SERVER = (
f"kill $(cat {_SERVER_PID_PATH} 2>/dev/null) 2>/dev/null || true; "
"pkill -f '[n]b_sql_v2_kernel_server' 2>/dev/null; pkill -f '[n]b_data_v2_kernel_server' 2>/dev/null; "
)
# Echoed by the baked launch so the caller can tell it ran from a matching image
# rather than exiting early on a stale stamp.
_BAKED_LAUNCH_MARKER = "nb_kernel_baked_ok"
def _launch_server(package_root: str, port: int, version: str) -> str:
"""Shell that backgrounds the kernel-server out of `package_root` and records its PID.
`< /dev/null` detaches the server from the exec's pipes so `execute` returns.
The backgrounded launch must stay a single simple command (PYTHONPATH= prefix,
no `cd X && …`) — a compound list backgrounds a wrapper subshell, so `$!`
would record the wrapper's PID and the next redeploy would kill nothing.
Prefer the notebook venv python (has pyarrow).
"""
return (
'PY=/opt/notebook-venv/bin/python3; [ -x "$PY" ] || PY=python3; '
f'PYTHONPATH={package_root} nohup "$PY" -m {SANDBOX_PACKAGE_NAME}.server '
f"--port {port} --secret-file {_SECRET_PATH} --version {version} "
f"> {_SERVER_LOG_PATH} 2>&1 < /dev/null & echo $! > {_SERVER_PID_PATH}"
)
def _launch_baked_kernel_server(sandbox: SandboxBase, port: int, version: str) -> bool:
"""Launch the package the image baked in, unless its stamp predates `version`.
Returns False on a stale stamp, having stopped the old server but started no
new one, so the caller falls back to the tarball. The version is a hex digest,
so it needs no shell quoting beyond the comparison's own quotes.
"""
command = (
_STOP_PREVIOUS_SERVER
+ f'[ "$(cat {BAKED_VERSION_PATH} 2>/dev/null)" = "{version}" ] || exit 0; '
+ _launch_server(BAKED_PACKAGE_ROOT, port, version)
+ f"; echo {_BAKED_LAUNCH_MARKER}"
)
result = sandbox.execute(command, timeout_seconds=30)
return _BAKED_LAUNCH_MARKER in result.stdout
def _deploy_kernel_server(sandbox: SandboxBase, runtime: KernelRuntime, package: bytes, version: str) -> None:
"""Write the secret into the sandbox and (re)launch the kernel-server.
This is the only place the control plane (write_file/execute) is used, exactly
as Code bootstraps its agent-server. Per-run dispatch is a plain authed POST.
The image bakes the package, so the usual path is one execute that launches it
in place. Uploading the tarball is the fallback for a kernel edit no image build
has picked up yet, which keeps a merged kernel fix from waiting on an image.
"""
port = _CONTAINER_PORT_BY_BACKEND.get(runtime.backend, 47821)
sandbox.write_file(_SECRET_PATH, kernel_server_secret(str(runtime.id)).encode())
if _launch_baked_kernel_server(sandbox, port, version):
return
sandbox.write_file(_TARBALL_PATH, package)
launch = (
_STOP_PREVIOUS_SERVER
+ f"rm -rf {_PACKAGE_ROOT} && mkdir -p {_PACKAGE_ROOT} && tar -xzf {_TARBALL_PATH} -C {_PACKAGE_ROOT} && "
+ _launch_server(_PACKAGE_ROOT, port, version)
)
sandbox.execute(launch, timeout_seconds=30)
def ensure_sql_v2_server(notebook: Notebook, user: User | None) -> KernelRuntime:
"""Ensure the in-sandbox kernel-server is running the current package version.
Idempotent — a healthy server at the expected version is reused as-is; a stale
or unreachable one is redeployed, from the image's baked package when that
matches and from the freshly built tarball when it does not (this is the dev
loop: edit `kernel/`, next run redeploys, no image rebuild).
"""
runtime = _find_running_runtime(notebook, user)
if runtime is None:
raise SQLV2KernelNotRunning()
package, version = kernel_package_bytes_and_hash()
if runtime.server_url and _server_version(runtime.server_url, runtime.server_connect_token) == version:
return runtime
sandbox_class = get_sandbox_class_for_backend(runtime.backend)
assert runtime.sandbox_id # _find_running_runtime only returns runtimes with a sandbox
sandbox = sandbox_class.get_by_id(runtime.sandbox_id)
_deploy_kernel_server(sandbox, runtime, package, version)
credentials = sandbox.get_connect_credentials()
_wait_for_server_ready(credentials.url, credentials.token, version)
runtime.server_url = credentials.url
runtime.server_connect_token = credentials.token
# A redeploy relaunches the kernel inside a live sandbox and keeps this row, so the frame
# snapshot would outlive the kernel that produced it — the one path where the row's
# lifetime stops tracking the kernel's. Its registrations are gone with the namespace, so
# drop the snapshot; the next run repopulates it from the new kernel's catalog.
runtime.frames = None
runtime.save(update_fields=["server_url", "server_connect_token", "frames"])
return runtime
def dispatch_sql_v2_run(
notebook: Notebook,
user: User | None,
run: NotebookNodeRun,
code: str,
node_type: str = "hogql",
output_name: str = "",
inputs: list[dict] | None = None,
variables: dict | None = None,
) -> None:
"""Dispatch a run to the in-sandbox kernel-server with a single authed HTTP POST.
Returns as soon as the server accepts (202); the result arrives via the callback. A
kernel node (python or duckdb) carries the `node`/`inputs` shape the executor consumes;
a hogql node keeps the flat `code` the capped-fetch path reads — the paths stay additive.
`variables` are notebook-level values a python node reads as globals; the executor passes
the whole `node` object through, so they need no handling of their own on the way in.
"""
runtime = ensure_sql_v2_server(notebook, user)
assert runtime.server_url # ensure_sql_v2_server always returns a runtime with a live server_url
# Record which kernel took the run: the sandbox can't name it (its secret is a one-way
# derivation of the runtime id, not the id), and letting it name one would let a sandbox
# write over another kernel's state. So the backend decides here and the callback follows.
run.kernel_runtime_id = runtime.id
run.save(update_fields=["kernel_runtime_id", "updated_at"])
command_token = mint_command_token(kernel_server_secret(str(runtime.id)), str(run.id))
user_id = user.id if isinstance(user, User) else None
payload: dict = {
"run_id": str(run.id),
"callback_url": build_callback_url(str(run.id)),
"callback_token": mint_callback_token(str(run.id), notebook.team_id),
"data_plane_url": build_data_plane_url(),
"data_plane_token": mint_data_plane_token(notebook.short_id, notebook.team_id, user_id, str(run.id)),
# Authenticated with the data-plane token above.
"heartbeat_url": build_heartbeat_url(),
"page_limit": DISPLAY_PAGE_LIMIT,
"cache_limit": RESULT_CACHE_ROWS,
}
if node_type in ("python", "duckdb"):
payload["node"] = {
"type": node_type,
"code": code,
"output_name": output_name,
"variables": variables or {},
}
payload["inputs"] = inputs or []
else:
payload["code"] = code
response = requests.post(
f"{runtime.server_url.rstrip('/')}/run",
json=payload,
headers=_sandbox_auth_headers(runtime.server_connect_token, command_token),
timeout=_RUN_POST_TIMEOUT_SECONDS,
)
response.raise_for_status()
def fetch_sql_v2_page(notebook: Notebook, user: User | None, run: NotebookNodeRun, offset: int, limit: int) -> dict:
"""Fetch one result page through the kernel — a bounded synchronous read.
A hogql run pages by re-querying its stored code through the data plane with
LIMIT/OFFSET; a kernel run (python/duckdb) pages by slicing its on-sandbox result
frame, keyed by `result_id` — its code is not a HogQL query, so no data-plane
fallback exists for it. Unlike a run this never bootstraps the server (no control
plane from a web worker); a missing or unreachable server means the user has to re-run.
"""
runtime = _find_running_runtime(notebook, user)
if runtime is None or not runtime.server_url:
raise SQLV2KernelNotRunning()
command_token = mint_command_token(kernel_server_secret(str(runtime.id)), str(run.id))
user_id = user.id if isinstance(user, User) else None
payload: dict = {
"run_id": str(run.id),
"offset": offset,
"limit": limit,
}
if run.node_type == NotebookNodeRun.NodeType.HOGQL:
payload["code"] = run.code
payload["data_plane_url"] = build_data_plane_url()
payload["data_plane_token"] = mint_data_plane_token(notebook.short_id, notebook.team_id, user_id, str(run.id))
else:
payload["result_id"] = str(run.result_id)
try:
response = requests.post(
f"{runtime.server_url.rstrip('/')}/page",
json=payload,
headers=_sandbox_auth_headers(runtime.server_connect_token, command_token),
timeout=_PAGE_POST_TIMEOUT_SECONDS,
)
except requests.RequestException as exc:
raise SQLV2KernelNotRunning() from exc
if response.status_code in (404, 401, 403):
# 404 → kernel-server predates the /page route; 401/403 → the command token expired or
# the kernel was redeployed with a new secret. All are session-level, not query errors —
# a re-run redeploys/reissues, so surface them as "re-run" (503), not a page failure.
raise SQLV2KernelNotRunning()
if response.status_code == 400:
raise SQLV2PageError(_kernel_error_detail(response))
if response.status_code != 200:
# Any other non-200 (e.g. a kernel 500) is an infrastructure problem, not a bad query.
raise SQLV2KernelNotRunning()
return response.json()
def interrupt_sql_v2_run(notebook: Notebook, user: User | None, run: NotebookNodeRun) -> bool:
"""Ask the kernel-server to interrupt a run; return whether the kernel knew the run.
False means the run never reached the kernel (dispatch still in flight) or already
finished there; the caller decides how to surface that. Raises SQLV2KernelNotRunning
when no reachable kernel exists at all, in which case the run's callback can never
arrive and the caller may mark the run terminal itself.
"""
runtime = _find_running_runtime(notebook, user)
if runtime is None or not runtime.server_url:
raise SQLV2KernelNotRunning()
command_token = mint_command_token(kernel_server_secret(str(runtime.id)), str(run.id))
try:
response = requests.post(
f"{runtime.server_url.rstrip('/')}/interrupt",
json={"run_id": str(run.id)},
headers=_sandbox_auth_headers(runtime.server_connect_token, command_token),
timeout=_INTERRUPT_POST_TIMEOUT_SECONDS,
)
except requests.RequestException as exc:
raise SQLV2KernelNotRunning() from exc
if response.status_code != 200:
raise SQLV2KernelNotRunning()
try:
body = response.json()
except ValueError:
return True
# A pre-run-scoped kernel-server omits `known`; treat its interrupt as delivered.
return bool(body.get("known", True))
def introspect_in_kernel(
notebook: Notebook, user: User | None, route: Literal["complete", "inspect"], payload: dict
) -> dict | None:
"""Ask the running kernel for completions or a name's docs; None when no kernel can answer.
Like a page fetch this never bootstraps a server: completion only makes sense against a
namespace a run already built. An older kernel-server without the route answers 404 until
the next run redeploys it.
"""
runtime = _find_running_runtime(notebook, user)
if runtime is None or not runtime.server_url:
return None
# The command token is bound to an id; an introspection request is not a run, so it gets its own.
request_id = str(uuid.uuid4())
command_token = mint_command_token(kernel_server_secret(str(runtime.id)), request_id)
try:
response = requests.post(
f"{runtime.server_url.rstrip('/')}/{route}",
json={**payload, "run_id": request_id},
headers=_sandbox_auth_headers(runtime.server_connect_token, command_token),
timeout=_INTROSPECTION_POST_TIMEOUT_SECONDS,
)
except requests.RequestException:
return None
if response.status_code != 200:
return None
try:
body = response.json()
except ValueError:
return None
return body if isinstance(body, dict) else None
def _kernel_error_detail(response: requests.Response) -> str:
try:
detail = response.json().get("error")
if isinstance(detail, str) and detail:
return detail
except ValueError:
pass
return "Page fetch failed in the sandbox."