:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
"""Eval cases for authoring notebook cells over MCP.
Both lanes a cell run can take are graded here, because they fail in different places:
a SQL cell rides the direct HogQL lane and never provisions anything, while a python
cell dispatches a Temporal workflow that provisions the notebook kernel sandbox.
To run one case:
hogli evals eval_notebook_cells --eval sql_cell_report
"""
from __future__ import annotations
from products.notebooks.evals.offline_config import offline_suite
from products.notebooks.evals.scorers import CellRunsCompleted, NotebookCreated, NotebookRunCompleted
from products.notebooks.evals.seeders import seed_case_team
from products.posthog_ai.eval_harness.base import SandboxedEval
from products.posthog_ai.eval_harness.config import SandboxedEvalCase
from products.posthog_ai.eval_harness.harness.context import EvalContext
from products.posthog_ai.eval_harness.scorers import RequiredToolCall
async def eval_notebook_cells(ctx: EvalContext) -> None:
cases: list[SandboxedEvalCase] = [
SandboxedEvalCase(
# Cheapest possible kernel-lane check: no query, no dataframe, no dependency
# between cells. When this passes and the cases below fail, the sandbox is fine
# and the agent's analysis is what broke.
name="python_cell_arithmetic",
prompt=("Create a notebook called 'Kernel smoke test'. Add a Python cell that prints the result of 1 + 2."),
expected={
"notebook_created": {},
"cell_runs_completed": {"node_types": ["python"]},
},
setup=seed_case_team,
),
SandboxedEvalCase(
name="sql_cell_report",
prompt=(
"Create a notebook called 'Signup momentum'. Add a SQL cell that returns weekly counts "
"of `signed_up` events over the last 8 weeks, and finish with a short markdown summary "
"of what the numbers show."
),
expected={
"notebook_created": {},
"cell_runs_completed": {"node_types": ["hogql"]},
},
setup=seed_case_team,
),
SandboxedEvalCase(
name="python_cell_from_dataframe",
prompt=(
"Create a notebook called 'Signup momentum'. Add a SQL cell that returns weekly counts "
"of `signed_up` events over the last 8 weeks, then add a Python cell that reads that "
"cell's dataframe and computes the week-over-week percentage change. Finish with a short "
"markdown summary naming the week that grew the most."
),
expected={
"notebook_created": {},
# Both lanes: the SQL cell proves the direct path, the python cell proves the
# kernel sandbox was provisioned, ran the code, and reported back.
"cell_runs_completed": {"node_types": ["hogql", "python"]},
},
setup=seed_case_team,
),
SandboxedEvalCase(
# The whole-notebook run: the agent has to set the window and re-run every cell
# in one step, rather than walking the cells one at a time.
name="rerun_notebook_for_a_new_window",
prompt=(
"Create a notebook called 'Signup window'. Declare a variable `days_back` set to 30, "
"add a SQL cell that counts `signed_up` events over the last `days_back` days, and add "
"a Python cell that reads that cell's dataframe and prints the total. Then change "
"`days_back` to 7 and run the whole notebook again so both cells reflect the new window."
),
expected={
"notebook_created": {},
"cell_runs_completed": {"node_types": ["hogql", "python"]},
# The point of the case. Walking the cells one at a time satisfies
# cell_runs_completed just as well, so only the run row proves the agent used
# the single call — and only its frozen variables prove the run that finished
# is the second one, under the new window.
"notebook_run_completed": {"variables": {"days_back": 7}},
},
setup=seed_case_team,
),
]
await SandboxedEval(
experiment_name="sandboxed-notebook-cells-cli",
offline_suite=offline_suite(
"notebooks/eval_notebook_cells::eval_notebook_cells",
metrics=["required_tool_call", "notebook_created", "cell_runs_completed", "notebook_run_completed"],
),
cases=cases,
scorers=[
RequiredToolCall({"notebooks-add-cell"}),
NotebookCreated(),
CellRunsCompleted(),
NotebookRunCompleted(),
],
ctx=ctx,
)