Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

4.8 KB
Raw Blame
"""Eval cases for authoring notebook cells over MCP.

Both lanes a cell run can take are graded here, because they fail in different places:
a SQL cell rides the direct HogQL lane and never provisions anything, while a python
cell dispatches a Temporal workflow that provisions the notebook kernel sandbox.

To run one case:
    hogli evals eval_notebook_cells --eval sql_cell_report
"""

from __future__ import annotations

from products.notebooks.evals.offline_config import offline_suite
from products.notebooks.evals.scorers import CellRunsCompleted, NotebookCreated, NotebookRunCompleted
from products.notebooks.evals.seeders import seed_case_team
from products.posthog_ai.eval_harness.base import SandboxedEval
from products.posthog_ai.eval_harness.config import SandboxedEvalCase
from products.posthog_ai.eval_harness.harness.context import EvalContext
from products.posthog_ai.eval_harness.scorers import RequiredToolCall


async def eval_notebook_cells(ctx: EvalContext) -> None:
    cases: list[SandboxedEvalCase] = [
        SandboxedEvalCase(
            # Cheapest possible kernel-lane check: no query, no dataframe, no dependency
            # between cells. When this passes and the cases below fail, the sandbox is fine
            # and the agent's analysis is what broke.
            name="python_cell_arithmetic",
            prompt=("Create a notebook called 'Kernel smoke test'. Add a Python cell that prints the result of 1 + 2."),
            expected={
                "notebook_created": {},
                "cell_runs_completed": {"node_types": ["python"]},
            },
            setup=seed_case_team,
        ),
        SandboxedEvalCase(
            name="sql_cell_report",
            prompt=(
                "Create a notebook called 'Signup momentum'. Add a SQL cell that returns weekly counts "
                "of `signed_up` events over the last 8 weeks, and finish with a short markdown summary "
                "of what the numbers show."
            ),
            expected={
                "notebook_created": {},
                "cell_runs_completed": {"node_types": ["hogql"]},
            },
            setup=seed_case_team,
        ),
        SandboxedEvalCase(
            name="python_cell_from_dataframe",
            prompt=(
                "Create a notebook called 'Signup momentum'. Add a SQL cell that returns weekly counts "
                "of `signed_up` events over the last 8 weeks, then add a Python cell that reads that "
                "cell's dataframe and computes the week-over-week percentage change. Finish with a short "
                "markdown summary naming the week that grew the most."
            ),
            expected={
                "notebook_created": {},
                # Both lanes: the SQL cell proves the direct path, the python cell proves the
                # kernel sandbox was provisioned, ran the code, and reported back.
                "cell_runs_completed": {"node_types": ["hogql", "python"]},
            },
            setup=seed_case_team,
        ),
        SandboxedEvalCase(
            # The whole-notebook run: the agent has to set the window and re-run every cell
            # in one step, rather than walking the cells one at a time.
            name="rerun_notebook_for_a_new_window",
            prompt=(
                "Create a notebook called 'Signup window'. Declare a variable `days_back` set to 30, "
                "add a SQL cell that counts `signed_up` events over the last `days_back` days, and add "
                "a Python cell that reads that cell's dataframe and prints the total. Then change "
                "`days_back` to 7 and run the whole notebook again so both cells reflect the new window."
            ),
            expected={
                "notebook_created": {},
                "cell_runs_completed": {"node_types": ["hogql", "python"]},
                # The point of the case. Walking the cells one at a time satisfies
                # cell_runs_completed just as well, so only the run row proves the agent used
                # the single call — and only its frozen variables prove the run that finished
                # is the second one, under the new window.
                "notebook_run_completed": {"variables": {"days_back": 7}},
            },
            setup=seed_case_team,
        ),
    ]

    await SandboxedEval(
        experiment_name="sandboxed-notebook-cells-cli",
        offline_suite=offline_suite(
            "notebooks/eval_notebook_cells::eval_notebook_cells",
            metrics=["required_tool_call", "notebook_created", "cell_runs_completed", "notebook_run_completed"],
        ),
        cases=cases,
        scorers=[
            RequiredToolCall({"notebooks-add-cell"}),
            NotebookCreated(),
            CellRunsCompleted(),
            NotebookRunCompleted(),
        ],
        ctx=ctx,
    )