:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
chore(replay): remove replay vision evals (#119865)
Radu Raicea6ae08f080827
diff --git a/.github/scripts/trunk-impacted-targets.js b/.github/scripts/trunk-impacted-targets.jsindex 821962fe7536..b37bf128921a 100644--- a/.github/scripts/trunk-impacted-targets.js+++ b/.github/scripts/trunk-impacted-targets.js@@ -364,11 +364,10 @@ const TRIPWIRE_RULES = [['.github/workflows/weekly-flaky-report.yml', REPO_AUTOMATION],['.github/workflows/weekly-slow-tests-report.yml', REPO_AUTOMATION],// More single-suite workflows, held to the trees their suites read:- // replay-vision evals and ClickHouse HCL checks are Python; the+ // ClickHouse HCL checks are Python; the// hogql parser builds wheels (python), an npm package (both families), and// a crate (rust); deltalite spans its crates and the wheel's python// consumers.- ['.github/workflows/ci-replay-vision-evals.yml', PYTHON],['.github/workflows/ci-clickhouse-hcl-schema.yml', PYTHON],['.github/workflows/build-hogql-parser.yml', PYTHON],['.github/workflows/build-hogql-parser-npm.yml', FULLSTACK],diff --git a/.github/workflows/ci-replay-vision-evals.yml b/.github/workflows/ci-replay-vision-evals.ymldeleted file mode 100644index 7d02dbd3d5c6..000000000000--- a/.github/workflows/ci-replay-vision-evals.yml+++ /dev/null@@ -1,202 +0,0 @@-# Golden-dataset eval for Replay Vision scanner prompts (products/replay_vision/evals).-#-# Manual dispatch only, deliberately. The suite re-runs production scans over real dogfood session-# recordings, so a run spends secrets and LLM calls, and the number it returns moves on its own:-# Gemini is nondeterministic and the dataset is re-sampled per run. That makes the score a-# directional signal rather than a merge gate, which is not worth attaching to every push.-#-# The dataset is collected fresh on the ephemeral runner each run, which re-verifies AI-# data-processing consent; it is never cached, uploaded as an artifact, or otherwise persisted.-# Only aggregate scores reach the public job log and step summary.-# If any required secret is missing the job skips green with a warning, so a fork or a-# not-yet-configured environment is unaffected.-name: Replay Vision Evals--on:- workflow_dispatch:- inputs:- per_type:- description: 'Cases to collect per scanner type'- default: '5'- type: string- trials:- description: 'Trials per case, for variance on Gemini nondeterminism'- default: '1'- type: string--permissions:- contents: read--env:- DATASET_DIR: /tmp/replay-vision-golden-dataset--jobs:- golden-eval:- name: Golden-dataset scanner eval- timeout-minutes: 60- runs-on: ubuntu-24.04- env:- DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USER }}- DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}-- steps:- - name: Check required secrets- id: gate- # All five up front: one rotated key must not boot the Docker stack and collect a- # dataset before failing inside the eval.- env:- REPLAY_VISION_EVAL_POSTHOG_API_KEY: ${{ secrets.REPLAY_VISION_EVAL_POSTHOG_API_KEY }}- GEMINI_OFFLINE_EVALS_API_KEY: ${{ secrets.GEMINI_OFFLINE_EVALS_API_KEY }}- OPENAI_OFFLINE_EVALS_API_KEY: ${{ secrets.OPENAI_OFFLINE_EVALS_API_KEY }}- ANTHROPIC_OFFLINE_EVALS_API_KEY: ${{ secrets.ANTHROPIC_OFFLINE_EVALS_API_KEY }}- BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}- run: |- missing=""- for name in REPLAY_VISION_EVAL_POSTHOG_API_KEY GEMINI_OFFLINE_EVALS_API_KEY OPENAI_OFFLINE_EVALS_API_KEY ANTHROPIC_OFFLINE_EVALS_API_KEY BRAINTRUST_API_KEY; do- [ -n "${!name}" ] || missing="$missing $name"- done- if [ -n "$missing" ]; then- echo "enabled=false" >> "$GITHUB_OUTPUT"- echo "::warning::Missing secrets:$missing; skipping the golden-dataset eval"- else- echo "enabled=true" >> "$GITHUB_OUTPUT"- fi-- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2- if: steps.gate.outputs.enabled == 'true'-- - name: Log in to Docker Hub- continue-on-error: true- if: steps.gate.outputs.enabled == 'true' && env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != ''- uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0- with:- username: ${{ vars.DOCKERHUB_USER }}- password: ${{ secrets.DOCKERHUB_TOKEN }}-- - name: Start stack with Docker Compose- if: steps.gate.outputs.enabled == 'true'- env:- COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml- run: bin/ci-wait-for-docker launch --down-- - name: Wait for Docker services- if: steps.gate.outputs.enabled == 'true'- env:- COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml- run: bin/ci-wait-for-docker wait-- - name: Set up Python- if: steps.gate.outputs.enabled == 'true'- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0- with:- python-version-file: 'pyproject.toml'-- - name: Install uv- if: steps.gate.outputs.enabled == 'true'- uses: ./.github/actions/setup-uv- with:- save-cache: false-- - name: Install python dependencies- if: steps.gate.outputs.enabled == 'true'- shell: bash- run: UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev-- - name: Install Rust- if: steps.gate.outputs.enabled == 'true'- uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9- with:- toolchain: 1.91.1- components: cargo-- - name: Install protoc- # A one-shot suite builds the personhog binaries during preflight, and their proto- # crate compiles .proto files in a build script. The runner image has no protoc.- if: steps.gate.outputs.enabled == 'true'- uses: ./.github/actions/setup-protoc-- - name: Install sqlx-cli- if: steps.gate.outputs.enabled == 'true'- uses: ./.github/actions/setup-sqlx-cli-- - name: Add service hostnames to /etc/hosts- if: steps.gate.outputs.enabled == 'true'- run: sudo echo "127.0.0.1 db redis7 kafka clickhouse clickhouse-coordinator objectstorage seaweedfs temporal" | sudo tee -a /etc/hosts-- - name: Collect golden dataset- if: steps.gate.outputs.enabled == 'true'- shell: bash- env:- POSTHOG_API_KEY: ${{ secrets.REPLAY_VISION_EVAL_POSTHOG_API_KEY }}- # Django refuses to boot with the checked-in SECRET_KEY unless DEBUG is set.- DEBUG: '1'- # Through env, not inline ${{ }}, so a dispatch input can never inject into the shell.- PER_TYPE: ${{ inputs.per_type }}- run: |- # To the runner-local file only, never the public job log: the collector emits- # session and observation ids for missing-event and failed cases.- python -m products.replay_vision.evals.collect \- --project-id 2 \- --per-type "$PER_TYPE" \- --output "$DATASET_DIR" > /tmp/collect-output.log 2>&1 || {- status=$?- echo "collect exited with status $status; collector output stays off the public log"- exit "$status"- }- # An empty dataset (expired assets, warehouse lag) must fail loudly, not score green over nothing.- python - <<'EOF'- import json, os, pathlib, sys- cases = json.loads((pathlib.Path(os.environ["DATASET_DIR"]) / "manifest.json").read_text())["cases"]- print(f"{len(cases)} cases in the collected dataset")- sys.exit(1 if not cases else 0)- EOF-- - name: Run scanner quality suite- if: steps.gate.outputs.enabled == 'true'- shell: bash- env:- REPLAY_VISION_EVAL_DATASET: ${{ env.DATASET_DIR }}- GEMINI_API_KEY: ${{ secrets.GEMINI_OFFLINE_EVALS_API_KEY }}- BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}- # The one-shot preflight name for a plain Anthropic key (used directly, no gateway).- LLM_GATEWAY_ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_OFFLINE_EVALS_API_KEY }}- # The summary_alignment judge (autoevals) calls its model with the OpenAI client.- OPENAI_API_KEY: ${{ secrets.OPENAI_OFFLINE_EVALS_API_KEY }}- TRIALS: ${{ inputs.trials }}- run: |- # To the runner-local file only, never the public job log: the eval calls run_scan- # directly, without the activity-level exception sanitizer, so a provider error can- # echo prompt and session-derived request content. The summary step publishes the- # safe aggregate lines.- python -m products.posthog_ai.eval_harness.harness eval_scanner_quality \- --trials "$TRIALS" > /tmp/eval-output.log 2>&1-- - name: Publish score summary- # Publish whatever the run produced, including on failure; the suite is private, so- # scores exist only in this runner's local logs.- if: always() && steps.gate.outputs.enabled == 'true'- shell: bash- run: |- # tee: the same sanitized lines double as the job log's only eval output.- {- echo "## Replay Vision scanner eval"- echo ""- echo "Scores compare fresh scans against recorded outputs and human thumbs labels."- echo "Directional signal only: Gemini is nondeterministic and the dataset is re-sampled per run."- echo ""- echo '```'- # Only the stable aggregate lines: this summary is public, and case payloads- # (session-derived content) must never leave the runner even if the harness's- # console output grows. CASE lines stay out too: their names carry observation ids.- # The indented PostHog line links the /ai-evals experiment the harness published- # this run's scores to. It is safe to publish: the URL carries the project id and- # a per-run random experiment UUID, nothing case-derived.- grep -E '^(SUITE|EXPERIMENT DONE|Sandboxed eval summary|Status:|Suites:|Cases:|Mean score:|Score gate:|Duration:| +PostHog: )' \- /tmp/eval-output.log 2>/dev/null || echo "no eval output produced"- echo '```'- } | tee -a "$GITHUB_STEP_SUMMARY"-- - name: Remove dataset- # The runner is ephemeral anyway; this just makes the no-persistence guarantee explicit.- if: always()- run: rm -rf "$DATASET_DIR"diff --git a/products/replay_vision/backend/temporal/activities/call_scanner_provider.py b/products/replay_vision/backend/temporal/activities/call_scanner_provider.pyindex fd9e694e524b..ffb709104232 100644--- a/products/replay_vision/backend/temporal/activities/call_scanner_provider.py+++ b/products/replay_vision/backend/temporal/activities/call_scanner_provider.py@@ -267,12 +267,9 @@ async def run_scan() -> ScannerCallOutput:"""Run the scanner conversation over an already-uploaded video, independent of where the inputs came from.- The activity path above loads the snapshot and inputs from the observation row and Redis; the golden-dataset- eval suite (products/replay_vision/evals) feeds this same function from files on disk so prompt changes are- tested against the exact production pipeline. `scanner` is required (no snapshot fallback) so no caller can+ `scanner` is required (no snapshot fallback) so no caller cansilently skip the known-freeform-tags injection. The production activity checks AI data-processing consent- before calling this; any other caller must do the same before recording data reaches the provider (the eval- suite is covered because dataset collection is consent-gated and time-boxed).+ before calling this; any other caller must do the same before recording data reaches the provider."""# Built before the preamble so one object decides both the wording and what a network lookup returns, which# keeps the prompt from promising network data the scan does not carry.diff --git a/products/replay_vision/evals/README.md b/products/replay_vision/evals/README.mddeleted file mode 100644index 66bd62afb511..000000000000--- a/products/replay_vision/evals/README.md+++ /dev/null@@ -1,84 +0,0 @@-# Replay Vision golden-dataset evals--Test scanner prompt changes against a fixed set of real, already-observed sessions instead of shipping and watching production.-The suite re-runs the exact production scan pipeline (`run_scan`: same Jinja templates, response schemas, lookup round) over collected videos, then scores the fresh output against the recorded output and its human thumbs label.--## The loop--1. Collect a dataset once (and re-collect occasionally as labels accumulate).-2. Run the suite to get a baseline.-3. Edit prompt templates under `../backend/temporal/scanners/prompts/`.-4. Re-run the suite and compare scores across runs (same experiment name accumulates history).--## Collecting a dataset--Uses only the public API plus the dogfood warehouse, with a personal API key that can read the source project (scanner read + session recording read + query scopes):--```bash-POSTHOG_API_KEY=... python -m products.replay_vision.evals.collect \- --project-id 2 --per-type 25 --output ~/.posthog/replay-vision-golden-dataset-```--Selection is per scanner type: human-labeled observations first (thumbs up/down carry ground truth), then a seeded uniform sample of unlabeled ones.-Only observations whose original rasterized MP4 still exists are collectable: the system assets expire after 90 days, a session re-rasterized after that is a different video than the one behind the recorded output (so those are skipped too), and the warehouse copy of the exports table lags up to a day, so very fresh observations are skipped.-Each case also captures the prompt context the production scan carried: the project's product context (Max core memory needs a full-access personal key; otherwise the project description is used), the session's custom-event descriptions, and, for freeform classifiers, the known-tag vocabulary.-Re-running the collector is idempotent per case: completed cases are reused as-is, half-written ones are re-collected, and the manifest is merged with previously collected cases that are still valid on disk.-After a collector change that alters what a case captures, collect into a fresh directory so reused cases don't keep the old shape.--## Running the suite--```bash-REPLAY_VISION_EVAL_DATASET=~/.posthog/replay-vision-golden-dataset \-GEMINI_API_KEY=... \-LLM_GATEWAY_ANTHROPIC_API_KEY=... \-BRAINTRUST_API_KEY=... \-hogli evals eval_scanner_quality-```--`GEMINI_API_KEY` runs the scans themselves.-`LLM_GATEWAY_ANTHROPIC_API_KEY` satisfies the suite env preflight (judge models are routed through the internal LLM gateway).-`BRAINTRUST_API_KEY` is required by the harness engine even though this private suite only logs locally.-Without `REPLAY_VISION_EVAL_DATASET` the suite logs a warning and runs nothing, so it never breaks a full `hogli evals` run.-Use `--trials N` for variance on Gemini nondeterminism and `--eval <case-substring>` for one case.--## Scorers--| Scorer | Applies to | Meaning |-| ------------------- | ---------------------------------------- | ----------------------------------------------------------------------------- |-| `scan_completed` | all cases | The scan produced schema-valid, semantically-valid output. |-| `labeled_outcome` | labeled monitor/classifier | kept/fixed = 1, regressed/still_wrong = 0 |-| `output_stability` | unlabeled monitor/classifier | Fresh outcome matches the recorded baseline; measures churn, not correctness. |-| `score_alignment` | scorer (reference not thumbs-downed) | 1 minus the scale-normalized distance from the recorded score. |-| `summary_alignment` | summarizer (reference not thumbs-downed) | LLM judge: does the fresh summary tell the same story as the recorded one? |--`output_stability` and `labeled_outcome` deliberately pull in opposite directions, so read them as a pair: a prompt change that only adds churn shows up as `labeled_outcome` flat or up while `output_stability` drops.--Primary verdict agreement and signal counts do not measure recommendation precision.-The label scorer compares primary outcomes; it does not check each signal's evidence or usefulness.-Model confidence does not establish whether a signal supports a useful recommendation.-Successful scans retain the complete structured `signals` list in the private per-case output alongside `signals_count`; no findings produces `signals: []`.-Compare each finding's type, time range, URL, description, and confidence with the recording before using it as recommendation evidence.-This suite does not calculate recommendation precision.--## Running it on a GitHub runner--`.github/workflows/ci-replay-vision-evals.yml` runs the same loop on an ephemeral runner, on manual dispatch only, with `per_type` and `trials` as inputs (GitHub only offers the dispatch once the workflow is on master).-It is deliberately not attached to `pull_request`: Gemini is nondeterministic and the dataset is re-sampled per run, so the score is a directional signal rather than a merge gate, and spending secrets and LLM calls on every push buys nothing that a dispatch after a prompt change does not.--It collects a fresh dataset on the runner (so consent is re-verified every run; nothing is cached, uploaded, or persisted), runs the suite, and writes the aggregate scores to the job's step summary.-Only those allowlisted summary lines are public: collector and harness output stay in runner-local files, because they carry session and observation ids.-The job needs `REPLAY_VISION_EVAL_POSTHOG_API_KEY` (a personal API key with scanner, session recording, export, and query read access to the dogfood project) plus the `GEMINI_OFFLINE_EVALS_API_KEY`, `OPENAI_OFFLINE_EVALS_API_KEY`, `ANTHROPIC_OFFLINE_EVALS_API_KEY`, and `BRAINTRUST_API_KEY` secrets; if any is missing the job skips green and warns which.-The dedicated provider secrets are exposed to the suite as `GEMINI_API_KEY`, `OPENAI_API_KEY`, and `LLM_GATEWAY_ANTHROPIC_API_KEY`.--## Data handling--The dataset contains real session recordings and event data.--- Keep it in a local or internal location only; never commit it, upload it, or reference its contents in PRs.-- A dataset expires 30 days after its last collection: the suite refuses to run it, because the consent verification (and the recordings themselves) can lapse after collection. Re-running collect.py re-verifies consent and refreshes the manifest.-- The suite is `OneShotPrivateEval`, so per-case logs stay in the local `eval_harness/logs/` directory and nothing goes to Braintrust.-- The retained `signals` can contain session content. Keep these payloads private; never include them in public CI summaries, commits, or pull requests.-- Dataset-derived content reaches two model providers: the Gemini scans use the same provider call production makes through `run_scan`, and the `summary_alignment` judge sends the recorded and fresh summaries to `gpt-5.4`.-- The judge calls the OpenAI API directly with `OPENAI_API_KEY`. It does not go through the internal LLM gateway, which a one-shot suite never starts, so nothing sits in front of the provider on that path.-- This private suite does not publish scores to PostHog's offline experiments. Results remain in local logs and the aggregate CI summary.-- Collecting from projects other than PostHog's own requires a data-governance decision first; see the product's consent gating (`backend/consent.py`) and note that customer-facing copy does not cover internal reuse today.diff --git a/products/replay_vision/evals/__init__.py b/products/replay_vision/evals/__init__.pydeleted file mode 100644index 37fea886e23c..000000000000--- a/products/replay_vision/evals/__init__.py+++ /dev/null@@ -1 +0,0 @@-"""Replay Vision golden-dataset eval suites (see README.md in this directory)."""diff --git a/products/replay_vision/evals/collect.py b/products/replay_vision/evals/collect.pydeleted file mode 100644index 04e40803fa36..000000000000--- a/products/replay_vision/evals/collect.py+++ /dev/null@@ -1,70 +0,0 @@-"""Collect a Replay Vision golden dataset from an existing project's observations.--Usage, from the repo root, with a personal API key that can read the source project:-- POSTHOG_API_KEY=... python -m products.replay_vision.evals.collect \\- --project-id 2 --per-type 25 --output ~/.posthog/replay-vision-golden-dataset--Then point REPLAY_VISION_EVAL_DATASET at the output directory to run the eval suite.-The dataset contains real session data: keep it out of the repo and off any public surface.-"""--import os-import argparse-from pathlib import Path--import django--_DEFAULT_OUTPUT = "~/.posthog/replay-vision-golden-dataset"---def main() -> None:- parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)- parser.add_argument("--host", default="https://us.posthog.com", help="PostHog instance to collect from.")- parser.add_argument("--project-id", type=int, default=2, help="Source project (default: PostHog dogfood).")- parser.add_argument("--output", default=_DEFAULT_OUTPUT, help="Dataset directory to write.")- parser.add_argument("--per-type", type=int, default=25, help="Cases to collect per scanner type.")- parser.add_argument(- "--scanner-id", action="append", dest="scanner_ids", help="Restrict to specific scanner ids (repeatable)."- )- parser.add_argument("--seed", type=int, default=42, help="Seed for the uniform sample, for reproducibility.")- parser.add_argument(- "--max-observations",- type=int,- default=200,- help="Newest observations to consider per scanner; lower paginates much faster.",- )- args = parser.parse_args()-- # Env var only, deliberately: a key passed as a CLI flag leaks into shell history and ps output.- api_key = os.environ.get("POSTHOG_API_KEY", "")- if not api_key:- parser.error("set POSTHOG_API_KEY")-- os.environ.setdefault("DJANGO_SETTINGS_MODULE", "posthog.settings")- django.setup()- from products.replay_vision.evals.collector import collect # noqa: PLC0415 - needs Django configured first-- dataset = collect(- host=args.host,- project_id=args.project_id,- api_key=api_key,- output=Path(args.output).expanduser(),- per_type=args.per_type,- scanner_ids=args.scanner_ids,- seed=args.seed,- max_observations_per_scanner=args.max_observations,- )- by_type: dict[str, int] = {}- for case in dataset.cases:- by_type[case.scanner_type] = by_type.get(case.scanner_type, 0) + 1- labeled = sum(1 for case in dataset.cases if case.label_is_correct is not None)- print(f"Collected {len(dataset.cases)} cases ({labeled} labeled): {by_type}") # noqa: T201- print( # noqa: T201- f"Run the suite with: REPLAY_VISION_EVAL_DATASET={args.output} GEMINI_API_KEY=... "- "LLM_GATEWAY_ANTHROPIC_API_KEY=... BRAINTRUST_API_KEY=... hogli evals eval_scanner_quality"- )---if __name__ == "__main__":- main()diff --git a/products/replay_vision/evals/collector.py b/products/replay_vision/evals/collector.pydeleted file mode 100644index 1aaf53e3cd33..000000000000--- a/products/replay_vision/evals/collector.py+++ /dev/null@@ -1,607 +0,0 @@-"""Implementation of the golden-dataset collector; run it via collect.py (needs Django configured).--Everything goes through a PostHog instance's public API with a personal API key, so it works-against any project the key can read: the replay-vision scanner/observation endpoints drive the-selection, the synced ``postgres.posthog_exportedasset`` warehouse table locates each session's-rasterized MP4 (system assets are invisible to the exports list endpoint), the exports content-endpoint downloads the bytes, and HogQL queries mirroring ``fetch_session_events`` rebuild each-session's ``ScannerLlmInputs``. The core-memory, project, and event-definition endpoints rebuild-the prompt context (``product_context``, ``event_descriptions``) the production preamble carries.-"""--import os-import json-import random-import datetime as dt-from collections.abc import Iterator-from dataclasses import field-from pathlib import Path-from typing import Any--import requests-import structlog--from posthog.dataclasses import frozen-from posthog.session_recordings.queries.session_replay_events import DEFAULT_EVENT_FIELDS--from products.replay_vision.backend.queries.session_identity import (- IDENTITY_TIMESTAMP_SLACK,- SESSION_PERSON_IDENTITY_QUERY,- person_display_name,- person_email,- person_organization,- person_properties_from_row,-)-from products.replay_vision.backend.temporal.activities.call_scanner_provider import (- _KNOWN_FREEFORM_TAGS_DAYS,- _KNOWN_FREEFORM_TAGS_MAX_ROWS,- rank_freeform_tags,-)-from products.replay_vision.backend.temporal.activities.ensure_session_asset import (- _EXPORT_FORMAT,- _MOUSE_TAIL,- _PLAYBACK_SPEED,- _RECORDING_FPS,- _SHOW_METADATA_FOOTER,-)-from products.replay_vision.backend.temporal.activities.fetch_session_events import (- _EVENTS_PER_PAGE,- _EVENTS_TO_IGNORE,- _EXTRA_FIELDS,- _MAX_TOTAL_EVENT_ROWS,- _process_events,-)-from products.replay_vision.backend.temporal.team_context import (- sanitize_product_context,- select_event_descriptions,- session_custom_event_names,-)-from products.replay_vision.backend.temporal.types import (- EventTable,- ScannerLlmInputs,- ScannerSnapshot,- SessionIdentity,- SessionMetadata,-)-from products.replay_vision.evals.dataset import (- MANIFEST_NAME,- GoldenCase,- GoldenDataset,- load_dataset,- parse_utc,- save_dataset,-)--logger = structlog.get_logger(__name__)--# The exports table syncs into the warehouse with up to ~a day of lag, so freshly-created-# observations often have no findable asset yet; we over-fetch candidates and skip those.-_ASSET_LOOKUP_CHUNK = 100-_OBSERVATION_PAGE = 100-_EVENT_DEFINITION_NAMES_CHUNK = 100---class PostHogApi:- """Minimal authenticated client for the endpoints the collector needs."""-- def __init__(self, host: str, project_id: int, api_key: str) -> None:- self.host = host.rstrip("/")- self.project_id = project_id- self._session = requests.Session()- self._session.headers["Authorization"] = f"Bearer {api_key}"-- def get_json(self, path: str, params: dict[str, Any] | None = None) -> dict[str, Any]:- response = self._session.get(f"{self.host}{path}", params=params, timeout=60)- response.raise_for_status()- return response.json()-- def paginate(- self, path: str, params: dict[str, Any] | None = None, max_items: int | None = None- ) -> Iterator[dict[str, Any]]:- url: str | None = f"{self.host}{path}"- yielded = 0- while url:- response = self._session.get(url, params=params, timeout=60)- response.raise_for_status()- data = response.json()- for item in data.get("results", []):- yield item- yielded += 1- if max_items is not None and yielded >= max_items:- return- url = data.get("next")- params = None-- def hogql(self, query: str, values: dict[str, Any] | None = None) -> list[list[Any]]:- payload = {"query": {"kind": "HogQLQuery", "query": query, "values": values or {}}}- response = self._session.post(- f"{self.host}/api/environments/{self.project_id}/query/", json=payload, timeout=120- )- response.raise_for_status()- return response.json().get("results") or []-- def download(self, path: str, target: Path) -> None:- # The content endpoint 302s to a presigned S3 URL; requests drops the auth header on the- # cross-host redirect, which is exactly right for a presigned link.- partial = target.with_name(target.name + ".partial")- with self._session.get(f"{self.host}{path}", timeout=300, stream=True) as response:- response.raise_for_status()- with partial.open("wb") as handle:- # iter_content applies the transport decoding (gzip/deflate) that response.raw skips.- for chunk in response.iter_content(chunk_size=1 << 20):- handle.write(chunk)- # Publish atomically so an interrupted download can never be mistaken for a complete file.- os.replace(partial, target)---def order_candidates(candidates: list[dict[str, Any]], rng: random.Random) -> dict[str, list[dict[str, Any]]]:- """Per scanner type: labeled observations first (they carry ground truth), then the rest uniformly shuffled."""- by_type: dict[str, list[dict[str, Any]]] = {}- for candidate in candidates:- by_type.setdefault(candidate["scanner_type"], []).append(candidate)- ordered: dict[str, list[dict[str, Any]]] = {}- for scanner_type, group in by_type.items():- labeled = [c for c in group if c["observation"].get("label")]- unlabeled = [c for c in group if not c["observation"].get("label")]- rng.shuffle(unlabeled)- ordered[scanner_type] = labeled + unlabeled- return ordered---@frozen-class _VideoAsset:- asset_id: int- created_at: dt.datetime- # Absent on assets rendered before the rasterizer recorded its cut map.- inactivity_periods: list[dict[str, Any]] = field(default_factory=list)- # Absent on assets rendered before the renderer recorded the page and tab changes it shows.- navigation: list[dict[str, Any]] = field(default_factory=list)---def _load_json_list(raw: Any) -> list[dict[str, Any]]:- """A list the rasterizer wrote into `export_context`, empty on assets rendered before it wrote it."""- if not raw:- return []- try:- parsed = json.loads(raw)- except (TypeError, ValueError):- return []- return parsed if isinstance(parsed, list) else []---def lookup_video_assets(api: PostHogApi, team_id: int, session_ids: list[str]) -> dict[str, _VideoAsset]:- """Map session id to its system rasterized-MP4 asset, for sessions that still have one.-- min(id) mirrors ensure_session_asset's oldest-first get-or-create, so this is the asset the- production scans actually watched, for as long as it lives.- """- found: dict[str, _VideoAsset] = {}- for start in range(0, len(session_ids), _ASSET_LOOKUP_CHUNK):- chunk = session_ids[start : start + _ASSET_LOOKUP_CHUNK]- rows = api.hogql(- """- SELECT JSONExtractString(toString(export_context), 'session_recording_id') AS sid,- min(id) AS asset_id,- argMin(toTimeZone(created_at, 'UTC'), id) AS asset_created_at,- argMin(JSONExtractString(toString(export_context), 'inactivity_periods'), id) AS inactivity_periods,- argMin(JSONExtractRaw(toString(export_context), 'navigation'), id) AS navigation- FROM postgres.posthog_exportedasset- WHERE team_id = {team_id}- AND export_format = {export_format}- AND is_system- AND expires_after > now()- AND JSONExtractFloat(toString(export_context), 'playback_speed') = {playback_speed}- AND JSONExtractInt(toString(export_context), 'recording_fps') = {recording_fps}- AND JSONExtractBool(toString(export_context), 'show_metadata_footer') = {show_metadata_footer}- AND JSONExtractBool(toString(export_context), 'mouse_tail') = {mouse_tail}- AND JSONExtractString(toString(export_context), 'session_recording_id') IN {session_ids}- GROUP BY sid- """,- {- "team_id": team_id,- "export_format": _EXPORT_FORMAT,- "playback_speed": _PLAYBACK_SPEED,- "recording_fps": _RECORDING_FPS,- "show_metadata_footer": _SHOW_METADATA_FOOTER,- "mouse_tail": _MOUSE_TAIL,- "session_ids": chunk,- },- )- for sid, asset_id, created_raw, periods_raw, navigation_raw in rows:- found[str(sid)] = _VideoAsset(- asset_id=int(asset_id),- created_at=parse_utc(created_raw),- inactivity_periods=_load_json_list(periods_raw),- navigation=_load_json_list(navigation_raw),- )- return found---def asset_is_recorded_video(asset: _VideoAsset, observation: dict[str, Any]) -> bool:- """An asset created after the observation completed is a re-rasterization from after the original- expired: not the video behind recorded_output, so the case would score the wrong footage."""- completed_raw = observation.get("completed_at")- return bool(completed_raw) and asset.created_at <= parse_utc(completed_raw)---# toTimeZone pins the timestamps to UTC regardless of the project timezone, so parse_utc never-# sees a value it would have to guess a timezone for.-_METADATA_QUERY = """-SELECT distinct_id, toTimeZone(start_time, 'UTC') AS start_time, toTimeZone(end_time, 'UTC') AS end_time,- click_count, keypress_count, mouse_activity_count,- active_milliseconds, console_error_count, first_url-FROM session_replay_events-WHERE session_id = {session_id}-"""---@frozen-class _SessionEvents:- columns: list[str]- rows: list[list[Any]]- truncated: bool---def _fetch_events(api: PostHogApi, session_id: str, start: dt.datetime, end: dt.datetime) -> _SessionEvents:- fields = [*DEFAULT_EVENT_FIELDS, *_EXTRA_FIELDS]- select_exprs = [f"toTimeZone({field}, 'UTC') AS timestamp" if field == "timestamp" else field for field in fields]- # The uuid tie-break makes OFFSET paging deterministic when many events share a timestamp.- query = (- f"SELECT {', '.join(select_exprs)} FROM events"- " WHERE timestamp >= {start_time} AND timestamp <= {end_time} AND $session_id = {session_id}"- " AND event NOT IN {events_to_ignore}"- " ORDER BY timestamp ASC, uuid ASC LIMIT {limit} OFFSET {offset}"- )- rows: list[list[Any]] = []- truncated = False- offset = 0- while True:- page = api.hogql(- query,- {- # Same 100s wiggle as production get_events_query: the range only bounds the scan.- "start_time": (start - dt.timedelta(seconds=100)).isoformat(),- "end_time": (end + dt.timedelta(seconds=100)).isoformat(),- "session_id": session_id,- "events_to_ignore": _EVENTS_TO_IGNORE,- "limit": _EVENTS_PER_PAGE,- "offset": offset,- },- )- rows.extend(list(row) for row in page)- if len(rows) >= _MAX_TOTAL_EVENT_ROWS:- # Same cap as the production fetch: eligibility bounds active seconds, not event count,- # so an instrumentation loop can still emit millions of rows.- truncated = len(rows) > _MAX_TOTAL_EVENT_ROWS or len(page) == _EVENTS_PER_PAGE- del rows[_MAX_TOTAL_EVENT_ROWS:]- break- if len(page) < _EVENTS_PER_PAGE:- break- offset += _EVENTS_PER_PAGE- return _SessionEvents(columns=list(fields), rows=rows, truncated=truncated)---class EventDescriptionLookup:- """API-backed twin of team_context.fetch_event_descriptions, cached across sessions in one run."""-- def __init__(self, api: PostHogApi) -> None:- self._api = api- self._cache: dict[str, str] = {}-- def for_session(self, columns: list[str], rows: list[list[Any]]) -> dict[str, str]:- names = session_custom_event_names(columns, rows)- missing = [name for name in names if name not in self._cache]- for start in range(0, len(missing), _EVENT_DEFINITION_NAMES_CHUNK):- chunk = missing[start : start + _EVENT_DEFINITION_NAMES_CHUNK]- # Pre-cache the whole chunk as undescribed so names the API doesn't return are not refetched.- self._cache.update(dict.fromkeys(chunk, ""))- for definition in self._api.paginate(- f"/api/projects/{self._api.project_id}/event_definitions/",- {"names": ",".join(chunk), "limit": _EVENT_DEFINITION_NAMES_CHUNK},- ):- self._cache[str(definition.get("name") or "")] = str(definition.get("description") or "")- return select_event_descriptions(names, self._cache)---def fetch_product_context_via_api(api: PostHogApi) -> str:- """Product context the way production builds it: Max core memory, then the project's product description."""- text = ""- try:- results = api.get_json(f"/api/projects/{api.project_id}/core_memory/").get("results") or []- text = str(results[0].get("text") or "").strip() if results else ""- except requests.HTTPError as exc:- # Core memory is an INTERNAL-scope endpoint, unreadable with a scoped personal key; degrade to the fallback.- logger.warning("collector.core_memory_unreadable", error=str(exc))- if not text:- text = str(api.get_json(f"/api/projects/{api.project_id}/").get("product_description") or "").strip()- return sanitize_product_context(text)---def build_llm_inputs(- api: PostHogApi,- team_id: int,- session_id: str,- *,- product_context: str = "",- event_descriptions: EventDescriptionLookup | None = None,-) -> ScannerLlmInputs | None:- """Rebuild the ScannerLlmInputs production stashed in Redis, from the query API instead of ClickHouse."""- metadata_rows = api.hogql(_METADATA_QUERY, {"session_id": session_id})- if not metadata_rows:- return None- (distinct_id, start_raw, end_raw, clicks, keypresses, mouse, active_ms, console_errors, first_url) = metadata_rows[- 0- ]- start, end = parse_utc(start_raw), parse_utc(end_raw)- duration_seconds = (end - start).total_seconds()-- fetched = _fetch_events(api, session_id, start, end)- if not fetched.rows:- return None- # _process_events needs real datetimes to compute per-event offsets from session start.- timestamp_index = fetched.columns.index("timestamp")- for row in fetched.rows:- row[timestamp_index] = parse_utc(row[timestamp_index])- # HogQL surfaces `properties.$window_id` as a bare `$window_id` column, matching production's runner output.- columns = [column.removeprefix("properties.") for column in fetched.columns]- processed = _process_events(columns, fetched.rows, session_start=start)-- active_seconds = float(active_ms or 0) / 1000- return ScannerLlmInputs(- session_id=session_id,- team_id=team_id,- events_truncated=fetched.truncated,- events=EventTable(columns=processed.columns, rows=processed.rows),- url_mapping=processed.url_mapping,- window_mapping=processed.window_mapping,- event_timestamps=processed.event_timestamps,- product_context=product_context,- event_descriptions=event_descriptions.for_session(processed.columns, processed.rows)- if event_descriptions- else {},- distinct_id=str(distinct_id) if distinct_id else None,- identity=_build_identity(api, session_id, str(distinct_id) if distinct_id else None, start, end),- metadata=SessionMetadata(- start_time=start,- end_time=end,- duration_seconds=duration_seconds,- active_seconds=active_seconds,- inactive_seconds=max(0.0, duration_seconds - active_seconds),- click_count=clicks,- keypress_count=keypresses,- mouse_activity_count=mouse,- start_url=first_url or None,- console_error_count=console_errors,- ),- )---def _build_identity(- api: PostHogApi, session_id: str, distinct_id: str | None, start: dt.datetime, end: dt.datetime-) -> SessionIdentity:- """The identity production renders into the preamble, read through the query API.-- Runs the same person query production runs, bound to the same subject, so a collected case exercises the- same prompt. Group names are left out: they need the project's group-type labels, and a case without them- still renders the block.- """- if not distinct_id:- return SessionIdentity()- rows = api.hogql(- SESSION_PERSON_IDENTITY_QUERY,- {- "session_id": session_id,- "distinct_id": distinct_id,- "start": (start - IDENTITY_TIMESTAMP_SLACK).isoformat(),- "end": (end + IDENTITY_TIMESTAMP_SLACK).isoformat(),- },- )- if not rows:- return SessionIdentity()- properties = person_properties_from_row(rows[0])- return SessionIdentity(- person_email=person_email(properties),- person_name=person_display_name(properties),- person_organization=person_organization(properties),- )---def _known_freeform_tags(observations: list[dict[str, Any]]) -> list[str]:- """Approximate the tag vocabulary production injects at scan time (recent window, ranked by frequency);- the vocabulary each recorded scan actually saw is not reconstructable after the fact."""- cutoff = dt.datetime.now(dt.UTC) - dt.timedelta(days=_KNOWN_FREEFORM_TAGS_DAYS)- recent = [- ((observation.get("scanner_result") or {}).get("model_output") or {}).get("tags_freeform")- for observation in observations- if observation.get("created_at") and parse_utc(observation["created_at"]) >= cutoff- ]- return rank_freeform_tags(recent[:_KNOWN_FREEFORM_TAGS_MAX_ROWS])---def _fetch_candidates(- api: PostHogApi, scanner_ids: list[str] | None, max_observations_per_scanner: int-) -> list[dict[str, Any]]:- candidates: list[dict[str, Any]] = []- for scanner in api.paginate(f"/api/projects/{api.project_id}/vision/scanners/", {"limit": 100}):- if scanner_ids and scanner["id"] not in scanner_ids:- continue- observations = list(- api.paginate(- f"/api/projects/{api.project_id}/vision/scanners/{scanner['id']}/observations/",- {"status": "succeeded", "limit": _OBSERVATION_PAGE},- max_items=max_observations_per_scanner,- )- )- known_tags = _known_freeform_tags(observations) if scanner["scanner_type"] == "classifier" else []- for observation in observations:- if not (observation.get("scanner_result") or {}).get("model_output"):- continue- candidates.append(- {- "observation": observation,- "scanner": scanner,- "scanner_type": scanner["scanner_type"],- "known_freeform_tags": known_tags,- }- )- return candidates---@frozen-class _SourceProject:- """Per-run context of the project being collected from, shared by every case."""-- team_id: int- team_name: str- product_context: str- event_descriptions: EventDescriptionLookup---def _write_case(- api: PostHogApi, root: Path, candidate: dict[str, Any], asset: _VideoAsset, source: _SourceProject-) -> GoldenCase | None:- observation = candidate["observation"]- scanner = candidate["scanner"]- label = observation.get("label") or {}- case = GoldenCase(- case_id=observation["id"],- scanner_id=scanner["id"],- scanner_name=scanner["name"],- scanner_type=scanner["scanner_type"],- session_id=observation["session_id"],- team_id=source.team_id,- team_name=source.team_name,- snapshot=ScannerSnapshot.model_validate(observation["scanner_snapshot"]),- recorded_output=observation["scanner_result"]["model_output"],- known_freeform_tags=candidate.get("known_freeform_tags") or [],- label_is_correct=label.get("is_correct"),- label_feedback=label.get("feedback") or "",- inactivity_periods=asset.inactivity_periods,- navigation=asset.navigation,- collected_at=dt.datetime.now(dt.UTC).isoformat(),- )- case_dir = case.case_dir(root)- if case.video_path(root).exists() and case.inputs_path(root).exists():- try:- case.load_inputs(root)- return case- except Exception:- logger.warning("collector.reused_case_invalid", case_id=case.case_id)-- inputs = build_llm_inputs(- api,- source.team_id,- case.session_id,- product_context=source.product_context,- event_descriptions=source.event_descriptions,- )- if inputs is None:- logger.warning("collector.no_events_for_session", session_id=case.session_id)- return None- case_dir.mkdir(parents=True, exist_ok=True)- api.download(- f"/api/environments/{api.project_id}/exports/{asset.asset_id}/content/?download=true", case.video_path(root)- )- # inputs.json lands last, so the reuse shortcut above only ever sees fully written cases.- case.inputs_path(root).write_text(inputs.model_dump_json())- return case---def _reusable_existing_cases(root: Path) -> dict[str, GoldenCase]:- """Previously collected cases whose files are still present and parseable, keyed by case id."""- if not (root / MANIFEST_NAME).exists():- return {}- try:- existing = load_dataset(root)- except Exception:- logger.warning("collector.existing_manifest_unreadable", root=str(root))- return {}- reusable: dict[str, GoldenCase] = {}- for case in existing.cases:- try:- case.load_inputs(root)- except Exception:- continue- if case.video_path(root).exists():- reusable[case.case_id] = case- return reusable---def _check_ai_consent(api: PostHogApi, environment: dict[str, Any]) -> None:- """The scan pipeline only runs over consented orgs; collecting recordings for evals holds the same bar."""- organization = api.get_json(f"/api/organizations/{environment['organization']}/")- if not organization.get("is_ai_data_processing_approved"):- raise RuntimeError(- "The source project's organization has not approved AI data processing; "- "collecting recordings for LLM evals requires that consent"- )---def collect(- *,- host: str,- project_id: int,- api_key: str,- output: Path,- per_type: int,- scanner_ids: list[str] | None = None,- seed: int = 42,- max_observations_per_scanner: int = 200,-) -> GoldenDataset:- api = PostHogApi(host, project_id, api_key)- # The environments API is keyed by team id; resolve the real team id once instead of assuming- # it equals the project id, and fail loudly when the two point at different projects.- environment = api.get_json(f"/api/environments/{project_id}/")- team_id = int(environment["id"])- if int(environment["project_id"]) != project_id:- raise RuntimeError(f"Environment {team_id} belongs to project {environment['project_id']}, not {project_id}")- _check_ai_consent(api, environment)- source = _SourceProject(- team_id=team_id,- team_name=str(environment.get("name", "")),- product_context=fetch_product_context_via_api(api),- event_descriptions=EventDescriptionLookup(api),- )- rng = random.Random(seed)-- candidates = _fetch_candidates(api, scanner_ids, max_observations_per_scanner)- ordered = order_candidates(candidates, rng)- all_session_ids = sorted({c["observation"]["session_id"] for c in candidates})- assets = lookup_video_assets(api, team_id, all_session_ids)- logger.info(- "collector.candidates",- candidates=len(candidates),- with_video_asset=sum(1 for c in candidates if c["observation"]["session_id"] in assets),- types={t: len(group) for t, group in ordered.items()},- )-- cases: list[GoldenCase] = []- for scanner_type, group in sorted(ordered.items()):- collected = 0- for candidate in group:- if collected >= per_type:- break- session_id = candidate["observation"]["session_id"]- asset = assets.get(session_id)- if asset is None or not asset_is_recorded_video(asset, candidate["observation"]):- continue- try:- case = _write_case(api, output, candidate, asset, source)- except requests.HTTPError as exc:- logger.warning("collector.case_failed", observation_id=candidate["observation"]["id"], error=str(exc))- continue- if case is None:- continue- cases.append(case)- collected += 1- logger.info("collector.type_done", scanner_type=scanner_type, collected=collected, target=per_type)-- # Merge with previously collected cases so re-runs extend the dataset instead of orphaning the- # case folders of earlier runs; this run's freshly collected version wins on overlap.- merged = _reusable_existing_cases(output)- merged.update({case.case_id: case for case in cases})- dataset = GoldenDataset(- created_at=dt.datetime.now(dt.UTC).isoformat(), host=host, project_id=project_id, cases=list(merged.values())- )- save_dataset(output, dataset)- return datasetdiff --git a/products/replay_vision/evals/dataset.py b/products/replay_vision/evals/dataset.pydeleted file mode 100644index 605c407a0228..000000000000--- a/products/replay_vision/evals/dataset.py+++ /dev/null@@ -1,108 +0,0 @@-"""Golden-dataset layout shared by the collector (collect.py) and the eval suite.--A dataset is a plain directory, never committed to the repo (it contains real session data):-- manifest.json # GoldenDataset- cases/<case_id>/video.mp4 # rasterized recording, byte-identical to what production sent to Gemini- cases/<case_id>/inputs.json # ScannerLlmInputs snapshot (events table, session metadata)-"""--import os-import datetime as dt-from pathlib import Path-from typing import Any--from pydantic import BaseModel, Field--from products.replay_vision.backend.temporal.types import ScannerLlmInputs, ScannerSnapshot--DATASET_ENV_VAR = "REPLAY_VISION_EVAL_DATASET"-MANIFEST_NAME = "manifest.json"-VIDEO_NAME = "video.mp4"-INPUTS_NAME = "inputs.json"-# Production re-checks AI data-processing consent right before every generation; for the eval the only-# check happens when collect.py runs, so bound how long that verification (and the collected recordings-# themselves) may be trusted.-DATASET_MAX_AGE_DAYS = 30---def parse_utc(raw: Any) -> dt.datetime:- """Parse an ISO timestamp into UTC; a naive string would silently shift by this machine's timezone, so reject it."""- parsed = dt.datetime.fromisoformat(str(raw))- if parsed.tzinfo is None:- raise ValueError(f"timestamp {raw!r} has no timezone; expected an offset-aware ISO string")- return parsed.astimezone(dt.UTC)---class GoldenCase(BaseModel, frozen=True):- """One collected observation: the frozen scanner config, its recorded output, and the human label if any."""-- case_id: str = Field(description="Source ReplayObservation id; doubles as the case directory name.")- scanner_id: str- scanner_name: str- scanner_type: str- session_id: str- team_id: int- team_name: str- snapshot: ScannerSnapshot- recorded_output: dict[str, Any] = Field(description="scanner_result.model_output at collection time.")- known_freeform_tags: list[str] = Field(- default_factory=list,- description="Tag vocabulary a freeform classifier scans with, captured at collection time.",- )- inactivity_periods: list[dict[str, Any]] = Field(- default_factory=list,- description="The rendered video's active/inactive map, for converting the model's video-time citations. "- "Empty on cases collected before it was captured, which then score as if nothing was cut.",- )- navigation: list[dict[str, Any]] = Field(- default_factory=list,- description="The page and tab changes the rendered video shows, for the prompt's navigation timeline. "- "Empty on cases collected before the renderer recorded them, which then scan without one.",- )- label_is_correct: bool | None = None- label_feedback: str = ""- collected_at: str-- def case_dir(self, root: Path) -> Path:- return root / "cases" / self.case_id-- def video_path(self, root: Path) -> Path:- return self.case_dir(root) / VIDEO_NAME-- def inputs_path(self, root: Path) -> Path:- return self.case_dir(root) / INPUTS_NAME-- def load_inputs(self, root: Path) -> ScannerLlmInputs:- return ScannerLlmInputs.model_validate_json(self.inputs_path(root).read_text())---class GoldenDataset(BaseModel, frozen=True):- created_at: str- host: str- project_id: int- cases: list[GoldenCase] = Field(default_factory=list)---def dataset_root() -> Path | None:- raw = os.environ.get(DATASET_ENV_VAR, "").strip()- return Path(raw).expanduser() if raw else None---def load_dataset(root: Path) -> GoldenDataset:- return GoldenDataset.model_validate_json((root / MANIFEST_NAME).read_text())---def ensure_dataset_fresh(dataset: GoldenDataset, root: Path) -> None:- """Refuse to scan a dataset whose consent verification has lapsed (see DATASET_MAX_AGE_DAYS)."""- age = dt.datetime.now(dt.UTC) - parse_utc(dataset.created_at)- if age > dt.timedelta(days=DATASET_MAX_AGE_DAYS):- raise RuntimeError(- f"Dataset at {root} was collected {age.days} days ago (limit {DATASET_MAX_AGE_DAYS}); "- "re-run collect.py, which re-verifies AI data-processing consent"- )---def save_dataset(root: Path, dataset: GoldenDataset) -> None:- root.mkdir(parents=True, exist_ok=True)- (root / MANIFEST_NAME).write_text(dataset.model_dump_json(indent=2))diff --git a/products/replay_vision/evals/eval_mcp_tools.py b/products/replay_vision/evals/eval_mcp_tools.pydeleted file mode 100644index 88b2244884d8..000000000000--- a/products/replay_vision/evals/eval_mcp_tools.py+++ /dev/null@@ -1,60 +0,0 @@-"""Does a coding agent drive the Replay Vision MCP tools correctly?--Each case asks for one multi-step job an agent is likely to be given: set up an alert with a-destination, backfill history, and recover a failed scan.--Run: hogli evals eval_replay_vision_mcp_tools-"""--from __future__ import annotations--from products.posthog_ai.eval_harness.base import SandboxedPrivateEval-from products.posthog_ai.eval_harness.config import SandboxedEvalCase-from products.posthog_ai.eval_harness.harness.context import EvalContext-from products.replay_vision.evals.mcp_scorers import (- CreatedMatchAlertWithWebhook,- EstimatedBeforeBackfill,- RetriedFailedObservation,-)-from products.replay_vision.evals.mcp_seeders import (- FAILED_SESSION_ID,- SCANNER_NAME,- seed_replay_vision_scanner,- seed_replay_vision_scanner_with_recordings,-)--WEBHOOK_URL = "https://example.com/hooks/replay-vision"---async def eval_replay_vision_mcp_tools(ctx: EvalContext) -> None:- await SandboxedPrivateEval(- experiment_name="sandboxed-replay-vision-mcp-tools-cli",- cases=[- SandboxedEvalCase(- name="alert_with_webhook",- prompt=(- f'Send a webhook to {WEBHOOK_URL} every time the "{SCANNER_NAME}" Replay Vision scanner '- "flags a session."- ),- setup=seed_replay_vision_scanner,- expected={"created_match_alert_with_webhook": {"webhook_url": WEBHOOK_URL}},- ),- SandboxedEvalCase(- name="backfill_last_week",- prompt=f'Run the "{SCANNER_NAME}" Replay Vision scanner over the last 7 days of recordings.',- setup=seed_replay_vision_scanner_with_recordings,- expected={"estimated_before_backfill": {}},- ),- SandboxedEvalCase(- name="retry_failed_scan",- prompt=(- f'The "{SCANNER_NAME}" scanner has no result for session {FAILED_SESSION_ID}. '- "Find out why and get it scanned."- ),- setup=seed_replay_vision_scanner,- expected={"retried_failed_observation": {}},- ),- ],- scorers=[CreatedMatchAlertWithWebhook(), EstimatedBeforeBackfill(), RetriedFailedObservation()],- ctx=ctx,- )diff --git a/products/replay_vision/evals/eval_scanner_quality.py b/products/replay_vision/evals/eval_scanner_quality.pydeleted file mode 100644index af724072181a..000000000000--- a/products/replay_vision/evals/eval_scanner_quality.py+++ /dev/null@@ -1,220 +0,0 @@-"""Golden-dataset quality suite for Replay Vision scanner prompts.--Each case re-runs the production scan pipeline (same Jinja templates, response schemas, events-round, and citation handling, via run_scan) against a collected session video plus its event-snapshot, then scores the fresh output against the recorded output and its human thumbs label.-Edit the templates under backend/temporal/scanners/prompts/ and re-run to compare experiments-in the local logs (this suite is private; nothing is sent to Braintrust).--Requires REPLAY_VISION_EVAL_DATASET (a directory written by collect.py) and GEMINI_API_KEY.-"""--import time-import asyncio-from functools import partial-from pathlib import Path-from typing import Any--import structlog-from google.genai import (- Client as RawGenAIClient,- types as genai_types,-)--from posthog.temporal.session_replay.rasterize_recording.types import NavigationChange--from products.posthog_ai.eval_harness.config import BaseEvalCase-from products.posthog_ai.eval_harness.harness.context import EvalContext-from products.posthog_ai.eval_harness.harness.requirements import SuiteKind-from products.posthog_ai.eval_harness.one_shot import OneShotPrivateEval-from products.replay_vision.backend.temporal.activities.call_scanner_provider import apply_known_freeform_tags, run_scan-from products.replay_vision.backend.temporal.errors import ScannerFailureError-from products.replay_vision.backend.temporal.gemini import gemini_api_key-from products.replay_vision.backend.temporal.scanners import scanner_from_snapshot-from products.replay_vision.backend.temporal.video_clock import VideoClock, video_clock_from_export_context-from products.replay_vision.evals.dataset import (- DATASET_ENV_VAR,- GoldenCase,- dataset_root,- ensure_dataset_fresh,- load_dataset,-)-from products.replay_vision.evals.outcomes import primary_outcome-from products.replay_vision.evals.scorers import (- SUMMARY_FIELDS,- LabeledOutcome,- OutputStability,- ScanCompleted,- ScoreAlignment,- SummaryAlignment,-)--SUITE_KIND = SuiteKind.ONE_SHOT--logger = structlog.get_logger(__name__)--_MAX_PROCESSING_WAIT_SECONDS = 300---def build_case(golden: GoldenCase) -> BaseEvalCase:- """Map a golden case onto the scorer contract: which scorer applies, and what its reference is."""- expected: dict[str, Any] = {}- recorded_primary = primary_outcome(golden.recorded_output)- if golden.scanner_type in ("monitor", "classifier"):- if golden.label_is_correct is not None:- expected["labeled_outcome"] = {- "is_correct": golden.label_is_correct,- "recorded_primary": recorded_primary,- }- else:- expected["output_stability"] = {"recorded_primary": recorded_primary}- elif golden.scanner_type == "scorer":- recorded_score = golden.recorded_output.get("score")- scale = golden.snapshot.scanner_config.get("scale") or {}- scale_min, scale_max = scale.get("min"), scale.get("max")- # Default the bounds as a pair: defaulting them independently can fabricate a zero-width- # scale (e.g. min set to 1.0 with max defaulting to 1.0) that no scanner actually has.- if scale_min is None and scale_max is None:- scale_min, scale_max = 0.0, 1.0- # A thumbs-downed recorded score is a known-bad reference, so those cases only get scan_completed.- if (- golden.label_is_correct is not False- and isinstance(recorded_score, int | float)- and scale_min is not None- and scale_max is not None- ):- expected["score_alignment"] = {- "recorded_score": recorded_score,- "scale_min": scale_min,- "scale_max": scale_max,- }- elif golden.scanner_type == "summarizer":- if golden.label_is_correct is not False:- expected["summary_alignment"] = {- "reference": {field: golden.recorded_output.get(field) for field in SUMMARY_FIELDS}- }- return BaseEvalCase(- # The full case id: UUIDv7 ids collected in the same minute share their first characters,- # so any truncation makes same-type cases collide.- name=f"{golden.scanner_type}-{golden.case_id}",- prompt=str(golden.snapshot.scanner_config.get("prompt", "")),- expected=expected,- metadata={- "case_id": golden.case_id,- "scanner_name": golden.scanner_name,- "scanner_type": golden.scanner_type,- "session_id": golden.session_id,- "label_is_correct": golden.label_is_correct,- },- )---def _upload_video(client: RawGenAIClient, path: Path) -> genai_types.File:- uploaded = client.files.upload(- file=str(path),- config=genai_types.UploadFileConfig(- mime_type="video/mp4", display_name=f"replay-vision-eval-{path.parent.name}"- ),- )- waited = 0.0- while uploaded.state and uploaded.state.name == "PROCESSING":- if waited >= _MAX_PROCESSING_WAIT_SECONDS:- raise RuntimeError(f"Gemini file for {path} stuck in PROCESSING after {waited:.0f}s")- time.sleep(0.5)- waited += 0.5- uploaded = client.files.get(name=uploaded.name or "")- state = uploaded.state.name if uploaded.state else None- if state != "ACTIVE" or not uploaded.uri:- raise RuntimeError(f"Gemini upload for {path} ended in state {state!r}")- return uploaded---def _delete_file_quiet(client: RawGenAIClient, name: str | None) -> None:- if not name:- return- try:- client.files.delete(name=name)- except Exception:- logger.warning("replay_vision.eval.gemini_delete_failed", file=name)---async def _scan_task(- root: Path, golden_by_case_id: dict[str, GoldenCase], case: BaseEvalCase, ctx: EvalContext-) -> dict[str, Any]:- golden = golden_by_case_id[case.metadata["case_id"]]- llm_inputs = await asyncio.to_thread(golden.load_inputs, root)- # Rebuild the scanner the way production does, tag vocabulary included, so a freeform classifier- # is scored on its prompt rather than on missing context.- scanner = apply_known_freeform_tags(scanner_from_snapshot(golden.snapshot), golden.known_freeform_tags)- client = RawGenAIClient(api_key=gemini_api_key())- uploaded = await asyncio.to_thread(_upload_video, client, golden.video_path(root))- try:- try:- result = await run_scan(- snapshot=golden.snapshot,- scanner=scanner,- llm_inputs=llm_inputs,- team_name=golden.team_name,- file_uri=uploaded.uri or "",- mime_type=uploaded.mime_type or "video/mp4",- team_id=golden.team_id,- # A case collected before the map was captured scores as if nothing was cut.- video_clock=video_clock_from_export_context({"inactivity_periods": golden.inactivity_periods})- or VideoClock(spans=()),- navigation=[NavigationChange.model_validate(change) for change in golden.navigation],- )- except ScannerFailureError as exc:- # A scan the model cannot complete is a prompt-quality signal (broken schema compliance),- # not an infra error, so it must score 0 rather than be excluded from the aggregate.- return {- "exit_code": 1,- "model_output": None,- "error": str(exc),- "scanner_type": golden.scanner_type,- "last_message": f"scan failed: {exc}",- }- finally:- await asyncio.to_thread(_delete_file_quiet, client, uploaded.name)-- model_output = result.model_output.model_dump(mode="json")- primary = primary_outcome(model_output)- return {- "exit_code": 0,- "model_output": model_output,- "error": None,- "scanner_type": golden.scanner_type,- "signals_count": len(result.signals),- "signals": [signal.model_dump(mode="json") for signal in result.signals],- "primary": primary,- "last_message": primary or "",- }---async def eval_scanner_quality(ctx: EvalContext) -> None:- root = dataset_root()- if root is None:- # Skip rather than fail: the dataset is local-only and most environments (CI included) won't have one.- logger.warning("replay_vision.eval.skipped_no_dataset", env_var=DATASET_ENV_VAR)- return- if not gemini_api_key():- raise RuntimeError("Set GEMINI_API_KEY (or REPLAY_VISION_GEMINI_API_KEY) to run replay-vision scans")-- dataset = load_dataset(root)- ensure_dataset_fresh(dataset, root)- golden_cases = dataset.cases- missing = [g.case_id for g in golden_cases if not (g.video_path(root).exists() and g.inputs_path(root).exists())]- if missing:- raise RuntimeError(f"Dataset at {root} is missing files for cases {missing[:5]}; re-run collect.py")-- cases = [build_case(golden) for golden in golden_cases]- golden_by_case_id = {case.metadata["case_id"]: golden for case, golden in zip(cases, golden_cases)}- # A collision here would silently score one observation against another session's video.- if len(golden_by_case_id) != len(golden_cases):- raise RuntimeError(f"Dataset at {root} contains duplicate case ids; re-collect into a clean directory")- await OneShotPrivateEval(- experiment_name="replay-vision-scanner-quality",- cases=cases,- scorers=[ScanCompleted(), LabeledOutcome(), OutputStability(), ScoreAlignment(), SummaryAlignment()],- task=partial(_scan_task, root, golden_by_case_id),- ctx=ctx,- )diff --git a/products/replay_vision/evals/mcp_scorers.py b/products/replay_vision/evals/mcp_scorers.pydeleted file mode 100644index 233419e6316b..000000000000--- a/products/replay_vision/evals/mcp_scorers.py+++ /dev/null@@ -1,103 +0,0 @@-"""Deterministic scorers for the Replay Vision MCP tool-use suite."""--from __future__ import annotations--from datetime import datetime, timedelta-from typing import Any--from products.posthog_ai.eval_harness.log_parser import LogParser-from products.posthog_ai.eval_harness.scorers.contract import Score, Scorer---class _McpToolScorer(Scorer):- """Skips cases that don't expect this scorer and fails runs with no log, then defers to `_score`."""-- def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs: Any) -> Score:- if not expected or self._name() not in expected:- return Score(name=self._name(), score=None, metadata={"reason": "Not expected for this case"})- if not output or not output.get("raw_log"):- return Score(name=self._name(), score=0.0, metadata={"reason": "No raw log"})- parser = LogParser.cached(output["raw_log"], initial_prompt=output.get("prompt", "") or "")- return self._score(parser, output.get("seed") or {}, expected[self._name()])-- def _score(self, parser: LogParser, seed: dict[str, Any], expected: dict[str, Any]) -> Score:- raise NotImplementedError---class CreatedMatchAlertWithWebhook(_McpToolScorer):- """A match alert on the seeded scanner for verdict yes, then a webhook destination on that alert."""-- def _name(self) -> str:- return "created_match_alert_with_webhook"-- def _score(self, parser: LogParser, seed: dict[str, Any], expected: dict[str, Any]) -> Score:- alert_created = any(- not c.is_error- and c.input.get("scanner_id") == seed.get("scanner_id")- and c.input.get("kind") == "match"- and "yes" in ((c.input.get("selection") or {}).get("verdict") or [])- for c in parser.get_tool_calls("vision-alerts-create")- )- destination_added = any(- not c.is_error- and c.input.get("type") == "webhook"- and c.input.get("webhook_url") == expected["webhook_url"]- for c in parser.get_tool_calls("vision-alerts-destinations-create")- )- return Score(- name=self._name(),- score=(int(alert_created) + int(destination_added)) / 2,- metadata={"alert_created": alert_created, "destination_added": destination_added},- )---def _spans_about_a_week(call_input: dict[str, Any]) -> bool:- try:- start = datetime.fromisoformat(str(call_input["window_start"]).replace("Z", "+00:00"))- end = datetime.fromisoformat(str(call_input["window_end"]).replace("Z", "+00:00"))- except (KeyError, ValueError):- return False- # A day of slack either way, since the agent picks its own clock and rounding.- return timedelta(days=6) <= end - start <= timedelta(days=8)---class EstimatedBeforeBackfill(_McpToolScorer):- """A successful estimate and no backfill create.-- Nobody in the sandbox can agree to the quoted cost, and the tool requires that agreement first, so the- right move is to estimate and stop to ask.- """-- def _name(self) -> str:- return "estimated_before_backfill"-- def _score(self, parser: LogParser, seed: dict[str, Any], expected: dict[str, Any]) -> Score:- estimates = [- c- for c in parser.get_tool_calls("vision-scanners-backfills-estimate")- if not c.is_error and _spans_about_a_week(c.input)- ]- creates = [c.call_id for c in parser.get_tool_calls("vision-scanners-backfills-create")]- if not estimates:- return Score(name=self._name(), score=0.0, metadata={"reason": "No successful estimate over the last week"})- return Score(- name=self._name(),- score=0.0 if creates else 1.0,- metadata={"estimates": len(estimates), "unagreed_creates": creates},- )---class RetriedFailedObservation(_McpToolScorer):- """The seeded failed observation was retried by its own id, not the session or scanner id."""-- def _name(self) -> str:- return "retried_failed_observation"-- def _score(self, parser: LogParser, seed: dict[str, Any], expected: dict[str, Any]) -> Score:- retries = parser.get_tool_calls("vision-observations-retry")- hit = [c for c in retries if not c.is_error and str(c.input.get("id")) == seed.get("failed_observation_id")]- return Score(- name=self._name(),- score=1.0 if hit else 0.0,- metadata={"retries": len(retries), "succeeded_on_target": len(hit)},- )diff --git a/products/replay_vision/evals/mcp_seeders.py b/products/replay_vision/evals/mcp_seeders.pydeleted file mode 100644index 16780d65a7fe..000000000000--- a/products/replay_vision/evals/mcp_seeders.py+++ /dev/null@@ -1,68 +0,0 @@-"""Seeders for the Replay Vision MCP tool-use suite."""--from __future__ import annotations--from datetime import timedelta-from typing import Any--from django.utils import timezone--from posthog.models import Team-from posthog.session_recordings.queries.test.session_replay_sql import produce_replay_summary--from products.replay_vision.backend.models.replay_observation import (- ObservationStatus,- ObservationTrigger,- ReplayObservation,-)-from products.replay_vision.backend.models.replay_scanner import ReplayScanner, ScannerModel, ScannerType-from products.tasks.backend.facade.agents import CustomPromptSandboxContext--SCANNER_NAME = "Checkout friction"-FAILED_SESSION_ID = "0192f0e1-eval-4a5b-8c7d-checkoutfail01"---def seed_replay_vision_scanner(context: CustomPromptSandboxContext) -> dict[str, Any]:- """One paused monitor scanner with a transiently failed observation, and AI consent on."""- team = Team.objects.select_related("organization").get(id=context.team_id)- # The scanner, alert, backfill, and retry tools all refuse to run without AI consent.- team.organization.is_ai_data_processing_approved = True- team.organization.save(update_fields=["is_ai_data_processing_approved"])-- scanner = ReplayScanner.objects.create(- team=team,- name=SCANNER_NAME,- scanner_type=ScannerType.MONITOR,- scanner_config={"prompt": "Did the user struggle to complete checkout?"},- model=ScannerModel.GEMINI_3_8_FLASH,- # Paused so no scheduled sweep runs during the case.- enabled=False,- )- observation = ReplayObservation.objects.create(- team=team,- scanner=scanner,- session_id=FAILED_SESSION_ID,- status=ObservationStatus.FAILED,- error_reason="provider_transient:The model provider timed out.",- triggered_by=ObservationTrigger.SCHEDULE,- completed_at=timezone.now(),- )- return {"scanner_id": str(scanner.id), "failed_observation_id": str(observation.id)}---def seed_replay_vision_scanner_with_recordings(context: CustomPromptSandboxContext) -> dict[str, Any]:- """The same scanner plus recordings from the last week, so a backfill estimate finds sessions."""- seed = seed_replay_vision_scanner(context)- now = timezone.now()- for day in range(1, 4):- start = now - timedelta(days=day, hours=2)- # Long and active enough to clear the video scanner's eligibility floor.- produce_replay_summary(- team_id=context.team_id,- session_id=f"replay-vision-eval-backfill-{day}",- distinct_id=f"replay-vision-eval-user-{day}",- first_timestamp=start,- last_timestamp=start + timedelta(minutes=5),- active_milliseconds=120_000,- )- return seeddiff --git a/products/replay_vision/evals/outcomes.py b/products/replay_vision/evals/outcomes.pydeleted file mode 100644index 8de8d34a18ff..000000000000--- a/products/replay_vision/evals/outcomes.py+++ /dev/null@@ -1,42 +0,0 @@-from typing import Any, Literal--from products.replay_vision.backend.tags import slugify_tag--Outcome = Literal["kept", "regressed", "fixed", "still_wrong"]--# A summary can run long, so before/after previews cap it rather than showing the whole body.-_SUMMARY_PREVIEW_CAP = 200---def primary_outcome(model_output: dict[str, Any] | None) -> str | None:- """The display string for before/after comparison: a discrete verdict/tags for monitors and classifiers,- or the raw score/summary for the other types."""- output = model_output or {}- verdict = output.get("verdict")- if isinstance(verdict, str) and verdict:- return f"Verdict: {verdict.strip().lower()}"- # Freeform tags are part of a classifier's output, so a change that only touches them must not read as- # "no change". Matches `describe_output`, which merges both lists.- raw_tags = [*(output.get("tags") or []), *(output.get("tags_freeform") or [])]- tags = sorted({slug for t in raw_tags if isinstance(t, str) and (slug := slugify_tag(t))})- if tags:- return f"Tags: {', '.join(tags)}"- score = output.get("score")- if isinstance(score, int | float) and not isinstance(score, bool):- return f"Score: {score}"- title = output.get("title")- if isinstance(title, str) and title.strip():- return title.strip()- summary = output.get("summary")- if isinstance(summary, str) and summary.strip():- trimmed = summary.strip()- return trimmed if len(trimmed) <= _SUMMARY_PREVIEW_CAP else trimmed[:_SUMMARY_PREVIEW_CAP].rstrip() + "…"- return None---def classify_outcome(rated_correct: bool, before: str | None, after: str | None) -> Outcome:- """A `None` outcome is valid (e.g. a classifier with no tags)."""- changed = before != after- if rated_correct:- return "regressed" if changed else "kept"- return "fixed" if changed else "still_wrong"diff --git a/products/replay_vision/evals/scorers.py b/products/replay_vision/evals/scorers.pydeleted file mode 100644index 38e4744fabed..000000000000--- a/products/replay_vision/evals/scorers.py+++ /dev/null@@ -1,167 +0,0 @@-"""Scorers for the replay-vision golden-dataset suite.--Every scorer reads the task output dict built in eval_scanner_quality._scan_task plus its own-sub-dict of ``case.expected``, and self-skips (score=None) when its key is absent, so one scorer-list spans all four scanner types. Correctness scoring compares the primary outcome against the-human label: kept/fixed score 1, regressed/still_wrong score 0.-"""--import json-from typing import Any--from products.posthog_ai.eval_harness.scorers import GRADED_ALIGNMENT_CHOICE_SCORES, JUDGE_MODEL, JudgedScorer-from products.posthog_ai.eval_harness.scorers.contract import Score, Scorer-from products.replay_vision.evals.outcomes import classify_outcome, primary_outcome--_OUTCOME_SCORES = {"kept": 1.0, "fixed": 1.0, "regressed": 0.0, "still_wrong": 0.0}--# The summarizer output fields the judge compares; build_case snapshots the same fields as the reference.-SUMMARY_FIELDS = ("title", "summary")---def _spec(expected: dict[str, Any] | None, name: str) -> dict[str, Any] | None:- value = (expected or {}).get(name)- return value if isinstance(value, dict) else None---class ScanCompleted(Scorer):- """Did the scan produce a validated output at all? A prompt change that breaks the response- schema or the semantic validators fails here before any quality scorer gets a say."""-- def _name(self) -> str:- return "scan_completed"-- def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs: Any) -> Score:- if not output:- return Score(name=self._name(), score=0.0, metadata={"reason": "no output"})- if output.get("model_output"):- return Score(name=self._name(), score=1.0)- return Score(name=self._name(), score=0.0, metadata={"reason": output.get("error") or "no model_output"})---class LabeledOutcome(Scorer):- """For human-labeled monitor/classifier cases: kept and fixed score 1, regressed and- still_wrong score 0."""-- def _name(self) -> str:- return "labeled_outcome"-- def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs: Any) -> Score:- spec = _spec(expected, self._name())- if spec is None:- return Score(name=self._name(), score=None, metadata={"reason": "not a labeled discrete case"})- if not output or not output.get("model_output"):- return Score(name=self._name(), score=0.0, metadata={"reason": "scan produced no output"})- after = primary_outcome(output["model_output"])- outcome = classify_outcome(bool(spec["is_correct"]), spec.get("recorded_primary"), after)- return Score(- name=self._name(),- score=_OUTCOME_SCORES[outcome],- metadata={"outcome": outcome, "before": spec.get("recorded_primary"), "after": after},- )---class OutputStability(Scorer):- """For unlabeled monitor/classifier cases there is no ground truth; score whether the fresh- outcome matches the recorded baseline. This measures churn introduced by a prompt change,- not correctness, so it is reported separately from labeled_outcome."""-- def _name(self) -> str:- return "output_stability"-- def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs: Any) -> Score:- spec = _spec(expected, self._name())- if spec is None:- return Score(name=self._name(), score=None, metadata={"reason": "not an unlabeled discrete case"})- if not output or not output.get("model_output"):- return Score(name=self._name(), score=0.0, metadata={"reason": "scan produced no output"})- after = primary_outcome(output["model_output"])- stable = after == spec.get("recorded_primary")- return Score(- name=self._name(),- score=1.0 if stable else 0.0,- metadata={"before": spec.get("recorded_primary"), "after": after},- )---class ScoreAlignment(Scorer):- """For scorer scanners: distance from the reference score, normalized by the configured scale.-- The reference is the recorded production score; cases whose recorded run was thumbs-downed- carry no spec (a known-bad reference would reward reproducing the mistake)."""-- def _name(self) -> str:- return "score_alignment"-- def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs: Any) -> Score:- spec = _spec(expected, self._name())- if spec is None:- return Score(name=self._name(), score=None, metadata={"reason": "not a scorer case with a reference"})- span = float(spec["scale_max"]) - float(spec["scale_min"])- if span <= 0:- # A degenerate scale makes distance meaningless: the scorer is inapplicable, not failing.- return Score(name=self._name(), score=None, metadata={"reason": "zero-width scale"})- model_output = (output or {}).get("model_output") or {}- fresh = model_output.get("score")- if not isinstance(fresh, int | float) or isinstance(fresh, bool):- return Score(name=self._name(), score=0.0, metadata={"reason": "scan produced no numeric score"})- alignment = max(0.0, 1.0 - abs(float(fresh) - float(spec["recorded_score"])) / span)- return Score(- name=self._name(),- score=alignment,- metadata={"recorded_score": spec["recorded_score"], "fresh_score": fresh},- )---_SUMMARY_JUDGE_PROMPT = """-You are comparing two AI-generated analyses of the same session recording of a user using a product.--The reference analysis was produced earlier and is trusted:-{{expected}}--The candidate analysis was just produced by a new version of the analyzer:-{{output}}--Grade how well the candidate captures the same user journey as the reference: the user's intent,-what happened, the outcome, and any friction encountered. Wording differences are fine; missing or-invented events, a different outcome, or lost friction points are not.--Answer by selecting exactly one option:-- perfect: same journey, intent, outcome, and friction; nothing meaningful lost or invented-- near_perfect: same story with trivial omissions or additions-- slightly_off: mostly the same story, but one meaningful detail is missing or invented-- somewhat_misaligned: the story partially matches, but intent, outcome, or major friction diverges-- strongly_misaligned: the candidate describes a substantially different session-- useless: the candidate is empty, incoherent, or unrelated-"""---class SummaryAlignment(JudgedScorer):- """LLM judge for summarizer cases: does the fresh summary tell the same story as the reference?-- The reference is the recorded output; thumbs-downed references are excluded at case-build time."""-- def __init__(self, **kwargs: Any) -> None:- super().__init__(- name="summary_alignment",- prompt_template=_SUMMARY_JUDGE_PROMPT,- choice_scores=GRADED_ALIGNMENT_CHOICE_SCORES,- model=JUDGE_MODEL,- max_completion_tokens=512,- **kwargs,- )-- def _prepare(self, output: dict[str, Any] | None, expected: dict[str, Any] | None) -> dict[str, Any] | Score:- spec = _spec(expected, "summary_alignment")- if spec is None:- return Score(- name=self._name(), score=None, metadata={"reason": "not a summarizer case with a trusted reference"}- )- model_output = (output or {}).get("model_output")- if not model_output:- return Score(name=self._name(), score=0.0, metadata={"reason": "scan produced no output"})- candidate = {field: model_output.get(field) for field in SUMMARY_FIELDS}- return {- "output": json.dumps(candidate, indent=2),- "expected": json.dumps(spec.get("reference"), indent=2),- }diff --git a/products/replay_vision/evals/test/__init__.py b/products/replay_vision/evals/test/__init__.pydeleted file mode 100644index e69de29bb2d1..000000000000diff --git a/products/replay_vision/evals/test/test_eval_scanner_quality.py b/products/replay_vision/evals/test/test_eval_scanner_quality.pydeleted file mode 100644index 1b70620bf933..000000000000--- a/products/replay_vision/evals/test/test_eval_scanner_quality.py+++ /dev/null@@ -1,498 +0,0 @@-from __future__ import annotations--import json-import time-import random-import asyncio-import datetime as dt-from pathlib import Path-from typing import Any--import pytest-from unittest.mock import AsyncMock, MagicMock, patch--from google.genai import types as genai_types--from products.posthog_ai.eval_harness.harness.context import EvalContext-from products.posthog_ai.eval_harness.scorers.contract import Score, Scorer-from products.replay_vision.backend.temporal.activities.call_scanner_provider import apply_known_freeform_tags-from products.replay_vision.backend.temporal.scanners import ClassifierScanner-from products.replay_vision.backend.temporal.scanners.base import SignalFinding-from products.replay_vision.backend.temporal.scanners.monitor import MonitorOutput-from products.replay_vision.backend.temporal.types import ScannerCallOutput, ScannerLlmInputs, ScannerSnapshot-from products.replay_vision.evals import collector, eval_scanner_quality-from products.replay_vision.evals.collector import (- _VideoAsset,- asset_is_recorded_video,- build_llm_inputs,- order_candidates,-)-from products.replay_vision.evals.dataset import GoldenCase, GoldenDataset, ensure_dataset_fresh, parse_utc-from products.replay_vision.evals.eval_scanner_quality import build_case-from products.replay_vision.evals.scorers import (- LabeledOutcome,- OutputStability,- ScanCompleted,- ScoreAlignment,- SummaryAlignment,-)---def _eval(scorer: Scorer, output: dict[str, Any] | None, expected: dict[str, Any] | None) -> Score:- # eval_async is the entrypoint the harness dispatches through.- score = asyncio.run(scorer.eval_async(output, expected))- assert isinstance(score, Score)- return score---def _monitor_output(verdict: str) -> dict[str, Any]:- return {"model_output": {"verdict": verdict, "reasoning": "r", "confidence": 0.9}}---def _golden(- scanner_type: str,- label_is_correct: bool | None,- recorded_output: dict[str, Any],- case_id: str = "0199aaaa-0000-7000-8000-000000000001",- scale: dict[str, Any] | None = None,-) -> GoldenCase:- scanner_config: dict[str, Any] = {"prompt": "watch for rage clicks"}- if scanner_type == "classifier":- scanner_config["tags"] = ["a", "b"]- if scanner_type == "scorer":- scanner_config["scale"] = {"min": 1, "max": 5} if scale is None else scale- return GoldenCase(- case_id=case_id,- scanner_id="s1",- scanner_name="Test scanner",- scanner_type=scanner_type,- session_id="sess-1",- team_id=2,- team_name="Team",- snapshot=ScannerSnapshot(- name="Test scanner",- scanner_type=scanner_type,- scanner_version=1,- model="gemini-3.6-flash",- provider="google",- emits_signals=False,- scanner_config=scanner_config,- ),- recorded_output=recorded_output,- label_is_correct=label_is_correct,- collected_at="2026-07-30T00:00:00+00:00",- )--- "is_correct,fresh_verdict,expected_score,expected_outcome",- [- (True, "yes", 1.0, "kept"),- (True, "no", 0.0, "regressed"),- (False, "no", 1.0, "fixed"),- (False, "yes", 0.0, "still_wrong"),- ],-)-def test_labeled_outcome_scores_thumbs_semantics(- is_correct: bool, fresh_verdict: str, expected_score: float, expected_outcome: str-) -> None:- expected = {"labeled_outcome": {"is_correct": is_correct, "recorded_primary": "Verdict: yes"}}- score = _eval(LabeledOutcome(), _monitor_output(fresh_verdict), expected)- assert score.score == expected_score- assert score.metadata["outcome"] == expected_outcome--- "scorer",- [LabeledOutcome(), OutputStability(), ScoreAlignment()],- ids=lambda s: s._name(),-)-def test_inapplicable_cases_skip_instead_of_failing(scorer: Any) -> None:- # None means "skipped" in the aggregate; returning 0.0 here would silently drag every- # experiment's mean down for cases the scorer was never meant to grade.- score = _eval(scorer, _monitor_output("yes"), {})- assert score.score is None---def test_labeled_outcome_fails_when_scan_produced_nothing() -> None:- expected = {"labeled_outcome": {"is_correct": True, "recorded_primary": "Verdict: yes"}}- score = _eval(LabeledOutcome(), {"model_output": None, "error": "boom"}, expected)- assert score.score == 0.0--- "fresh,expected_score",- [- (3.0, 1.0),- (4.0, 0.75),- (1.0, 0.5),- (None, 0.0),- ],-)-def test_score_alignment_normalizes_distance_by_scale(fresh: float | None, expected_score: float) -> None:- expected = {"score_alignment": {"recorded_score": 3.0, "scale_min": 1.0, "scale_max": 5.0}}- output = {"model_output": {"score": fresh} if fresh is not None else None}- score = _eval(ScoreAlignment(), output, expected)- assert score.score == pytest.approx(expected_score)---def test_score_alignment_skips_zero_width_scale() -> None:- # min == max used to divide by zero and kill the case; a degenerate scale is inapplicable, not failing.- expected = {"score_alignment": {"recorded_score": 3.0, "scale_min": 3.0, "scale_max": 3.0}}- score = _eval(ScoreAlignment(), {"model_output": {"score": 3.0}}, expected)- assert score.score is None- assert score.metadata["reason"] == "zero-width scale"--- "scale,expected_bounds",- [- ({}, (0.0, 1.0)),- ({"min": 1, "max": 5}, (1, 5)),- ({"min": 1}, None),- ({"max": 5}, None),- ],-)-def test_build_case_only_defaults_scale_bounds_as_a_pair(- scale: dict[str, Any], expected_bounds: tuple[float, float] | None-) -> None:- case = build_case(_golden("scorer", None, {"score": 3.0}, scale=scale))- spec = case.expected.get("score_alignment")- if expected_bounds is None:- assert spec is None- else:- assert spec is not None- assert (spec["scale_min"], spec["scale_max"]) == expected_bounds---def test_output_stability_compares_primary_outcomes() -> None:- expected = {"output_stability": {"recorded_primary": "Verdict: yes"}}- assert _eval(OutputStability(), _monitor_output("yes"), expected).score == 1.0- assert _eval(OutputStability(), _monitor_output("no"), expected).score == 0.0---def test_scan_completed_fails_on_schema_breakage() -> None:- assert _eval(ScanCompleted(), _monitor_output("yes"), {}).score == 1.0- failed = _eval(ScanCompleted(), {"model_output": None, "error": "required step rejected"}, {})- assert failed.score == 0.0- assert "rejected" in failed.metadata["reason"]--- "signals",- [- [],- [- {- "problem_type": "design_flaw",- "headline": "Navigation panel covers the book list",- "start_time": 7,- "end_time": 11,- "url": "https://example.com/library",- "description": "The navigation panel overlaps the book list.",- "confidence": 0.8,- },- {- "problem_type": "crash",- "headline": "Reader errors out after a book opens",- "start_time": 23,- "end_time": 29,- "url": "https://example.com/reader",- "description": "The reader shows an error screen after the book opens.",- "confidence": 0.95,- },- ],- ],- ids=["empty", "multiple_findings"],-)-def test_scan_task_retains_structured_signals(signals: list[dict[str, str | int | float]], tmp_path: Path) -> None:- golden = _golden("monitor", True, {"verdict": "yes"})- golden = golden.model_copy(update={"snapshot": golden.snapshot.model_copy(update={"emits_signals": True})})- scan_output = ScannerCallOutput(- model_output=MonitorOutput(verdict="yes", reasoning="The book list is covered.", confidence=0.9),- signals=[SignalFinding.model_validate(signal) for signal in signals],- )- with (- patch.object(GoldenCase, "load_inputs", return_value=MagicMock(spec=ScannerLlmInputs)),- patch.object(eval_scanner_quality, "gemini_api_key", return_value="fake-eval-api-key"),- patch.object(eval_scanner_quality, "RawGenAIClient"),- patch.object(- eval_scanner_quality,- "_upload_video",- return_value=genai_types.File(- name="files/eval-video", uri="https://example.com/video.mp4", mime_type="video/mp4"- ),- ),- patch.object(eval_scanner_quality, "_delete_file_quiet"),- patch.object(eval_scanner_quality, "run_scan", new=AsyncMock(return_value=scan_output)),- ):- output = asyncio.run(- eval_scanner_quality._scan_task(- tmp_path, {golden.case_id: golden}, build_case(golden), MagicMock(spec=EvalContext)- )- )-- assert json.loads(json.dumps(output)) == {- "exit_code": 0,- "model_output": scan_output.model_output.model_dump(mode="json"),- "error": None,- "scanner_type": "monitor",- "signals_count": len(signals),- "signals": signals,- "primary": "Verdict: yes",- "last_message": "Verdict: yes",- }---def test_summary_alignment_prepare_gates_on_reference() -> None:- # _prepare directly: going through eval_async would call the LLM judge.- judge = SummaryAlignment()- skipped = judge._prepare(_monitor_output("yes"), {})- assert isinstance(skipped, Score)- assert skipped.score is None- spec = {"summary_alignment": {"reference": {"title": "t", "summary": "s"}}}- no_output = judge._prepare({"model_output": None}, spec)- assert isinstance(no_output, Score)- assert no_output.score == 0.0- prepared = judge._prepare({"model_output": {"title": "t2", "summary": "s2"}}, spec)- assert isinstance(prepared, dict)- assert "t2" in prepared["output"]- assert "t" in prepared["expected"]--- "scanner_type,label_is_correct,recorded_output,expected_key",- [- ("monitor", True, {"verdict": "yes"}, "labeled_outcome"),- ("monitor", None, {"verdict": "yes"}, "output_stability"),- ("classifier", False, {"tags": ["a"]}, "labeled_outcome"),- ("scorer", None, {"score": 3.0}, "score_alignment"),- ("scorer", True, {"score": 3.0}, "score_alignment"),- ("summarizer", None, {"title": "t", "summary": "s"}, "summary_alignment"),- ],-)-def test_build_case_routes_to_the_right_scorer(- scanner_type: str, label_is_correct: bool | None, recorded_output: dict[str, Any], expected_key: str-) -> None:- case = build_case(_golden(scanner_type, label_is_correct, recorded_output))- assert list(case.expected.keys()) == [expected_key]--- "scanner_type,recorded_output",- [- ("scorer", {"score": 3.0}),- ("summarizer", {"title": "t", "summary": "s"}),- ],-)-def test_build_case_never_trusts_a_thumbs_downed_reference(scanner_type: str, recorded_output: dict[str, Any]) -> None:- case = build_case(_golden(scanner_type, False, recorded_output))- assert case.expected == {}---def test_case_names_do_not_collide_on_shared_uuid_prefix() -> None:- # UUIDv7 ids minted in the same minute share their leading characters; a truncated name- # collapsed such cases into one dataset entry, scoring one against another session's video.- goldens = [- _golden("monitor", True, {"verdict": "yes"}, case_id="0199aaaa-0000-7000-8000-000000000001"),- _golden("monitor", True, {"verdict": "yes"}, case_id="0199aaaa-0000-7000-8000-000000000002"),- ]- cases = [build_case(golden) for golden in goldens]- assert cases[0].name != cases[1].name- golden_by_case_id = {case.metadata["case_id"]: golden for case, golden in zip(cases, goldens)}- assert len(golden_by_case_id) == len(goldens)---def test_order_candidates_puts_labeled_first_per_type() -> None:- def candidate(scanner_type: str, obs_id: str, labeled: bool) -> dict[str, Any]:- observation: dict[str, Any] = {"id": obs_id, "label": {"is_correct": True} if labeled else None}- return {"observation": observation, "scanner": {}, "scanner_type": scanner_type}-- candidates = [- candidate("monitor", "u1", False),- candidate("monitor", "l1", True),- candidate("monitor", "u2", False),- candidate("scorer", "u3", False),- candidate("monitor", "l2", True),- ]- ordered = order_candidates(candidates, random.Random(42))- monitor_ids = [c["observation"]["id"] for c in ordered["monitor"]]- assert monitor_ids[:2] == ["l1", "l2"]- assert sorted(monitor_ids[2:]) == ["u1", "u2"]- assert [c["observation"]["id"] for c in ordered["scorer"]] == ["u3"]- assert ordered == order_candidates(candidates, random.Random(42))---def test_parse_utc_rejects_naive_timestamps() -> None:- with pytest.raises(ValueError, match="no timezone"):- parse_utc("2026-08-01T12:00:00")- assert parse_utc("2026-08-01T12:00:00+12:00") == dt.datetime(2026, 8, 1, 0, 0, 0, tzinfo=dt.UTC)--[email protected]("age_days,fresh", [(0, True), (29, True), (31, False)])-def test_ensure_dataset_fresh_enforces_consent_ttl(age_days: int, fresh: bool) -> None:- created = (dt.datetime.now(dt.UTC) - dt.timedelta(days=age_days)).isoformat()- dataset = GoldenDataset(created_at=created, host="https://us.posthog.com", project_id=2, cases=[])- if fresh:- ensure_dataset_fresh(dataset, Path("/tmp/dataset"))- else:- with pytest.raises(RuntimeError, match="re-run collect.py"):- ensure_dataset_fresh(dataset, Path("/tmp/dataset"))---def test_apply_known_freeform_tags_only_touches_freeform_classifiers() -> None:- freeform = ClassifierScanner(prompt="x", tags=["a"], allow_freeform_tags=True)- tagged = apply_known_freeform_tags(freeform, ["search_error"])- assert isinstance(tagged, ClassifierScanner)- assert tagged.known_freeform_tags == ["search_error"]- # A fixed-vocab classifier rejects freeform output, so injecting tags there would break its validation.- fixed = ClassifierScanner(prompt="x", tags=["a"], allow_freeform_tags=False)- assert apply_known_freeform_tags(fixed, ["search_error"]) is fixed- assert apply_known_freeform_tags(freeform, []) is freeform---def test_asset_is_recorded_video_rejects_rerasterized_assets() -> None:- asset = _VideoAsset(asset_id=1, created_at=dt.datetime(2026, 8, 1, tzinfo=dt.UTC))- assert asset_is_recorded_video(asset, {"completed_at": "2026-08-01T00:00:01+00:00"})- assert not asset_is_recorded_video(asset, {"completed_at": "2026-07-31T23:59:59+00:00"})- assert not asset_is_recorded_video(asset, {"completed_at": None})---def test_known_freeform_tags_ranks_recent_observations_only() -> None:- now = dt.datetime.now(dt.UTC)-- def observation(days_ago: int, model_output: dict[str, Any] | None) -> dict[str, Any]:- return {- "created_at": (now - dt.timedelta(days=days_ago)).isoformat(),- "scanner_result": {"model_output": model_output},- }-- observations = [- observation(1, {"tags_freeform": ["Search Error", "slow page"]}),- observation(2, {"tags_freeform": ["search_error"]}),- observation(60, {"tags_freeform": ["ancient_tag"]}),- observation(0, None),- ]- assert collector._known_freeform_tags(observations) == ["search_error", "slow_page"]---_SESSION_START = "2026-08-01T12:00:00+12:00"-_SESSION_END = "2026-08-01T12:05:00+12:00"---class _FakeApi:- project_id = 2-- def hogql(self, query: str, values: dict[str, Any] | None = None) -> list[list[Any]]:- if "session_replay_events" in query:- return [["d1", _SESSION_START, _SESSION_END, 3, 4, 5, 60_000, 0, "https://example.com/a"]]- # Field order: DEFAULT_EVENT_FIELDS + _EXTRA_FIELDS.- return [- [- "$pageview",- "2026-08-01T12:00:10+12:00",- "",- [],- [],- "w1",- "https://example.com/a",- None,- "00000000-0000-0000-0000-00000000000a",- [],- [],- [],- None,- ],- [- "$autocapture",- "2026-08-01T12:01:10+12:00",- "",- [],- [],- "w1",- "https://example.com/b",- "click",- "00000000-0000-0000-0000-00000000000b",- [],- [],- [],- None,- ],- [- "file_uploaded",- "2026-08-01T12:02:10+12:00",- "",- [],- [],- "w1",- "https://example.com/b",- None,- "00000000-0000-0000-0000-00000000000c",- [],- [],- [],- None,- ],- ]---def test_event_offsets_do_not_depend_on_local_timezone(monkeypatch: pytest.MonkeyPatch) -> None:- def offsets_under(tz: str) -> dict[str, int]:- monkeypatch.setenv("TZ", tz)- time.tzset()- inputs = build_llm_inputs(_FakeApi(), 2, "sess-1") # type: ignore[arg-type]- assert inputs is not None- return inputs.event_timestamps-- try:- utc_offsets = offsets_under("UTC")- nz_offsets = offsets_under("Pacific/Auckland")- finally:- monkeypatch.undo()- time.tzset()- assert utc_offsets == nz_offsets- assert utc_offsets["00000000-0000-0000-0000-00000000000a"] == 10_000- assert utc_offsets["00000000-0000-0000-0000-00000000000b"] == 70_000---class _DefinitionsApi(_FakeApi):- def __init__(self) -> None:- self.definition_requests = 0-- def paginate(self, path: str, params: dict[str, Any] | None = None, max_items: int | None = None) -> Any:- self.definition_requests += 1- yield {"name": "file_uploaded", "description": "User uploaded a file"}---def test_build_llm_inputs_carries_prompt_context() -> None:- api = _DefinitionsApi()- lookup = collector.EventDescriptionLookup(api) # type: ignore[arg-type]- for _ in range(2):- inputs = build_llm_inputs(api, 2, "sess-1", product_context="Acme sells anvils", event_descriptions=lookup) # type: ignore[arg-type]- assert inputs is not None- assert inputs.product_context == "Acme sells anvils"- assert inputs.event_descriptions == {"file_uploaded": "User uploaded a file"}- # The description cache spans sessions, so the second build must not refetch.- assert api.definition_requests == 1---class _PagedApi:- def __init__(self, pages: list[list[list[Any]]]) -> None:- self._pages = pages- self._served = 0-- def hogql(self, query: str, values: dict[str, Any] | None = None) -> list[list[Any]]:- page = self._pages[self._served] if self._served < len(self._pages) else []- self._served += 1- return page---def test_fetch_events_caps_total_rows(monkeypatch: pytest.MonkeyPatch) -> None:- monkeypatch.setattr(collector, "_EVENTS_PER_PAGE", 2)- monkeypatch.setattr(collector, "_MAX_TOTAL_EVENT_ROWS", 5)- row = ["$pageview", _SESSION_START, "u"]- api = _PagedApi([[row, row], [row, row], [row, row], [row, row]])- start = dt.datetime(2026, 8, 1, 0, 0, 0, tzinfo=dt.UTC)- fetched = collector._fetch_events(api, "sess-1", start, start + dt.timedelta(minutes=5)) # type: ignore[arg-type]- assert len(fetched.rows) == 5- assert fetched.truncateddiff --git a/products/replay_vision/package.json b/products/replay_vision/package.jsonindex f4c1ccfe19d2..e235e1aaa9b6 100644--- a/products/replay_vision/package.json+++ b/products/replay_vision/package.json@@ -1,7 +1,7 @@{"name": "@posthog/products-replay-vision","scripts": {- "backend:test": "pytest -c ../../pytest.ini --rootdir ../.. backend/tests evals/test/ -v --tb=short"+ "backend:test": "pytest -c ../../pytest.ini --rootdir ../.. backend/tests -v --tb=short"},"dependencies": {"@posthog/quill": "workspace:*",