Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(replay): remove replay vision evals (#119865)

Radu Raicea6ae08f080827
diff --git a/.github/scripts/trunk-impacted-targets.js b/.github/scripts/trunk-impacted-targets.js
index 821962fe7536..b37bf128921a 100644
--- a/.github/scripts/trunk-impacted-targets.js
+++ b/.github/scripts/trunk-impacted-targets.js
@@ -364,11 +364,10 @@ const TRIPWIRE_RULES = [
['.github/workflows/weekly-flaky-report.yml', REPO_AUTOMATION],
['.github/workflows/weekly-slow-tests-report.yml', REPO_AUTOMATION],
// More single-suite workflows, held to the trees their suites read:
- // replay-vision evals and ClickHouse HCL checks are Python; the
+ // ClickHouse HCL checks are Python; the
// hogql parser builds wheels (python), an npm package (both families), and
// a crate (rust); deltalite spans its crates and the wheel's python
// consumers.
- ['.github/workflows/ci-replay-vision-evals.yml', PYTHON],
['.github/workflows/ci-clickhouse-hcl-schema.yml', PYTHON],
['.github/workflows/build-hogql-parser.yml', PYTHON],
['.github/workflows/build-hogql-parser-npm.yml', FULLSTACK],
diff --git a/.github/workflows/ci-replay-vision-evals.yml b/.github/workflows/ci-replay-vision-evals.yml
deleted file mode 100644
index 7d02dbd3d5c6..000000000000
--- a/.github/workflows/ci-replay-vision-evals.yml
+++ /dev/null
@@ -1,202 +0,0 @@
-# Golden-dataset eval for Replay Vision scanner prompts (products/replay_vision/evals).
-#
-# Manual dispatch only, deliberately. The suite re-runs production scans over real dogfood session
-# recordings, so a run spends secrets and LLM calls, and the number it returns moves on its own:
-# Gemini is nondeterministic and the dataset is re-sampled per run. That makes the score a
-# directional signal rather than a merge gate, which is not worth attaching to every push.
-#
-# The dataset is collected fresh on the ephemeral runner each run, which re-verifies AI
-# data-processing consent; it is never cached, uploaded as an artifact, or otherwise persisted.
-# Only aggregate scores reach the public job log and step summary.
-# If any required secret is missing the job skips green with a warning, so a fork or a
-# not-yet-configured environment is unaffected.
-name: Replay Vision Evals
-
-on:
- workflow_dispatch:
- inputs:
- per_type:
- description: 'Cases to collect per scanner type'
- default: '5'
- type: string
- trials:
- description: 'Trials per case, for variance on Gemini nondeterminism'
- default: '1'
- type: string
-
-permissions:
- contents: read
-
-env:
- DATASET_DIR: /tmp/replay-vision-golden-dataset
-
-jobs:
- golden-eval:
- name: Golden-dataset scanner eval
- timeout-minutes: 60
- runs-on: ubuntu-24.04
- env:
- DOCKERHUB_USERNAME: ${{ vars.DOCKERHUB_USER }}
- DOCKERHUB_TOKEN: ${{ secrets.DOCKERHUB_TOKEN }}
-
- steps:
- - name: Check required secrets
- id: gate
- # All five up front: one rotated key must not boot the Docker stack and collect a
- # dataset before failing inside the eval.
- env:
- REPLAY_VISION_EVAL_POSTHOG_API_KEY: ${{ secrets.REPLAY_VISION_EVAL_POSTHOG_API_KEY }}
- GEMINI_OFFLINE_EVALS_API_KEY: ${{ secrets.GEMINI_OFFLINE_EVALS_API_KEY }}
- OPENAI_OFFLINE_EVALS_API_KEY: ${{ secrets.OPENAI_OFFLINE_EVALS_API_KEY }}
- ANTHROPIC_OFFLINE_EVALS_API_KEY: ${{ secrets.ANTHROPIC_OFFLINE_EVALS_API_KEY }}
- BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
- run: |
- missing=""
- for name in REPLAY_VISION_EVAL_POSTHOG_API_KEY GEMINI_OFFLINE_EVALS_API_KEY OPENAI_OFFLINE_EVALS_API_KEY ANTHROPIC_OFFLINE_EVALS_API_KEY BRAINTRUST_API_KEY; do
- [ -n "${!name}" ] || missing="$missing $name"
- done
- if [ -n "$missing" ]; then
- echo "enabled=false" >> "$GITHUB_OUTPUT"
- echo "::warning::Missing secrets:$missing; skipping the golden-dataset eval"
- else
- echo "enabled=true" >> "$GITHUB_OUTPUT"
- fi
-
- - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
- if: steps.gate.outputs.enabled == 'true'
-
- - name: Log in to Docker Hub
- continue-on-error: true
- if: steps.gate.outputs.enabled == 'true' && env.DOCKERHUB_USERNAME != '' && env.DOCKERHUB_TOKEN != ''
- uses: docker/login-action@4907a6ddec9925e35a0a9e82d7399ccc52663121 # v4.1.0
- with:
- username: ${{ vars.DOCKERHUB_USER }}
- password: ${{ secrets.DOCKERHUB_TOKEN }}
-
- - name: Start stack with Docker Compose
- if: steps.gate.outputs.enabled == 'true'
- env:
- COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml
- run: bin/ci-wait-for-docker launch --down
-
- - name: Wait for Docker services
- if: steps.gate.outputs.enabled == 'true'
- env:
- COMPOSE_FILE: docker-compose.dev.yml:docker-compose.profiles.yml
- run: bin/ci-wait-for-docker wait
-
- - name: Set up Python
- if: steps.gate.outputs.enabled == 'true'
- uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405 # v6.2.0
- with:
- python-version-file: 'pyproject.toml'
-
- - name: Install uv
- if: steps.gate.outputs.enabled == 'true'
- uses: ./.github/actions/setup-uv
- with:
- save-cache: false
-
- - name: Install python dependencies
- if: steps.gate.outputs.enabled == 'true'
- shell: bash
- run: UV_PROJECT_ENVIRONMENT=$pythonLocation uv sync --frozen --dev
-
- - name: Install Rust
- if: steps.gate.outputs.enabled == 'true'
- uses: dtolnay/rust-toolchain@3c5f7ea28cd621ae0bf5283f0e981fb97b8a7af9
- with:
- toolchain: 1.91.1
- components: cargo
-
- - name: Install protoc
- # A one-shot suite builds the personhog binaries during preflight, and their proto
- # crate compiles .proto files in a build script. The runner image has no protoc.
- if: steps.gate.outputs.enabled == 'true'
- uses: ./.github/actions/setup-protoc
-
- - name: Install sqlx-cli
- if: steps.gate.outputs.enabled == 'true'
- uses: ./.github/actions/setup-sqlx-cli
-
- - name: Add service hostnames to /etc/hosts
- if: steps.gate.outputs.enabled == 'true'
- run: sudo echo "127.0.0.1 db redis7 kafka clickhouse clickhouse-coordinator objectstorage seaweedfs temporal" | sudo tee -a /etc/hosts
-
- - name: Collect golden dataset
- if: steps.gate.outputs.enabled == 'true'
- shell: bash
- env:
- POSTHOG_API_KEY: ${{ secrets.REPLAY_VISION_EVAL_POSTHOG_API_KEY }}
- # Django refuses to boot with the checked-in SECRET_KEY unless DEBUG is set.
- DEBUG: '1'
- # Through env, not inline ${{ }}, so a dispatch input can never inject into the shell.
- PER_TYPE: ${{ inputs.per_type }}
- run: |
- # To the runner-local file only, never the public job log: the collector emits
- # session and observation ids for missing-event and failed cases.
- python -m products.replay_vision.evals.collect \
- --project-id 2 \
- --per-type "$PER_TYPE" \
- --output "$DATASET_DIR" > /tmp/collect-output.log 2>&1 || {
- status=$?
- echo "collect exited with status $status; collector output stays off the public log"
- exit "$status"
- }
- # An empty dataset (expired assets, warehouse lag) must fail loudly, not score green over nothing.
- python - <<'EOF'
- import json, os, pathlib, sys
- cases = json.loads((pathlib.Path(os.environ["DATASET_DIR"]) / "manifest.json").read_text())["cases"]
- print(f"{len(cases)} cases in the collected dataset")
- sys.exit(1 if not cases else 0)
- EOF
-
- - name: Run scanner quality suite
- if: steps.gate.outputs.enabled == 'true'
- shell: bash
- env:
- REPLAY_VISION_EVAL_DATASET: ${{ env.DATASET_DIR }}
- GEMINI_API_KEY: ${{ secrets.GEMINI_OFFLINE_EVALS_API_KEY }}
- BRAINTRUST_API_KEY: ${{ secrets.BRAINTRUST_API_KEY }}
- # The one-shot preflight name for a plain Anthropic key (used directly, no gateway).
- LLM_GATEWAY_ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_OFFLINE_EVALS_API_KEY }}
- # The summary_alignment judge (autoevals) calls its model with the OpenAI client.
- OPENAI_API_KEY: ${{ secrets.OPENAI_OFFLINE_EVALS_API_KEY }}
- TRIALS: ${{ inputs.trials }}
- run: |
- # To the runner-local file only, never the public job log: the eval calls run_scan
- # directly, without the activity-level exception sanitizer, so a provider error can
- # echo prompt and session-derived request content. The summary step publishes the
- # safe aggregate lines.
- python -m products.posthog_ai.eval_harness.harness eval_scanner_quality \
- --trials "$TRIALS" > /tmp/eval-output.log 2>&1
-
- - name: Publish score summary
- # Publish whatever the run produced, including on failure; the suite is private, so
- # scores exist only in this runner's local logs.
- if: always() && steps.gate.outputs.enabled == 'true'
- shell: bash
- run: |
- # tee: the same sanitized lines double as the job log's only eval output.
- {
- echo "## Replay Vision scanner eval"
- echo ""
- echo "Scores compare fresh scans against recorded outputs and human thumbs labels."
- echo "Directional signal only: Gemini is nondeterministic and the dataset is re-sampled per run."
- echo ""
- echo '```'
- # Only the stable aggregate lines: this summary is public, and case payloads
- # (session-derived content) must never leave the runner even if the harness's
- # console output grows. CASE lines stay out too: their names carry observation ids.
- # The indented PostHog line links the /ai-evals experiment the harness published
- # this run's scores to. It is safe to publish: the URL carries the project id and
- # a per-run random experiment UUID, nothing case-derived.
- grep -E '^(SUITE|EXPERIMENT DONE|Sandboxed eval summary|Status:|Suites:|Cases:|Mean score:|Score gate:|Duration:| +PostHog: )' \
- /tmp/eval-output.log 2>/dev/null || echo "no eval output produced"
- echo '```'
- } | tee -a "$GITHUB_STEP_SUMMARY"
-
- - name: Remove dataset
- # The runner is ephemeral anyway; this just makes the no-persistence guarantee explicit.
- if: always()
- run: rm -rf "$DATASET_DIR"
diff --git a/products/replay_vision/backend/temporal/activities/call_scanner_provider.py b/products/replay_vision/backend/temporal/activities/call_scanner_provider.py
index fd9e694e524b..ffb709104232 100644
--- a/products/replay_vision/backend/temporal/activities/call_scanner_provider.py
+++ b/products/replay_vision/backend/temporal/activities/call_scanner_provider.py
@@ -267,12 +267,9 @@ async def run_scan(
) -> ScannerCallOutput:
"""Run the scanner conversation over an already-uploaded video, independent of where the inputs came from.
- The activity path above loads the snapshot and inputs from the observation row and Redis; the golden-dataset
- eval suite (products/replay_vision/evals) feeds this same function from files on disk so prompt changes are
- tested against the exact production pipeline. `scanner` is required (no snapshot fallback) so no caller can
+ `scanner` is required (no snapshot fallback) so no caller can
silently skip the known-freeform-tags injection. The production activity checks AI data-processing consent
- before calling this; any other caller must do the same before recording data reaches the provider (the eval
- suite is covered because dataset collection is consent-gated and time-boxed).
+ before calling this; any other caller must do the same before recording data reaches the provider.
"""
# Built before the preamble so one object decides both the wording and what a network lookup returns, which
# keeps the prompt from promising network data the scan does not carry.
diff --git a/products/replay_vision/evals/README.md b/products/replay_vision/evals/README.md
deleted file mode 100644
index 66bd62afb511..000000000000
--- a/products/replay_vision/evals/README.md
+++ /dev/null
@@ -1,84 +0,0 @@
-# Replay Vision golden-dataset evals
-
-Test scanner prompt changes against a fixed set of real, already-observed sessions instead of shipping and watching production.
-The suite re-runs the exact production scan pipeline (`run_scan`: same Jinja templates, response schemas, lookup round) over collected videos, then scores the fresh output against the recorded output and its human thumbs label.
-
-## The loop
-
-1. Collect a dataset once (and re-collect occasionally as labels accumulate).
-2. Run the suite to get a baseline.
-3. Edit prompt templates under `../backend/temporal/scanners/prompts/`.
-4. Re-run the suite and compare scores across runs (same experiment name accumulates history).
-
-## Collecting a dataset
-
-Uses only the public API plus the dogfood warehouse, with a personal API key that can read the source project (scanner read + session recording read + query scopes):
-
-```bash
-POSTHOG_API_KEY=... python -m products.replay_vision.evals.collect \
- --project-id 2 --per-type 25 --output ~/.posthog/replay-vision-golden-dataset
-```
-
-Selection is per scanner type: human-labeled observations first (thumbs up/down carry ground truth), then a seeded uniform sample of unlabeled ones.
-Only observations whose original rasterized MP4 still exists are collectable: the system assets expire after 90 days, a session re-rasterized after that is a different video than the one behind the recorded output (so those are skipped too), and the warehouse copy of the exports table lags up to a day, so very fresh observations are skipped.
-Each case also captures the prompt context the production scan carried: the project's product context (Max core memory needs a full-access personal key; otherwise the project description is used), the session's custom-event descriptions, and, for freeform classifiers, the known-tag vocabulary.
-Re-running the collector is idempotent per case: completed cases are reused as-is, half-written ones are re-collected, and the manifest is merged with previously collected cases that are still valid on disk.
-After a collector change that alters what a case captures, collect into a fresh directory so reused cases don't keep the old shape.
-
-## Running the suite
-
-```bash
-REPLAY_VISION_EVAL_DATASET=~/.posthog/replay-vision-golden-dataset \
-GEMINI_API_KEY=... \
-LLM_GATEWAY_ANTHROPIC_API_KEY=... \
-BRAINTRUST_API_KEY=... \
-hogli evals eval_scanner_quality
-```
-
-`GEMINI_API_KEY` runs the scans themselves.
-`LLM_GATEWAY_ANTHROPIC_API_KEY` satisfies the suite env preflight (judge models are routed through the internal LLM gateway).
-`BRAINTRUST_API_KEY` is required by the harness engine even though this private suite only logs locally.
-Without `REPLAY_VISION_EVAL_DATASET` the suite logs a warning and runs nothing, so it never breaks a full `hogli evals` run.
-Use `--trials N` for variance on Gemini nondeterminism and `--eval <case-substring>` for one case.
-
-## Scorers
-
-| Scorer | Applies to | Meaning |
-| ------------------- | ---------------------------------------- | ----------------------------------------------------------------------------- |
-| `scan_completed` | all cases | The scan produced schema-valid, semantically-valid output. |
-| `labeled_outcome` | labeled monitor/classifier | kept/fixed = 1, regressed/still_wrong = 0 |
-| `output_stability` | unlabeled monitor/classifier | Fresh outcome matches the recorded baseline; measures churn, not correctness. |
-| `score_alignment` | scorer (reference not thumbs-downed) | 1 minus the scale-normalized distance from the recorded score. |
-| `summary_alignment` | summarizer (reference not thumbs-downed) | LLM judge: does the fresh summary tell the same story as the recorded one? |
-
-`output_stability` and `labeled_outcome` deliberately pull in opposite directions, so read them as a pair: a prompt change that only adds churn shows up as `labeled_outcome` flat or up while `output_stability` drops.
-
-Primary verdict agreement and signal counts do not measure recommendation precision.
-The label scorer compares primary outcomes; it does not check each signal's evidence or usefulness.
-Model confidence does not establish whether a signal supports a useful recommendation.
-Successful scans retain the complete structured `signals` list in the private per-case output alongside `signals_count`; no findings produces `signals: []`.
-Compare each finding's type, time range, URL, description, and confidence with the recording before using it as recommendation evidence.
-This suite does not calculate recommendation precision.
-
-## Running it on a GitHub runner
-
-`.github/workflows/ci-replay-vision-evals.yml` runs the same loop on an ephemeral runner, on manual dispatch only, with `per_type` and `trials` as inputs (GitHub only offers the dispatch once the workflow is on master).
-It is deliberately not attached to `pull_request`: Gemini is nondeterministic and the dataset is re-sampled per run, so the score is a directional signal rather than a merge gate, and spending secrets and LLM calls on every push buys nothing that a dispatch after a prompt change does not.
-
-It collects a fresh dataset on the runner (so consent is re-verified every run; nothing is cached, uploaded, or persisted), runs the suite, and writes the aggregate scores to the job's step summary.
-Only those allowlisted summary lines are public: collector and harness output stay in runner-local files, because they carry session and observation ids.
-The job needs `REPLAY_VISION_EVAL_POSTHOG_API_KEY` (a personal API key with scanner, session recording, export, and query read access to the dogfood project) plus the `GEMINI_OFFLINE_EVALS_API_KEY`, `OPENAI_OFFLINE_EVALS_API_KEY`, `ANTHROPIC_OFFLINE_EVALS_API_KEY`, and `BRAINTRUST_API_KEY` secrets; if any is missing the job skips green and warns which.
-The dedicated provider secrets are exposed to the suite as `GEMINI_API_KEY`, `OPENAI_API_KEY`, and `LLM_GATEWAY_ANTHROPIC_API_KEY`.
-
-## Data handling
-
-The dataset contains real session recordings and event data.
-
-- Keep it in a local or internal location only; never commit it, upload it, or reference its contents in PRs.
-- A dataset expires 30 days after its last collection: the suite refuses to run it, because the consent verification (and the recordings themselves) can lapse after collection. Re-running collect.py re-verifies consent and refreshes the manifest.
-- The suite is `OneShotPrivateEval`, so per-case logs stay in the local `eval_harness/logs/` directory and nothing goes to Braintrust.
-- The retained `signals` can contain session content. Keep these payloads private; never include them in public CI summaries, commits, or pull requests.
-- Dataset-derived content reaches two model providers: the Gemini scans use the same provider call production makes through `run_scan`, and the `summary_alignment` judge sends the recorded and fresh summaries to `gpt-5.4`.
-- The judge calls the OpenAI API directly with `OPENAI_API_KEY`. It does not go through the internal LLM gateway, which a one-shot suite never starts, so nothing sits in front of the provider on that path.
-- This private suite does not publish scores to PostHog's offline experiments. Results remain in local logs and the aggregate CI summary.
-- Collecting from projects other than PostHog's own requires a data-governance decision first; see the product's consent gating (`backend/consent.py`) and note that customer-facing copy does not cover internal reuse today.
diff --git a/products/replay_vision/evals/__init__.py b/products/replay_vision/evals/__init__.py
deleted file mode 100644
index 37fea886e23c..000000000000
--- a/products/replay_vision/evals/__init__.py
+++ /dev/null
@@ -1 +0,0 @@
-"""Replay Vision golden-dataset eval suites (see README.md in this directory)."""
diff --git a/products/replay_vision/evals/collect.py b/products/replay_vision/evals/collect.py
deleted file mode 100644
index 04e40803fa36..000000000000
--- a/products/replay_vision/evals/collect.py
+++ /dev/null
@@ -1,70 +0,0 @@
-"""Collect a Replay Vision golden dataset from an existing project's observations.
-
-Usage, from the repo root, with a personal API key that can read the source project:
-
- POSTHOG_API_KEY=... python -m products.replay_vision.evals.collect \\
- --project-id 2 --per-type 25 --output ~/.posthog/replay-vision-golden-dataset
-
-Then point REPLAY_VISION_EVAL_DATASET at the output directory to run the eval suite.
-The dataset contains real session data: keep it out of the repo and off any public surface.
-"""
-
-import os
-import argparse
-from pathlib import Path
-
-import django
-
-_DEFAULT_OUTPUT = "~/.posthog/replay-vision-golden-dataset"
-
-
-def main() -> None:
- parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter)
- parser.add_argument("--host", default="https://us.posthog.com", help="PostHog instance to collect from.")
- parser.add_argument("--project-id", type=int, default=2, help="Source project (default: PostHog dogfood).")
- parser.add_argument("--output", default=_DEFAULT_OUTPUT, help="Dataset directory to write.")
- parser.add_argument("--per-type", type=int, default=25, help="Cases to collect per scanner type.")
- parser.add_argument(
- "--scanner-id", action="append", dest="scanner_ids", help="Restrict to specific scanner ids (repeatable)."
- )
- parser.add_argument("--seed", type=int, default=42, help="Seed for the uniform sample, for reproducibility.")
- parser.add_argument(
- "--max-observations",
- type=int,
- default=200,
- help="Newest observations to consider per scanner; lower paginates much faster.",
- )
- args = parser.parse_args()
-
- # Env var only, deliberately: a key passed as a CLI flag leaks into shell history and ps output.
- api_key = os.environ.get("POSTHOG_API_KEY", "")
- if not api_key:
- parser.error("set POSTHOG_API_KEY")
-
- os.environ.setdefault("DJANGO_SETTINGS_MODULE", "posthog.settings")
- django.setup()
- from products.replay_vision.evals.collector import collect # noqa: PLC0415 - needs Django configured first
-
- dataset = collect(
- host=args.host,
- project_id=args.project_id,
- api_key=api_key,
- output=Path(args.output).expanduser(),
- per_type=args.per_type,
- scanner_ids=args.scanner_ids,
- seed=args.seed,
- max_observations_per_scanner=args.max_observations,
- )
- by_type: dict[str, int] = {}
- for case in dataset.cases:
- by_type[case.scanner_type] = by_type.get(case.scanner_type, 0) + 1
- labeled = sum(1 for case in dataset.cases if case.label_is_correct is not None)
- print(f"Collected {len(dataset.cases)} cases ({labeled} labeled): {by_type}") # noqa: T201
- print( # noqa: T201
- f"Run the suite with: REPLAY_VISION_EVAL_DATASET={args.output} GEMINI_API_KEY=... "
- "LLM_GATEWAY_ANTHROPIC_API_KEY=... BRAINTRUST_API_KEY=... hogli evals eval_scanner_quality"
- )
-
-
-if __name__ == "__main__":
- main()
diff --git a/products/replay_vision/evals/collector.py b/products/replay_vision/evals/collector.py
deleted file mode 100644
index 1aaf53e3cd33..000000000000
--- a/products/replay_vision/evals/collector.py
+++ /dev/null
@@ -1,607 +0,0 @@
-"""Implementation of the golden-dataset collector; run it via collect.py (needs Django configured).
-
-Everything goes through a PostHog instance's public API with a personal API key, so it works
-against any project the key can read: the replay-vision scanner/observation endpoints drive the
-selection, the synced ``postgres.posthog_exportedasset`` warehouse table locates each session's
-rasterized MP4 (system assets are invisible to the exports list endpoint), the exports content
-endpoint downloads the bytes, and HogQL queries mirroring ``fetch_session_events`` rebuild each
-session's ``ScannerLlmInputs``. The core-memory, project, and event-definition endpoints rebuild
-the prompt context (``product_context``, ``event_descriptions``) the production preamble carries.
-"""
-
-import os
-import json
-import random
-import datetime as dt
-from collections.abc import Iterator
-from dataclasses import field
-from pathlib import Path
-from typing import Any
-
-import requests
-import structlog
-
-from posthog.dataclasses import frozen
-from posthog.session_recordings.queries.session_replay_events import DEFAULT_EVENT_FIELDS
-
-from products.replay_vision.backend.queries.session_identity import (
- IDENTITY_TIMESTAMP_SLACK,
- SESSION_PERSON_IDENTITY_QUERY,
- person_display_name,
- person_email,
- person_organization,
- person_properties_from_row,
-)
-from products.replay_vision.backend.temporal.activities.call_scanner_provider import (
- _KNOWN_FREEFORM_TAGS_DAYS,
- _KNOWN_FREEFORM_TAGS_MAX_ROWS,
- rank_freeform_tags,
-)
-from products.replay_vision.backend.temporal.activities.ensure_session_asset import (
- _EXPORT_FORMAT,
- _MOUSE_TAIL,
- _PLAYBACK_SPEED,
- _RECORDING_FPS,
- _SHOW_METADATA_FOOTER,
-)
-from products.replay_vision.backend.temporal.activities.fetch_session_events import (
- _EVENTS_PER_PAGE,
- _EVENTS_TO_IGNORE,
- _EXTRA_FIELDS,
- _MAX_TOTAL_EVENT_ROWS,
- _process_events,
-)
-from products.replay_vision.backend.temporal.team_context import (
- sanitize_product_context,
- select_event_descriptions,
- session_custom_event_names,
-)
-from products.replay_vision.backend.temporal.types import (
- EventTable,
- ScannerLlmInputs,
- ScannerSnapshot,
- SessionIdentity,
- SessionMetadata,
-)
-from products.replay_vision.evals.dataset import (
- MANIFEST_NAME,
- GoldenCase,
- GoldenDataset,
- load_dataset,
- parse_utc,
- save_dataset,
-)
-
-logger = structlog.get_logger(__name__)
-
-# The exports table syncs into the warehouse with up to ~a day of lag, so freshly-created
-# observations often have no findable asset yet; we over-fetch candidates and skip those.
-_ASSET_LOOKUP_CHUNK = 100
-_OBSERVATION_PAGE = 100
-_EVENT_DEFINITION_NAMES_CHUNK = 100
-
-
-class PostHogApi:
- """Minimal authenticated client for the endpoints the collector needs."""
-
- def __init__(self, host: str, project_id: int, api_key: str) -> None:
- self.host = host.rstrip("/")
- self.project_id = project_id
- self._session = requests.Session()
- self._session.headers["Authorization"] = f"Bearer {api_key}"
-
- def get_json(self, path: str, params: dict[str, Any] | None = None) -> dict[str, Any]:
- response = self._session.get(f"{self.host}{path}", params=params, timeout=60)
- response.raise_for_status()
- return response.json()
-
- def paginate(
- self, path: str, params: dict[str, Any] | None = None, max_items: int | None = None
- ) -> Iterator[dict[str, Any]]:
- url: str | None = f"{self.host}{path}"
- yielded = 0
- while url:
- response = self._session.get(url, params=params, timeout=60)
- response.raise_for_status()
- data = response.json()
- for item in data.get("results", []):
- yield item
- yielded += 1
- if max_items is not None and yielded >= max_items:
- return
- url = data.get("next")
- params = None
-
- def hogql(self, query: str, values: dict[str, Any] | None = None) -> list[list[Any]]:
- payload = {"query": {"kind": "HogQLQuery", "query": query, "values": values or {}}}
- response = self._session.post(
- f"{self.host}/api/environments/{self.project_id}/query/", json=payload, timeout=120
- )
- response.raise_for_status()
- return response.json().get("results") or []
-
- def download(self, path: str, target: Path) -> None:
- # The content endpoint 302s to a presigned S3 URL; requests drops the auth header on the
- # cross-host redirect, which is exactly right for a presigned link.
- partial = target.with_name(target.name + ".partial")
- with self._session.get(f"{self.host}{path}", timeout=300, stream=True) as response:
- response.raise_for_status()
- with partial.open("wb") as handle:
- # iter_content applies the transport decoding (gzip/deflate) that response.raw skips.
- for chunk in response.iter_content(chunk_size=1 << 20):
- handle.write(chunk)
- # Publish atomically so an interrupted download can never be mistaken for a complete file.
- os.replace(partial, target)
-
-
-def order_candidates(candidates: list[dict[str, Any]], rng: random.Random) -> dict[str, list[dict[str, Any]]]:
- """Per scanner type: labeled observations first (they carry ground truth), then the rest uniformly shuffled."""
- by_type: dict[str, list[dict[str, Any]]] = {}
- for candidate in candidates:
- by_type.setdefault(candidate["scanner_type"], []).append(candidate)
- ordered: dict[str, list[dict[str, Any]]] = {}
- for scanner_type, group in by_type.items():
- labeled = [c for c in group if c["observation"].get("label")]
- unlabeled = [c for c in group if not c["observation"].get("label")]
- rng.shuffle(unlabeled)
- ordered[scanner_type] = labeled + unlabeled
- return ordered
-
-
-@frozen
-class _VideoAsset:
- asset_id: int
- created_at: dt.datetime
- # Absent on assets rendered before the rasterizer recorded its cut map.
- inactivity_periods: list[dict[str, Any]] = field(default_factory=list)
- # Absent on assets rendered before the renderer recorded the page and tab changes it shows.
- navigation: list[dict[str, Any]] = field(default_factory=list)
-
-
-def _load_json_list(raw: Any) -> list[dict[str, Any]]:
- """A list the rasterizer wrote into `export_context`, empty on assets rendered before it wrote it."""
- if not raw:
- return []
- try:
- parsed = json.loads(raw)
- except (TypeError, ValueError):
- return []
- return parsed if isinstance(parsed, list) else []
-
-
-def lookup_video_assets(api: PostHogApi, team_id: int, session_ids: list[str]) -> dict[str, _VideoAsset]:
- """Map session id to its system rasterized-MP4 asset, for sessions that still have one.
-
- min(id) mirrors ensure_session_asset's oldest-first get-or-create, so this is the asset the
- production scans actually watched, for as long as it lives.
- """
- found: dict[str, _VideoAsset] = {}
- for start in range(0, len(session_ids), _ASSET_LOOKUP_CHUNK):
- chunk = session_ids[start : start + _ASSET_LOOKUP_CHUNK]
- rows = api.hogql(
- """
- SELECT JSONExtractString(toString(export_context), 'session_recording_id') AS sid,
- min(id) AS asset_id,
- argMin(toTimeZone(created_at, 'UTC'), id) AS asset_created_at,
- argMin(JSONExtractString(toString(export_context), 'inactivity_periods'), id) AS inactivity_periods,
- argMin(JSONExtractRaw(toString(export_context), 'navigation'), id) AS navigation
- FROM postgres.posthog_exportedasset
- WHERE team_id = {team_id}
- AND export_format = {export_format}
- AND is_system
- AND expires_after > now()
- AND JSONExtractFloat(toString(export_context), 'playback_speed') = {playback_speed}
- AND JSONExtractInt(toString(export_context), 'recording_fps') = {recording_fps}
- AND JSONExtractBool(toString(export_context), 'show_metadata_footer') = {show_metadata_footer}
- AND JSONExtractBool(toString(export_context), 'mouse_tail') = {mouse_tail}
- AND JSONExtractString(toString(export_context), 'session_recording_id') IN {session_ids}
- GROUP BY sid
- """,
- {
- "team_id": team_id,
- "export_format": _EXPORT_FORMAT,
- "playback_speed": _PLAYBACK_SPEED,
- "recording_fps": _RECORDING_FPS,
- "show_metadata_footer": _SHOW_METADATA_FOOTER,
- "mouse_tail": _MOUSE_TAIL,
- "session_ids": chunk,
- },
- )
- for sid, asset_id, created_raw, periods_raw, navigation_raw in rows:
- found[str(sid)] = _VideoAsset(
- asset_id=int(asset_id),
- created_at=parse_utc(created_raw),
- inactivity_periods=_load_json_list(periods_raw),
- navigation=_load_json_list(navigation_raw),
- )
- return found
-
-
-def asset_is_recorded_video(asset: _VideoAsset, observation: dict[str, Any]) -> bool:
- """An asset created after the observation completed is a re-rasterization from after the original
- expired: not the video behind recorded_output, so the case would score the wrong footage."""
- completed_raw = observation.get("completed_at")
- return bool(completed_raw) and asset.created_at <= parse_utc(completed_raw)
-
-
-# toTimeZone pins the timestamps to UTC regardless of the project timezone, so parse_utc never
-# sees a value it would have to guess a timezone for.
-_METADATA_QUERY = """
-SELECT distinct_id, toTimeZone(start_time, 'UTC') AS start_time, toTimeZone(end_time, 'UTC') AS end_time,
- click_count, keypress_count, mouse_activity_count,
- active_milliseconds, console_error_count, first_url
-FROM session_replay_events
-WHERE session_id = {session_id}
-"""
-
-
-@frozen
-class _SessionEvents:
- columns: list[str]
- rows: list[list[Any]]
- truncated: bool
-
-
-def _fetch_events(api: PostHogApi, session_id: str, start: dt.datetime, end: dt.datetime) -> _SessionEvents:
- fields = [*DEFAULT_EVENT_FIELDS, *_EXTRA_FIELDS]
- select_exprs = [f"toTimeZone({field}, 'UTC') AS timestamp" if field == "timestamp" else field for field in fields]
- # The uuid tie-break makes OFFSET paging deterministic when many events share a timestamp.
- query = (
- f"SELECT {', '.join(select_exprs)} FROM events"
- " WHERE timestamp >= {start_time} AND timestamp <= {end_time} AND $session_id = {session_id}"
- " AND event NOT IN {events_to_ignore}"
- " ORDER BY timestamp ASC, uuid ASC LIMIT {limit} OFFSET {offset}"
- )
- rows: list[list[Any]] = []
- truncated = False
- offset = 0
- while True:
- page = api.hogql(
- query,
- {
- # Same 100s wiggle as production get_events_query: the range only bounds the scan.
- "start_time": (start - dt.timedelta(seconds=100)).isoformat(),
- "end_time": (end + dt.timedelta(seconds=100)).isoformat(),
- "session_id": session_id,
- "events_to_ignore": _EVENTS_TO_IGNORE,
- "limit": _EVENTS_PER_PAGE,
- "offset": offset,
- },
- )
- rows.extend(list(row) for row in page)
- if len(rows) >= _MAX_TOTAL_EVENT_ROWS:
- # Same cap as the production fetch: eligibility bounds active seconds, not event count,
- # so an instrumentation loop can still emit millions of rows.
- truncated = len(rows) > _MAX_TOTAL_EVENT_ROWS or len(page) == _EVENTS_PER_PAGE
- del rows[_MAX_TOTAL_EVENT_ROWS:]
- break
- if len(page) < _EVENTS_PER_PAGE:
- break
- offset += _EVENTS_PER_PAGE
- return _SessionEvents(columns=list(fields), rows=rows, truncated=truncated)
-
-
-class EventDescriptionLookup:
- """API-backed twin of team_context.fetch_event_descriptions, cached across sessions in one run."""
-
- def __init__(self, api: PostHogApi) -> None:
- self._api = api
- self._cache: dict[str, str] = {}
-
- def for_session(self, columns: list[str], rows: list[list[Any]]) -> dict[str, str]:
- names = session_custom_event_names(columns, rows)
- missing = [name for name in names if name not in self._cache]
- for start in range(0, len(missing), _EVENT_DEFINITION_NAMES_CHUNK):
- chunk = missing[start : start + _EVENT_DEFINITION_NAMES_CHUNK]
- # Pre-cache the whole chunk as undescribed so names the API doesn't return are not refetched.
- self._cache.update(dict.fromkeys(chunk, ""))
- for definition in self._api.paginate(
- f"/api/projects/{self._api.project_id}/event_definitions/",
- {"names": ",".join(chunk), "limit": _EVENT_DEFINITION_NAMES_CHUNK},
- ):
- self._cache[str(definition.get("name") or "")] = str(definition.get("description") or "")
- return select_event_descriptions(names, self._cache)
-
-
-def fetch_product_context_via_api(api: PostHogApi) -> str:
- """Product context the way production builds it: Max core memory, then the project's product description."""
- text = ""
- try:
- results = api.get_json(f"/api/projects/{api.project_id}/core_memory/").get("results") or []
- text = str(results[0].get("text") or "").strip() if results else ""
- except requests.HTTPError as exc:
- # Core memory is an INTERNAL-scope endpoint, unreadable with a scoped personal key; degrade to the fallback.
- logger.warning("collector.core_memory_unreadable", error=str(exc))
- if not text:
- text = str(api.get_json(f"/api/projects/{api.project_id}/").get("product_description") or "").strip()
- return sanitize_product_context(text)
-
-
-def build_llm_inputs(
- api: PostHogApi,
- team_id: int,
- session_id: str,
- *,
- product_context: str = "",
- event_descriptions: EventDescriptionLookup | None = None,
-) -> ScannerLlmInputs | None:
- """Rebuild the ScannerLlmInputs production stashed in Redis, from the query API instead of ClickHouse."""
- metadata_rows = api.hogql(_METADATA_QUERY, {"session_id": session_id})
- if not metadata_rows:
- return None
- (distinct_id, start_raw, end_raw, clicks, keypresses, mouse, active_ms, console_errors, first_url) = metadata_rows[
- 0
- ]
- start, end = parse_utc(start_raw), parse_utc(end_raw)
- duration_seconds = (end - start).total_seconds()
-
- fetched = _fetch_events(api, session_id, start, end)
- if not fetched.rows:
- return None
- # _process_events needs real datetimes to compute per-event offsets from session start.
- timestamp_index = fetched.columns.index("timestamp")
- for row in fetched.rows:
- row[timestamp_index] = parse_utc(row[timestamp_index])
- # HogQL surfaces `properties.$window_id` as a bare `$window_id` column, matching production's runner output.
- columns = [column.removeprefix("properties.") for column in fetched.columns]
- processed = _process_events(columns, fetched.rows, session_start=start)
-
- active_seconds = float(active_ms or 0) / 1000
- return ScannerLlmInputs(
- session_id=session_id,
- team_id=team_id,
- events_truncated=fetched.truncated,
- events=EventTable(columns=processed.columns, rows=processed.rows),
- url_mapping=processed.url_mapping,
- window_mapping=processed.window_mapping,
- event_timestamps=processed.event_timestamps,
- product_context=product_context,
- event_descriptions=event_descriptions.for_session(processed.columns, processed.rows)
- if event_descriptions
- else {},
- distinct_id=str(distinct_id) if distinct_id else None,
- identity=_build_identity(api, session_id, str(distinct_id) if distinct_id else None, start, end),
- metadata=SessionMetadata(
- start_time=start,
- end_time=end,
- duration_seconds=duration_seconds,
- active_seconds=active_seconds,
- inactive_seconds=max(0.0, duration_seconds - active_seconds),
- click_count=clicks,
- keypress_count=keypresses,
- mouse_activity_count=mouse,
- start_url=first_url or None,
- console_error_count=console_errors,
- ),
- )
-
-
-def _build_identity(
- api: PostHogApi, session_id: str, distinct_id: str | None, start: dt.datetime, end: dt.datetime
-) -> SessionIdentity:
- """The identity production renders into the preamble, read through the query API.
-
- Runs the same person query production runs, bound to the same subject, so a collected case exercises the
- same prompt. Group names are left out: they need the project's group-type labels, and a case without them
- still renders the block.
- """
- if not distinct_id:
- return SessionIdentity()
- rows = api.hogql(
- SESSION_PERSON_IDENTITY_QUERY,
- {
- "session_id": session_id,
- "distinct_id": distinct_id,
- "start": (start - IDENTITY_TIMESTAMP_SLACK).isoformat(),
- "end": (end + IDENTITY_TIMESTAMP_SLACK).isoformat(),
- },
- )
- if not rows:
- return SessionIdentity()
- properties = person_properties_from_row(rows[0])
- return SessionIdentity(
- person_email=person_email(properties),
- person_name=person_display_name(properties),
- person_organization=person_organization(properties),
- )
-
-
-def _known_freeform_tags(observations: list[dict[str, Any]]) -> list[str]:
- """Approximate the tag vocabulary production injects at scan time (recent window, ranked by frequency);
- the vocabulary each recorded scan actually saw is not reconstructable after the fact."""
- cutoff = dt.datetime.now(dt.UTC) - dt.timedelta(days=_KNOWN_FREEFORM_TAGS_DAYS)
- recent = [
- ((observation.get("scanner_result") or {}).get("model_output") or {}).get("tags_freeform")
- for observation in observations
- if observation.get("created_at") and parse_utc(observation["created_at"]) >= cutoff
- ]
- return rank_freeform_tags(recent[:_KNOWN_FREEFORM_TAGS_MAX_ROWS])
-
-
-def _fetch_candidates(
- api: PostHogApi, scanner_ids: list[str] | None, max_observations_per_scanner: int
-) -> list[dict[str, Any]]:
- candidates: list[dict[str, Any]] = []
- for scanner in api.paginate(f"/api/projects/{api.project_id}/vision/scanners/", {"limit": 100}):
- if scanner_ids and scanner["id"] not in scanner_ids:
- continue
- observations = list(
- api.paginate(
- f"/api/projects/{api.project_id}/vision/scanners/{scanner['id']}/observations/",
- {"status": "succeeded", "limit": _OBSERVATION_PAGE},
- max_items=max_observations_per_scanner,
- )
- )
- known_tags = _known_freeform_tags(observations) if scanner["scanner_type"] == "classifier" else []
- for observation in observations:
- if not (observation.get("scanner_result") or {}).get("model_output"):
- continue
- candidates.append(
- {
- "observation": observation,
- "scanner": scanner,
- "scanner_type": scanner["scanner_type"],
- "known_freeform_tags": known_tags,
- }
- )
- return candidates
-
-
-@frozen
-class _SourceProject:
- """Per-run context of the project being collected from, shared by every case."""
-
- team_id: int
- team_name: str
- product_context: str
- event_descriptions: EventDescriptionLookup
-
-
-def _write_case(
- api: PostHogApi, root: Path, candidate: dict[str, Any], asset: _VideoAsset, source: _SourceProject
-) -> GoldenCase | None:
- observation = candidate["observation"]
- scanner = candidate["scanner"]
- label = observation.get("label") or {}
- case = GoldenCase(
- case_id=observation["id"],
- scanner_id=scanner["id"],
- scanner_name=scanner["name"],
- scanner_type=scanner["scanner_type"],
- session_id=observation["session_id"],
- team_id=source.team_id,
- team_name=source.team_name,
- snapshot=ScannerSnapshot.model_validate(observation["scanner_snapshot"]),
- recorded_output=observation["scanner_result"]["model_output"],
- known_freeform_tags=candidate.get("known_freeform_tags") or [],
- label_is_correct=label.get("is_correct"),
- label_feedback=label.get("feedback") or "",
- inactivity_periods=asset.inactivity_periods,
- navigation=asset.navigation,
- collected_at=dt.datetime.now(dt.UTC).isoformat(),
- )
- case_dir = case.case_dir(root)
- if case.video_path(root).exists() and case.inputs_path(root).exists():
- try:
- case.load_inputs(root)
- return case
- except Exception:
- logger.warning("collector.reused_case_invalid", case_id=case.case_id)
-
- inputs = build_llm_inputs(
- api,
- source.team_id,
- case.session_id,
- product_context=source.product_context,
- event_descriptions=source.event_descriptions,
- )
- if inputs is None:
- logger.warning("collector.no_events_for_session", session_id=case.session_id)
- return None
- case_dir.mkdir(parents=True, exist_ok=True)
- api.download(
- f"/api/environments/{api.project_id}/exports/{asset.asset_id}/content/?download=true", case.video_path(root)
- )
- # inputs.json lands last, so the reuse shortcut above only ever sees fully written cases.
- case.inputs_path(root).write_text(inputs.model_dump_json())
- return case
-
-
-def _reusable_existing_cases(root: Path) -> dict[str, GoldenCase]:
- """Previously collected cases whose files are still present and parseable, keyed by case id."""
- if not (root / MANIFEST_NAME).exists():
- return {}
- try:
- existing = load_dataset(root)
- except Exception:
- logger.warning("collector.existing_manifest_unreadable", root=str(root))
- return {}
- reusable: dict[str, GoldenCase] = {}
- for case in existing.cases:
- try:
- case.load_inputs(root)
- except Exception:
- continue
- if case.video_path(root).exists():
- reusable[case.case_id] = case
- return reusable
-
-
-def _check_ai_consent(api: PostHogApi, environment: dict[str, Any]) -> None:
- """The scan pipeline only runs over consented orgs; collecting recordings for evals holds the same bar."""
- organization = api.get_json(f"/api/organizations/{environment['organization']}/")
- if not organization.get("is_ai_data_processing_approved"):
- raise RuntimeError(
- "The source project's organization has not approved AI data processing; "
- "collecting recordings for LLM evals requires that consent"
- )
-
-
-def collect(
- *,
- host: str,
- project_id: int,
- api_key: str,
- output: Path,
- per_type: int,
- scanner_ids: list[str] | None = None,
- seed: int = 42,
- max_observations_per_scanner: int = 200,
-) -> GoldenDataset:
- api = PostHogApi(host, project_id, api_key)
- # The environments API is keyed by team id; resolve the real team id once instead of assuming
- # it equals the project id, and fail loudly when the two point at different projects.
- environment = api.get_json(f"/api/environments/{project_id}/")
- team_id = int(environment["id"])
- if int(environment["project_id"]) != project_id:
- raise RuntimeError(f"Environment {team_id} belongs to project {environment['project_id']}, not {project_id}")
- _check_ai_consent(api, environment)
- source = _SourceProject(
- team_id=team_id,
- team_name=str(environment.get("name", "")),
- product_context=fetch_product_context_via_api(api),
- event_descriptions=EventDescriptionLookup(api),
- )
- rng = random.Random(seed)
-
- candidates = _fetch_candidates(api, scanner_ids, max_observations_per_scanner)
- ordered = order_candidates(candidates, rng)
- all_session_ids = sorted({c["observation"]["session_id"] for c in candidates})
- assets = lookup_video_assets(api, team_id, all_session_ids)
- logger.info(
- "collector.candidates",
- candidates=len(candidates),
- with_video_asset=sum(1 for c in candidates if c["observation"]["session_id"] in assets),
- types={t: len(group) for t, group in ordered.items()},
- )
-
- cases: list[GoldenCase] = []
- for scanner_type, group in sorted(ordered.items()):
- collected = 0
- for candidate in group:
- if collected >= per_type:
- break
- session_id = candidate["observation"]["session_id"]
- asset = assets.get(session_id)
- if asset is None or not asset_is_recorded_video(asset, candidate["observation"]):
- continue
- try:
- case = _write_case(api, output, candidate, asset, source)
- except requests.HTTPError as exc:
- logger.warning("collector.case_failed", observation_id=candidate["observation"]["id"], error=str(exc))
- continue
- if case is None:
- continue
- cases.append(case)
- collected += 1
- logger.info("collector.type_done", scanner_type=scanner_type, collected=collected, target=per_type)
-
- # Merge with previously collected cases so re-runs extend the dataset instead of orphaning the
- # case folders of earlier runs; this run's freshly collected version wins on overlap.
- merged = _reusable_existing_cases(output)
- merged.update({case.case_id: case for case in cases})
- dataset = GoldenDataset(
- created_at=dt.datetime.now(dt.UTC).isoformat(), host=host, project_id=project_id, cases=list(merged.values())
- )
- save_dataset(output, dataset)
- return dataset
diff --git a/products/replay_vision/evals/dataset.py b/products/replay_vision/evals/dataset.py
deleted file mode 100644
index 605c407a0228..000000000000
--- a/products/replay_vision/evals/dataset.py
+++ /dev/null
@@ -1,108 +0,0 @@
-"""Golden-dataset layout shared by the collector (collect.py) and the eval suite.
-
-A dataset is a plain directory, never committed to the repo (it contains real session data):
-
- manifest.json # GoldenDataset
- cases/<case_id>/video.mp4 # rasterized recording, byte-identical to what production sent to Gemini
- cases/<case_id>/inputs.json # ScannerLlmInputs snapshot (events table, session metadata)
-"""
-
-import os
-import datetime as dt
-from pathlib import Path
-from typing import Any
-
-from pydantic import BaseModel, Field
-
-from products.replay_vision.backend.temporal.types import ScannerLlmInputs, ScannerSnapshot
-
-DATASET_ENV_VAR = "REPLAY_VISION_EVAL_DATASET"
-MANIFEST_NAME = "manifest.json"
-VIDEO_NAME = "video.mp4"
-INPUTS_NAME = "inputs.json"
-# Production re-checks AI data-processing consent right before every generation; for the eval the only
-# check happens when collect.py runs, so bound how long that verification (and the collected recordings
-# themselves) may be trusted.
-DATASET_MAX_AGE_DAYS = 30
-
-
-def parse_utc(raw: Any) -> dt.datetime:
- """Parse an ISO timestamp into UTC; a naive string would silently shift by this machine's timezone, so reject it."""
- parsed = dt.datetime.fromisoformat(str(raw))
- if parsed.tzinfo is None:
- raise ValueError(f"timestamp {raw!r} has no timezone; expected an offset-aware ISO string")
- return parsed.astimezone(dt.UTC)
-
-
-class GoldenCase(BaseModel, frozen=True):
- """One collected observation: the frozen scanner config, its recorded output, and the human label if any."""
-
- case_id: str = Field(description="Source ReplayObservation id; doubles as the case directory name.")
- scanner_id: str
- scanner_name: str
- scanner_type: str
- session_id: str
- team_id: int
- team_name: str
- snapshot: ScannerSnapshot
- recorded_output: dict[str, Any] = Field(description="scanner_result.model_output at collection time.")
- known_freeform_tags: list[str] = Field(
- default_factory=list,
- description="Tag vocabulary a freeform classifier scans with, captured at collection time.",
- )
- inactivity_periods: list[dict[str, Any]] = Field(
- default_factory=list,
- description="The rendered video's active/inactive map, for converting the model's video-time citations. "
- "Empty on cases collected before it was captured, which then score as if nothing was cut.",
- )
- navigation: list[dict[str, Any]] = Field(
- default_factory=list,
- description="The page and tab changes the rendered video shows, for the prompt's navigation timeline. "
- "Empty on cases collected before the renderer recorded them, which then scan without one.",
- )
- label_is_correct: bool | None = None
- label_feedback: str = ""
- collected_at: str
-
- def case_dir(self, root: Path) -> Path:
- return root / "cases" / self.case_id
-
- def video_path(self, root: Path) -> Path:
- return self.case_dir(root) / VIDEO_NAME
-
- def inputs_path(self, root: Path) -> Path:
- return self.case_dir(root) / INPUTS_NAME
-
- def load_inputs(self, root: Path) -> ScannerLlmInputs:
- return ScannerLlmInputs.model_validate_json(self.inputs_path(root).read_text())
-
-
-class GoldenDataset(BaseModel, frozen=True):
- created_at: str
- host: str
- project_id: int
- cases: list[GoldenCase] = Field(default_factory=list)
-
-
-def dataset_root() -> Path | None:
- raw = os.environ.get(DATASET_ENV_VAR, "").strip()
- return Path(raw).expanduser() if raw else None
-
-
-def load_dataset(root: Path) -> GoldenDataset:
- return GoldenDataset.model_validate_json((root / MANIFEST_NAME).read_text())
-
-
-def ensure_dataset_fresh(dataset: GoldenDataset, root: Path) -> None:
- """Refuse to scan a dataset whose consent verification has lapsed (see DATASET_MAX_AGE_DAYS)."""
- age = dt.datetime.now(dt.UTC) - parse_utc(dataset.created_at)
- if age > dt.timedelta(days=DATASET_MAX_AGE_DAYS):
- raise RuntimeError(
- f"Dataset at {root} was collected {age.days} days ago (limit {DATASET_MAX_AGE_DAYS}); "
- "re-run collect.py, which re-verifies AI data-processing consent"
- )
-
-
-def save_dataset(root: Path, dataset: GoldenDataset) -> None:
- root.mkdir(parents=True, exist_ok=True)
- (root / MANIFEST_NAME).write_text(dataset.model_dump_json(indent=2))
diff --git a/products/replay_vision/evals/eval_mcp_tools.py b/products/replay_vision/evals/eval_mcp_tools.py
deleted file mode 100644
index 88b2244884d8..000000000000
--- a/products/replay_vision/evals/eval_mcp_tools.py
+++ /dev/null
@@ -1,60 +0,0 @@
-"""Does a coding agent drive the Replay Vision MCP tools correctly?
-
-Each case asks for one multi-step job an agent is likely to be given: set up an alert with a
-destination, backfill history, and recover a failed scan.
-
-Run: hogli evals eval_replay_vision_mcp_tools
-"""
-
-from __future__ import annotations
-
-from products.posthog_ai.eval_harness.base import SandboxedPrivateEval
-from products.posthog_ai.eval_harness.config import SandboxedEvalCase
-from products.posthog_ai.eval_harness.harness.context import EvalContext
-from products.replay_vision.evals.mcp_scorers import (
- CreatedMatchAlertWithWebhook,
- EstimatedBeforeBackfill,
- RetriedFailedObservation,
-)
-from products.replay_vision.evals.mcp_seeders import (
- FAILED_SESSION_ID,
- SCANNER_NAME,
- seed_replay_vision_scanner,
- seed_replay_vision_scanner_with_recordings,
-)
-
-WEBHOOK_URL = "https://example.com/hooks/replay-vision"
-
-
-async def eval_replay_vision_mcp_tools(ctx: EvalContext) -> None:
- await SandboxedPrivateEval(
- experiment_name="sandboxed-replay-vision-mcp-tools-cli",
- cases=[
- SandboxedEvalCase(
- name="alert_with_webhook",
- prompt=(
- f'Send a webhook to {WEBHOOK_URL} every time the "{SCANNER_NAME}" Replay Vision scanner '
- "flags a session."
- ),
- setup=seed_replay_vision_scanner,
- expected={"created_match_alert_with_webhook": {"webhook_url": WEBHOOK_URL}},
- ),
- SandboxedEvalCase(
- name="backfill_last_week",
- prompt=f'Run the "{SCANNER_NAME}" Replay Vision scanner over the last 7 days of recordings.',
- setup=seed_replay_vision_scanner_with_recordings,
- expected={"estimated_before_backfill": {}},
- ),
- SandboxedEvalCase(
- name="retry_failed_scan",
- prompt=(
- f'The "{SCANNER_NAME}" scanner has no result for session {FAILED_SESSION_ID}. '
- "Find out why and get it scanned."
- ),
- setup=seed_replay_vision_scanner,
- expected={"retried_failed_observation": {}},
- ),
- ],
- scorers=[CreatedMatchAlertWithWebhook(), EstimatedBeforeBackfill(), RetriedFailedObservation()],
- ctx=ctx,
- )
diff --git a/products/replay_vision/evals/eval_scanner_quality.py b/products/replay_vision/evals/eval_scanner_quality.py
deleted file mode 100644
index af724072181a..000000000000
--- a/products/replay_vision/evals/eval_scanner_quality.py
+++ /dev/null
@@ -1,220 +0,0 @@
-"""Golden-dataset quality suite for Replay Vision scanner prompts.
-
-Each case re-runs the production scan pipeline (same Jinja templates, response schemas, events
-round, and citation handling, via run_scan) against a collected session video plus its event
-snapshot, then scores the fresh output against the recorded output and its human thumbs label.
-Edit the templates under backend/temporal/scanners/prompts/ and re-run to compare experiments
-in the local logs (this suite is private; nothing is sent to Braintrust).
-
-Requires REPLAY_VISION_EVAL_DATASET (a directory written by collect.py) and GEMINI_API_KEY.
-"""
-
-import time
-import asyncio
-from functools import partial
-from pathlib import Path
-from typing import Any
-
-import structlog
-from google.genai import (
- Client as RawGenAIClient,
- types as genai_types,
-)
-
-from posthog.temporal.session_replay.rasterize_recording.types import NavigationChange
-
-from products.posthog_ai.eval_harness.config import BaseEvalCase
-from products.posthog_ai.eval_harness.harness.context import EvalContext
-from products.posthog_ai.eval_harness.harness.requirements import SuiteKind
-from products.posthog_ai.eval_harness.one_shot import OneShotPrivateEval
-from products.replay_vision.backend.temporal.activities.call_scanner_provider import apply_known_freeform_tags, run_scan
-from products.replay_vision.backend.temporal.errors import ScannerFailureError
-from products.replay_vision.backend.temporal.gemini import gemini_api_key
-from products.replay_vision.backend.temporal.scanners import scanner_from_snapshot
-from products.replay_vision.backend.temporal.video_clock import VideoClock, video_clock_from_export_context
-from products.replay_vision.evals.dataset import (
- DATASET_ENV_VAR,
- GoldenCase,
- dataset_root,
- ensure_dataset_fresh,
- load_dataset,
-)
-from products.replay_vision.evals.outcomes import primary_outcome
-from products.replay_vision.evals.scorers import (
- SUMMARY_FIELDS,
- LabeledOutcome,
- OutputStability,
- ScanCompleted,
- ScoreAlignment,
- SummaryAlignment,
-)
-
-SUITE_KIND = SuiteKind.ONE_SHOT
-
-logger = structlog.get_logger(__name__)
-
-_MAX_PROCESSING_WAIT_SECONDS = 300
-
-
-def build_case(golden: GoldenCase) -> BaseEvalCase:
- """Map a golden case onto the scorer contract: which scorer applies, and what its reference is."""
- expected: dict[str, Any] = {}
- recorded_primary = primary_outcome(golden.recorded_output)
- if golden.scanner_type in ("monitor", "classifier"):
- if golden.label_is_correct is not None:
- expected["labeled_outcome"] = {
- "is_correct": golden.label_is_correct,
- "recorded_primary": recorded_primary,
- }
- else:
- expected["output_stability"] = {"recorded_primary": recorded_primary}
- elif golden.scanner_type == "scorer":
- recorded_score = golden.recorded_output.get("score")
- scale = golden.snapshot.scanner_config.get("scale") or {}
- scale_min, scale_max = scale.get("min"), scale.get("max")
- # Default the bounds as a pair: defaulting them independently can fabricate a zero-width
- # scale (e.g. min set to 1.0 with max defaulting to 1.0) that no scanner actually has.
- if scale_min is None and scale_max is None:
- scale_min, scale_max = 0.0, 1.0
- # A thumbs-downed recorded score is a known-bad reference, so those cases only get scan_completed.
- if (
- golden.label_is_correct is not False
- and isinstance(recorded_score, int | float)
- and scale_min is not None
- and scale_max is not None
- ):
- expected["score_alignment"] = {
- "recorded_score": recorded_score,
- "scale_min": scale_min,
- "scale_max": scale_max,
- }
- elif golden.scanner_type == "summarizer":
- if golden.label_is_correct is not False:
- expected["summary_alignment"] = {
- "reference": {field: golden.recorded_output.get(field) for field in SUMMARY_FIELDS}
- }
- return BaseEvalCase(
- # The full case id: UUIDv7 ids collected in the same minute share their first characters,
- # so any truncation makes same-type cases collide.
- name=f"{golden.scanner_type}-{golden.case_id}",
- prompt=str(golden.snapshot.scanner_config.get("prompt", "")),
- expected=expected,
- metadata={
- "case_id": golden.case_id,
- "scanner_name": golden.scanner_name,
- "scanner_type": golden.scanner_type,
- "session_id": golden.session_id,
- "label_is_correct": golden.label_is_correct,
- },
- )
-
-
-def _upload_video(client: RawGenAIClient, path: Path) -> genai_types.File:
- uploaded = client.files.upload(
- file=str(path),
- config=genai_types.UploadFileConfig(
- mime_type="video/mp4", display_name=f"replay-vision-eval-{path.parent.name}"
- ),
- )
- waited = 0.0
- while uploaded.state and uploaded.state.name == "PROCESSING":
- if waited >= _MAX_PROCESSING_WAIT_SECONDS:
- raise RuntimeError(f"Gemini file for {path} stuck in PROCESSING after {waited:.0f}s")
- time.sleep(0.5)
- waited += 0.5
- uploaded = client.files.get(name=uploaded.name or "")
- state = uploaded.state.name if uploaded.state else None
- if state != "ACTIVE" or not uploaded.uri:
- raise RuntimeError(f"Gemini upload for {path} ended in state {state!r}")
- return uploaded
-
-
-def _delete_file_quiet(client: RawGenAIClient, name: str | None) -> None:
- if not name:
- return
- try:
- client.files.delete(name=name)
- except Exception:
- logger.warning("replay_vision.eval.gemini_delete_failed", file=name)
-
-
-async def _scan_task(
- root: Path, golden_by_case_id: dict[str, GoldenCase], case: BaseEvalCase, ctx: EvalContext
-) -> dict[str, Any]:
- golden = golden_by_case_id[case.metadata["case_id"]]
- llm_inputs = await asyncio.to_thread(golden.load_inputs, root)
- # Rebuild the scanner the way production does, tag vocabulary included, so a freeform classifier
- # is scored on its prompt rather than on missing context.
- scanner = apply_known_freeform_tags(scanner_from_snapshot(golden.snapshot), golden.known_freeform_tags)
- client = RawGenAIClient(api_key=gemini_api_key())
- uploaded = await asyncio.to_thread(_upload_video, client, golden.video_path(root))
- try:
- try:
- result = await run_scan(
- snapshot=golden.snapshot,
- scanner=scanner,
- llm_inputs=llm_inputs,
- team_name=golden.team_name,
- file_uri=uploaded.uri or "",
- mime_type=uploaded.mime_type or "video/mp4",
- team_id=golden.team_id,
- # A case collected before the map was captured scores as if nothing was cut.
- video_clock=video_clock_from_export_context({"inactivity_periods": golden.inactivity_periods})
- or VideoClock(spans=()),
- navigation=[NavigationChange.model_validate(change) for change in golden.navigation],
- )
- except ScannerFailureError as exc:
- # A scan the model cannot complete is a prompt-quality signal (broken schema compliance),
- # not an infra error, so it must score 0 rather than be excluded from the aggregate.
- return {
- "exit_code": 1,
- "model_output": None,
- "error": str(exc),
- "scanner_type": golden.scanner_type,
- "last_message": f"scan failed: {exc}",
- }
- finally:
- await asyncio.to_thread(_delete_file_quiet, client, uploaded.name)
-
- model_output = result.model_output.model_dump(mode="json")
- primary = primary_outcome(model_output)
- return {
- "exit_code": 0,
- "model_output": model_output,
- "error": None,
- "scanner_type": golden.scanner_type,
- "signals_count": len(result.signals),
- "signals": [signal.model_dump(mode="json") for signal in result.signals],
- "primary": primary,
- "last_message": primary or "",
- }
-
-
-async def eval_scanner_quality(ctx: EvalContext) -> None:
- root = dataset_root()
- if root is None:
- # Skip rather than fail: the dataset is local-only and most environments (CI included) won't have one.
- logger.warning("replay_vision.eval.skipped_no_dataset", env_var=DATASET_ENV_VAR)
- return
- if not gemini_api_key():
- raise RuntimeError("Set GEMINI_API_KEY (or REPLAY_VISION_GEMINI_API_KEY) to run replay-vision scans")
-
- dataset = load_dataset(root)
- ensure_dataset_fresh(dataset, root)
- golden_cases = dataset.cases
- missing = [g.case_id for g in golden_cases if not (g.video_path(root).exists() and g.inputs_path(root).exists())]
- if missing:
- raise RuntimeError(f"Dataset at {root} is missing files for cases {missing[:5]}; re-run collect.py")
-
- cases = [build_case(golden) for golden in golden_cases]
- golden_by_case_id = {case.metadata["case_id"]: golden for case, golden in zip(cases, golden_cases)}
- # A collision here would silently score one observation against another session's video.
- if len(golden_by_case_id) != len(golden_cases):
- raise RuntimeError(f"Dataset at {root} contains duplicate case ids; re-collect into a clean directory")
- await OneShotPrivateEval(
- experiment_name="replay-vision-scanner-quality",
- cases=cases,
- scorers=[ScanCompleted(), LabeledOutcome(), OutputStability(), ScoreAlignment(), SummaryAlignment()],
- task=partial(_scan_task, root, golden_by_case_id),
- ctx=ctx,
- )
diff --git a/products/replay_vision/evals/mcp_scorers.py b/products/replay_vision/evals/mcp_scorers.py
deleted file mode 100644
index 233419e6316b..000000000000
--- a/products/replay_vision/evals/mcp_scorers.py
+++ /dev/null
@@ -1,103 +0,0 @@
-"""Deterministic scorers for the Replay Vision MCP tool-use suite."""
-
-from __future__ import annotations
-
-from datetime import datetime, timedelta
-from typing import Any
-
-from products.posthog_ai.eval_harness.log_parser import LogParser
-from products.posthog_ai.eval_harness.scorers.contract import Score, Scorer
-
-
-class _McpToolScorer(Scorer):
- """Skips cases that don't expect this scorer and fails runs with no log, then defers to `_score`."""
-
- def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs: Any) -> Score:
- if not expected or self._name() not in expected:
- return Score(name=self._name(), score=None, metadata={"reason": "Not expected for this case"})
- if not output or not output.get("raw_log"):
- return Score(name=self._name(), score=0.0, metadata={"reason": "No raw log"})
- parser = LogParser.cached(output["raw_log"], initial_prompt=output.get("prompt", "") or "")
- return self._score(parser, output.get("seed") or {}, expected[self._name()])
-
- def _score(self, parser: LogParser, seed: dict[str, Any], expected: dict[str, Any]) -> Score:
- raise NotImplementedError
-
-
-class CreatedMatchAlertWithWebhook(_McpToolScorer):
- """A match alert on the seeded scanner for verdict yes, then a webhook destination on that alert."""
-
- def _name(self) -> str:
- return "created_match_alert_with_webhook"
-
- def _score(self, parser: LogParser, seed: dict[str, Any], expected: dict[str, Any]) -> Score:
- alert_created = any(
- not c.is_error
- and c.input.get("scanner_id") == seed.get("scanner_id")
- and c.input.get("kind") == "match"
- and "yes" in ((c.input.get("selection") or {}).get("verdict") or [])
- for c in parser.get_tool_calls("vision-alerts-create")
- )
- destination_added = any(
- not c.is_error
- and c.input.get("type") == "webhook"
- and c.input.get("webhook_url") == expected["webhook_url"]
- for c in parser.get_tool_calls("vision-alerts-destinations-create")
- )
- return Score(
- name=self._name(),
- score=(int(alert_created) + int(destination_added)) / 2,
- metadata={"alert_created": alert_created, "destination_added": destination_added},
- )
-
-
-def _spans_about_a_week(call_input: dict[str, Any]) -> bool:
- try:
- start = datetime.fromisoformat(str(call_input["window_start"]).replace("Z", "+00:00"))
- end = datetime.fromisoformat(str(call_input["window_end"]).replace("Z", "+00:00"))
- except (KeyError, ValueError):
- return False
- # A day of slack either way, since the agent picks its own clock and rounding.
- return timedelta(days=6) <= end - start <= timedelta(days=8)
-
-
-class EstimatedBeforeBackfill(_McpToolScorer):
- """A successful estimate and no backfill create.
-
- Nobody in the sandbox can agree to the quoted cost, and the tool requires that agreement first, so the
- right move is to estimate and stop to ask.
- """
-
- def _name(self) -> str:
- return "estimated_before_backfill"
-
- def _score(self, parser: LogParser, seed: dict[str, Any], expected: dict[str, Any]) -> Score:
- estimates = [
- c
- for c in parser.get_tool_calls("vision-scanners-backfills-estimate")
- if not c.is_error and _spans_about_a_week(c.input)
- ]
- creates = [c.call_id for c in parser.get_tool_calls("vision-scanners-backfills-create")]
- if not estimates:
- return Score(name=self._name(), score=0.0, metadata={"reason": "No successful estimate over the last week"})
- return Score(
- name=self._name(),
- score=0.0 if creates else 1.0,
- metadata={"estimates": len(estimates), "unagreed_creates": creates},
- )
-
-
-class RetriedFailedObservation(_McpToolScorer):
- """The seeded failed observation was retried by its own id, not the session or scanner id."""
-
- def _name(self) -> str:
- return "retried_failed_observation"
-
- def _score(self, parser: LogParser, seed: dict[str, Any], expected: dict[str, Any]) -> Score:
- retries = parser.get_tool_calls("vision-observations-retry")
- hit = [c for c in retries if not c.is_error and str(c.input.get("id")) == seed.get("failed_observation_id")]
- return Score(
- name=self._name(),
- score=1.0 if hit else 0.0,
- metadata={"retries": len(retries), "succeeded_on_target": len(hit)},
- )
diff --git a/products/replay_vision/evals/mcp_seeders.py b/products/replay_vision/evals/mcp_seeders.py
deleted file mode 100644
index 16780d65a7fe..000000000000
--- a/products/replay_vision/evals/mcp_seeders.py
+++ /dev/null
@@ -1,68 +0,0 @@
-"""Seeders for the Replay Vision MCP tool-use suite."""
-
-from __future__ import annotations
-
-from datetime import timedelta
-from typing import Any
-
-from django.utils import timezone
-
-from posthog.models import Team
-from posthog.session_recordings.queries.test.session_replay_sql import produce_replay_summary
-
-from products.replay_vision.backend.models.replay_observation import (
- ObservationStatus,
- ObservationTrigger,
- ReplayObservation,
-)
-from products.replay_vision.backend.models.replay_scanner import ReplayScanner, ScannerModel, ScannerType
-from products.tasks.backend.facade.agents import CustomPromptSandboxContext
-
-SCANNER_NAME = "Checkout friction"
-FAILED_SESSION_ID = "0192f0e1-eval-4a5b-8c7d-checkoutfail01"
-
-
-def seed_replay_vision_scanner(context: CustomPromptSandboxContext) -> dict[str, Any]:
- """One paused monitor scanner with a transiently failed observation, and AI consent on."""
- team = Team.objects.select_related("organization").get(id=context.team_id)
- # The scanner, alert, backfill, and retry tools all refuse to run without AI consent.
- team.organization.is_ai_data_processing_approved = True
- team.organization.save(update_fields=["is_ai_data_processing_approved"])
-
- scanner = ReplayScanner.objects.create(
- team=team,
- name=SCANNER_NAME,
- scanner_type=ScannerType.MONITOR,
- scanner_config={"prompt": "Did the user struggle to complete checkout?"},
- model=ScannerModel.GEMINI_3_8_FLASH,
- # Paused so no scheduled sweep runs during the case.
- enabled=False,
- )
- observation = ReplayObservation.objects.create(
- team=team,
- scanner=scanner,
- session_id=FAILED_SESSION_ID,
- status=ObservationStatus.FAILED,
- error_reason="provider_transient:The model provider timed out.",
- triggered_by=ObservationTrigger.SCHEDULE,
- completed_at=timezone.now(),
- )
- return {"scanner_id": str(scanner.id), "failed_observation_id": str(observation.id)}
-
-
-def seed_replay_vision_scanner_with_recordings(context: CustomPromptSandboxContext) -> dict[str, Any]:
- """The same scanner plus recordings from the last week, so a backfill estimate finds sessions."""
- seed = seed_replay_vision_scanner(context)
- now = timezone.now()
- for day in range(1, 4):
- start = now - timedelta(days=day, hours=2)
- # Long and active enough to clear the video scanner's eligibility floor.
- produce_replay_summary(
- team_id=context.team_id,
- session_id=f"replay-vision-eval-backfill-{day}",
- distinct_id=f"replay-vision-eval-user-{day}",
- first_timestamp=start,
- last_timestamp=start + timedelta(minutes=5),
- active_milliseconds=120_000,
- )
- return seed
diff --git a/products/replay_vision/evals/outcomes.py b/products/replay_vision/evals/outcomes.py
deleted file mode 100644
index 8de8d34a18ff..000000000000
--- a/products/replay_vision/evals/outcomes.py
+++ /dev/null
@@ -1,42 +0,0 @@
-from typing import Any, Literal
-
-from products.replay_vision.backend.tags import slugify_tag
-
-Outcome = Literal["kept", "regressed", "fixed", "still_wrong"]
-
-# A summary can run long, so before/after previews cap it rather than showing the whole body.
-_SUMMARY_PREVIEW_CAP = 200
-
-
-def primary_outcome(model_output: dict[str, Any] | None) -> str | None:
- """The display string for before/after comparison: a discrete verdict/tags for monitors and classifiers,
- or the raw score/summary for the other types."""
- output = model_output or {}
- verdict = output.get("verdict")
- if isinstance(verdict, str) and verdict:
- return f"Verdict: {verdict.strip().lower()}"
- # Freeform tags are part of a classifier's output, so a change that only touches them must not read as
- # "no change". Matches `describe_output`, which merges both lists.
- raw_tags = [*(output.get("tags") or []), *(output.get("tags_freeform") or [])]
- tags = sorted({slug for t in raw_tags if isinstance(t, str) and (slug := slugify_tag(t))})
- if tags:
- return f"Tags: {', '.join(tags)}"
- score = output.get("score")
- if isinstance(score, int | float) and not isinstance(score, bool):
- return f"Score: {score}"
- title = output.get("title")
- if isinstance(title, str) and title.strip():
- return title.strip()
- summary = output.get("summary")
- if isinstance(summary, str) and summary.strip():
- trimmed = summary.strip()
- return trimmed if len(trimmed) <= _SUMMARY_PREVIEW_CAP else trimmed[:_SUMMARY_PREVIEW_CAP].rstrip() + "…"
- return None
-
-
-def classify_outcome(rated_correct: bool, before: str | None, after: str | None) -> Outcome:
- """A `None` outcome is valid (e.g. a classifier with no tags)."""
- changed = before != after
- if rated_correct:
- return "regressed" if changed else "kept"
- return "fixed" if changed else "still_wrong"
diff --git a/products/replay_vision/evals/scorers.py b/products/replay_vision/evals/scorers.py
deleted file mode 100644
index 38e4744fabed..000000000000
--- a/products/replay_vision/evals/scorers.py
+++ /dev/null
@@ -1,167 +0,0 @@
-"""Scorers for the replay-vision golden-dataset suite.
-
-Every scorer reads the task output dict built in eval_scanner_quality._scan_task plus its own
-sub-dict of ``case.expected``, and self-skips (score=None) when its key is absent, so one scorer
-list spans all four scanner types. Correctness scoring compares the primary outcome against the
-human label: kept/fixed score 1, regressed/still_wrong score 0.
-"""
-
-import json
-from typing import Any
-
-from products.posthog_ai.eval_harness.scorers import GRADED_ALIGNMENT_CHOICE_SCORES, JUDGE_MODEL, JudgedScorer
-from products.posthog_ai.eval_harness.scorers.contract import Score, Scorer
-from products.replay_vision.evals.outcomes import classify_outcome, primary_outcome
-
-_OUTCOME_SCORES = {"kept": 1.0, "fixed": 1.0, "regressed": 0.0, "still_wrong": 0.0}
-
-# The summarizer output fields the judge compares; build_case snapshots the same fields as the reference.
-SUMMARY_FIELDS = ("title", "summary")
-
-
-def _spec(expected: dict[str, Any] | None, name: str) -> dict[str, Any] | None:
- value = (expected or {}).get(name)
- return value if isinstance(value, dict) else None
-
-
-class ScanCompleted(Scorer):
- """Did the scan produce a validated output at all? A prompt change that breaks the response
- schema or the semantic validators fails here before any quality scorer gets a say."""
-
- def _name(self) -> str:
- return "scan_completed"
-
- def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs: Any) -> Score:
- if not output:
- return Score(name=self._name(), score=0.0, metadata={"reason": "no output"})
- if output.get("model_output"):
- return Score(name=self._name(), score=1.0)
- return Score(name=self._name(), score=0.0, metadata={"reason": output.get("error") or "no model_output"})
-
-
-class LabeledOutcome(Scorer):
- """For human-labeled monitor/classifier cases: kept and fixed score 1, regressed and
- still_wrong score 0."""
-
- def _name(self) -> str:
- return "labeled_outcome"
-
- def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs: Any) -> Score:
- spec = _spec(expected, self._name())
- if spec is None:
- return Score(name=self._name(), score=None, metadata={"reason": "not a labeled discrete case"})
- if not output or not output.get("model_output"):
- return Score(name=self._name(), score=0.0, metadata={"reason": "scan produced no output"})
- after = primary_outcome(output["model_output"])
- outcome = classify_outcome(bool(spec["is_correct"]), spec.get("recorded_primary"), after)
- return Score(
- name=self._name(),
- score=_OUTCOME_SCORES[outcome],
- metadata={"outcome": outcome, "before": spec.get("recorded_primary"), "after": after},
- )
-
-
-class OutputStability(Scorer):
- """For unlabeled monitor/classifier cases there is no ground truth; score whether the fresh
- outcome matches the recorded baseline. This measures churn introduced by a prompt change,
- not correctness, so it is reported separately from labeled_outcome."""
-
- def _name(self) -> str:
- return "output_stability"
-
- def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs: Any) -> Score:
- spec = _spec(expected, self._name())
- if spec is None:
- return Score(name=self._name(), score=None, metadata={"reason": "not an unlabeled discrete case"})
- if not output or not output.get("model_output"):
- return Score(name=self._name(), score=0.0, metadata={"reason": "scan produced no output"})
- after = primary_outcome(output["model_output"])
- stable = after == spec.get("recorded_primary")
- return Score(
- name=self._name(),
- score=1.0 if stable else 0.0,
- metadata={"before": spec.get("recorded_primary"), "after": after},
- )
-
-
-class ScoreAlignment(Scorer):
- """For scorer scanners: distance from the reference score, normalized by the configured scale.
-
- The reference is the recorded production score; cases whose recorded run was thumbs-downed
- carry no spec (a known-bad reference would reward reproducing the mistake)."""
-
- def _name(self) -> str:
- return "score_alignment"
-
- def _run_eval_sync(self, output: dict | None, expected: dict | None = None, **kwargs: Any) -> Score:
- spec = _spec(expected, self._name())
- if spec is None:
- return Score(name=self._name(), score=None, metadata={"reason": "not a scorer case with a reference"})
- span = float(spec["scale_max"]) - float(spec["scale_min"])
- if span <= 0:
- # A degenerate scale makes distance meaningless: the scorer is inapplicable, not failing.
- return Score(name=self._name(), score=None, metadata={"reason": "zero-width scale"})
- model_output = (output or {}).get("model_output") or {}
- fresh = model_output.get("score")
- if not isinstance(fresh, int | float) or isinstance(fresh, bool):
- return Score(name=self._name(), score=0.0, metadata={"reason": "scan produced no numeric score"})
- alignment = max(0.0, 1.0 - abs(float(fresh) - float(spec["recorded_score"])) / span)
- return Score(
- name=self._name(),
- score=alignment,
- metadata={"recorded_score": spec["recorded_score"], "fresh_score": fresh},
- )
-
-
-_SUMMARY_JUDGE_PROMPT = """
-You are comparing two AI-generated analyses of the same session recording of a user using a product.
-
-The reference analysis was produced earlier and is trusted:
-{{expected}}
-
-The candidate analysis was just produced by a new version of the analyzer:
-{{output}}
-
-Grade how well the candidate captures the same user journey as the reference: the user's intent,
-what happened, the outcome, and any friction encountered. Wording differences are fine; missing or
-invented events, a different outcome, or lost friction points are not.
-
-Answer by selecting exactly one option:
-- perfect: same journey, intent, outcome, and friction; nothing meaningful lost or invented
-- near_perfect: same story with trivial omissions or additions
-- slightly_off: mostly the same story, but one meaningful detail is missing or invented
-- somewhat_misaligned: the story partially matches, but intent, outcome, or major friction diverges
-- strongly_misaligned: the candidate describes a substantially different session
-- useless: the candidate is empty, incoherent, or unrelated
-"""
-
-
-class SummaryAlignment(JudgedScorer):
- """LLM judge for summarizer cases: does the fresh summary tell the same story as the reference?
-
- The reference is the recorded output; thumbs-downed references are excluded at case-build time."""
-
- def __init__(self, **kwargs: Any) -> None:
- super().__init__(
- name="summary_alignment",
- prompt_template=_SUMMARY_JUDGE_PROMPT,
- choice_scores=GRADED_ALIGNMENT_CHOICE_SCORES,
- model=JUDGE_MODEL,
- max_completion_tokens=512,
- **kwargs,
- )
-
- def _prepare(self, output: dict[str, Any] | None, expected: dict[str, Any] | None) -> dict[str, Any] | Score:
- spec = _spec(expected, "summary_alignment")
- if spec is None:
- return Score(
- name=self._name(), score=None, metadata={"reason": "not a summarizer case with a trusted reference"}
- )
- model_output = (output or {}).get("model_output")
- if not model_output:
- return Score(name=self._name(), score=0.0, metadata={"reason": "scan produced no output"})
- candidate = {field: model_output.get(field) for field in SUMMARY_FIELDS}
- return {
- "output": json.dumps(candidate, indent=2),
- "expected": json.dumps(spec.get("reference"), indent=2),
- }
diff --git a/products/replay_vision/evals/test/__init__.py b/products/replay_vision/evals/test/__init__.py
deleted file mode 100644
index e69de29bb2d1..000000000000
diff --git a/products/replay_vision/evals/test/test_eval_scanner_quality.py b/products/replay_vision/evals/test/test_eval_scanner_quality.py
deleted file mode 100644
index 1b70620bf933..000000000000
--- a/products/replay_vision/evals/test/test_eval_scanner_quality.py
+++ /dev/null
@@ -1,498 +0,0 @@
-from __future__ import annotations
-
-import json
-import time
-import random
-import asyncio
-import datetime as dt
-from pathlib import Path
-from typing import Any
-
-import pytest
-from unittest.mock import AsyncMock, MagicMock, patch
-
-from google.genai import types as genai_types
-
-from products.posthog_ai.eval_harness.harness.context import EvalContext
-from products.posthog_ai.eval_harness.scorers.contract import Score, Scorer
-from products.replay_vision.backend.temporal.activities.call_scanner_provider import apply_known_freeform_tags
-from products.replay_vision.backend.temporal.scanners import ClassifierScanner
-from products.replay_vision.backend.temporal.scanners.base import SignalFinding
-from products.replay_vision.backend.temporal.scanners.monitor import MonitorOutput
-from products.replay_vision.backend.temporal.types import ScannerCallOutput, ScannerLlmInputs, ScannerSnapshot
-from products.replay_vision.evals import collector, eval_scanner_quality
-from products.replay_vision.evals.collector import (
- _VideoAsset,
- asset_is_recorded_video,
- build_llm_inputs,
- order_candidates,
-)
-from products.replay_vision.evals.dataset import GoldenCase, GoldenDataset, ensure_dataset_fresh, parse_utc
-from products.replay_vision.evals.eval_scanner_quality import build_case
-from products.replay_vision.evals.scorers import (
- LabeledOutcome,
- OutputStability,
- ScanCompleted,
- ScoreAlignment,
- SummaryAlignment,
-)
-
-
-def _eval(scorer: Scorer, output: dict[str, Any] | None, expected: dict[str, Any] | None) -> Score:
- # eval_async is the entrypoint the harness dispatches through.
- score = asyncio.run(scorer.eval_async(output, expected))
- assert isinstance(score, Score)
- return score
-
-
-def _monitor_output(verdict: str) -> dict[str, Any]:
- return {"model_output": {"verdict": verdict, "reasoning": "r", "confidence": 0.9}}
-
-
-def _golden(
- scanner_type: str,
- label_is_correct: bool | None,
- recorded_output: dict[str, Any],
- case_id: str = "0199aaaa-0000-7000-8000-000000000001",
- scale: dict[str, Any] | None = None,
-) -> GoldenCase:
- scanner_config: dict[str, Any] = {"prompt": "watch for rage clicks"}
- if scanner_type == "classifier":
- scanner_config["tags"] = ["a", "b"]
- if scanner_type == "scorer":
- scanner_config["scale"] = {"min": 1, "max": 5} if scale is None else scale
- return GoldenCase(
- case_id=case_id,
- scanner_id="s1",
- scanner_name="Test scanner",
- scanner_type=scanner_type,
- session_id="sess-1",
- team_id=2,
- team_name="Team",
- snapshot=ScannerSnapshot(
- name="Test scanner",
- scanner_type=scanner_type,
- scanner_version=1,
- model="gemini-3.6-flash",
- provider="google",
- emits_signals=False,
- scanner_config=scanner_config,
- ),
- recorded_output=recorded_output,
- label_is_correct=label_is_correct,
- collected_at="2026-07-30T00:00:00+00:00",
- )
-
-
- "is_correct,fresh_verdict,expected_score,expected_outcome",
- [
- (True, "yes", 1.0, "kept"),
- (True, "no", 0.0, "regressed"),
- (False, "no", 1.0, "fixed"),
- (False, "yes", 0.0, "still_wrong"),
- ],
-)
-def test_labeled_outcome_scores_thumbs_semantics(
- is_correct: bool, fresh_verdict: str, expected_score: float, expected_outcome: str
-) -> None:
- expected = {"labeled_outcome": {"is_correct": is_correct, "recorded_primary": "Verdict: yes"}}
- score = _eval(LabeledOutcome(), _monitor_output(fresh_verdict), expected)
- assert score.score == expected_score
- assert score.metadata["outcome"] == expected_outcome
-
-
- "scorer",
- [LabeledOutcome(), OutputStability(), ScoreAlignment()],
- ids=lambda s: s._name(),
-)
-def test_inapplicable_cases_skip_instead_of_failing(scorer: Any) -> None:
- # None means "skipped" in the aggregate; returning 0.0 here would silently drag every
- # experiment's mean down for cases the scorer was never meant to grade.
- score = _eval(scorer, _monitor_output("yes"), {})
- assert score.score is None
-
-
-def test_labeled_outcome_fails_when_scan_produced_nothing() -> None:
- expected = {"labeled_outcome": {"is_correct": True, "recorded_primary": "Verdict: yes"}}
- score = _eval(LabeledOutcome(), {"model_output": None, "error": "boom"}, expected)
- assert score.score == 0.0
-
-
- "fresh,expected_score",
- [
- (3.0, 1.0),
- (4.0, 0.75),
- (1.0, 0.5),
- (None, 0.0),
- ],
-)
-def test_score_alignment_normalizes_distance_by_scale(fresh: float | None, expected_score: float) -> None:
- expected = {"score_alignment": {"recorded_score": 3.0, "scale_min": 1.0, "scale_max": 5.0}}
- output = {"model_output": {"score": fresh} if fresh is not None else None}
- score = _eval(ScoreAlignment(), output, expected)
- assert score.score == pytest.approx(expected_score)
-
-
-def test_score_alignment_skips_zero_width_scale() -> None:
- # min == max used to divide by zero and kill the case; a degenerate scale is inapplicable, not failing.
- expected = {"score_alignment": {"recorded_score": 3.0, "scale_min": 3.0, "scale_max": 3.0}}
- score = _eval(ScoreAlignment(), {"model_output": {"score": 3.0}}, expected)
- assert score.score is None
- assert score.metadata["reason"] == "zero-width scale"
-
-
- "scale,expected_bounds",
- [
- ({}, (0.0, 1.0)),
- ({"min": 1, "max": 5}, (1, 5)),
- ({"min": 1}, None),
- ({"max": 5}, None),
- ],
-)
-def test_build_case_only_defaults_scale_bounds_as_a_pair(
- scale: dict[str, Any], expected_bounds: tuple[float, float] | None
-) -> None:
- case = build_case(_golden("scorer", None, {"score": 3.0}, scale=scale))
- spec = case.expected.get("score_alignment")
- if expected_bounds is None:
- assert spec is None
- else:
- assert spec is not None
- assert (spec["scale_min"], spec["scale_max"]) == expected_bounds
-
-
-def test_output_stability_compares_primary_outcomes() -> None:
- expected = {"output_stability": {"recorded_primary": "Verdict: yes"}}
- assert _eval(OutputStability(), _monitor_output("yes"), expected).score == 1.0
- assert _eval(OutputStability(), _monitor_output("no"), expected).score == 0.0
-
-
-def test_scan_completed_fails_on_schema_breakage() -> None:
- assert _eval(ScanCompleted(), _monitor_output("yes"), {}).score == 1.0
- failed = _eval(ScanCompleted(), {"model_output": None, "error": "required step rejected"}, {})
- assert failed.score == 0.0
- assert "rejected" in failed.metadata["reason"]
-
-
- "signals",
- [
- [],
- [
- {
- "problem_type": "design_flaw",
- "headline": "Navigation panel covers the book list",
- "start_time": 7,
- "end_time": 11,
- "url": "https://example.com/library",
- "description": "The navigation panel overlaps the book list.",
- "confidence": 0.8,
- },
- {
- "problem_type": "crash",
- "headline": "Reader errors out after a book opens",
- "start_time": 23,
- "end_time": 29,
- "url": "https://example.com/reader",
- "description": "The reader shows an error screen after the book opens.",
- "confidence": 0.95,
- },
- ],
- ],
- ids=["empty", "multiple_findings"],
-)
-def test_scan_task_retains_structured_signals(signals: list[dict[str, str | int | float]], tmp_path: Path) -> None:
- golden = _golden("monitor", True, {"verdict": "yes"})
- golden = golden.model_copy(update={"snapshot": golden.snapshot.model_copy(update={"emits_signals": True})})
- scan_output = ScannerCallOutput(
- model_output=MonitorOutput(verdict="yes", reasoning="The book list is covered.", confidence=0.9),
- signals=[SignalFinding.model_validate(signal) for signal in signals],
- )
- with (
- patch.object(GoldenCase, "load_inputs", return_value=MagicMock(spec=ScannerLlmInputs)),
- patch.object(eval_scanner_quality, "gemini_api_key", return_value="fake-eval-api-key"),
- patch.object(eval_scanner_quality, "RawGenAIClient"),
- patch.object(
- eval_scanner_quality,
- "_upload_video",
- return_value=genai_types.File(
- name="files/eval-video", uri="https://example.com/video.mp4", mime_type="video/mp4"
- ),
- ),
- patch.object(eval_scanner_quality, "_delete_file_quiet"),
- patch.object(eval_scanner_quality, "run_scan", new=AsyncMock(return_value=scan_output)),
- ):
- output = asyncio.run(
- eval_scanner_quality._scan_task(
- tmp_path, {golden.case_id: golden}, build_case(golden), MagicMock(spec=EvalContext)
- )
- )
-
- assert json.loads(json.dumps(output)) == {
- "exit_code": 0,
- "model_output": scan_output.model_output.model_dump(mode="json"),
- "error": None,
- "scanner_type": "monitor",
- "signals_count": len(signals),
- "signals": signals,
- "primary": "Verdict: yes",
- "last_message": "Verdict: yes",
- }
-
-
-def test_summary_alignment_prepare_gates_on_reference() -> None:
- # _prepare directly: going through eval_async would call the LLM judge.
- judge = SummaryAlignment()
- skipped = judge._prepare(_monitor_output("yes"), {})
- assert isinstance(skipped, Score)
- assert skipped.score is None
- spec = {"summary_alignment": {"reference": {"title": "t", "summary": "s"}}}
- no_output = judge._prepare({"model_output": None}, spec)
- assert isinstance(no_output, Score)
- assert no_output.score == 0.0
- prepared = judge._prepare({"model_output": {"title": "t2", "summary": "s2"}}, spec)
- assert isinstance(prepared, dict)
- assert "t2" in prepared["output"]
- assert "t" in prepared["expected"]
-
-
- "scanner_type,label_is_correct,recorded_output,expected_key",
- [
- ("monitor", True, {"verdict": "yes"}, "labeled_outcome"),
- ("monitor", None, {"verdict": "yes"}, "output_stability"),
- ("classifier", False, {"tags": ["a"]}, "labeled_outcome"),
- ("scorer", None, {"score": 3.0}, "score_alignment"),
- ("scorer", True, {"score": 3.0}, "score_alignment"),
- ("summarizer", None, {"title": "t", "summary": "s"}, "summary_alignment"),
- ],
-)
-def test_build_case_routes_to_the_right_scorer(
- scanner_type: str, label_is_correct: bool | None, recorded_output: dict[str, Any], expected_key: str
-) -> None:
- case = build_case(_golden(scanner_type, label_is_correct, recorded_output))
- assert list(case.expected.keys()) == [expected_key]
-
-
- "scanner_type,recorded_output",
- [
- ("scorer", {"score": 3.0}),
- ("summarizer", {"title": "t", "summary": "s"}),
- ],
-)
-def test_build_case_never_trusts_a_thumbs_downed_reference(scanner_type: str, recorded_output: dict[str, Any]) -> None:
- case = build_case(_golden(scanner_type, False, recorded_output))
- assert case.expected == {}
-
-
-def test_case_names_do_not_collide_on_shared_uuid_prefix() -> None:
- # UUIDv7 ids minted in the same minute share their leading characters; a truncated name
- # collapsed such cases into one dataset entry, scoring one against another session's video.
- goldens = [
- _golden("monitor", True, {"verdict": "yes"}, case_id="0199aaaa-0000-7000-8000-000000000001"),
- _golden("monitor", True, {"verdict": "yes"}, case_id="0199aaaa-0000-7000-8000-000000000002"),
- ]
- cases = [build_case(golden) for golden in goldens]
- assert cases[0].name != cases[1].name
- golden_by_case_id = {case.metadata["case_id"]: golden for case, golden in zip(cases, goldens)}
- assert len(golden_by_case_id) == len(goldens)
-
-
-def test_order_candidates_puts_labeled_first_per_type() -> None:
- def candidate(scanner_type: str, obs_id: str, labeled: bool) -> dict[str, Any]:
- observation: dict[str, Any] = {"id": obs_id, "label": {"is_correct": True} if labeled else None}
- return {"observation": observation, "scanner": {}, "scanner_type": scanner_type}
-
- candidates = [
- candidate("monitor", "u1", False),
- candidate("monitor", "l1", True),
- candidate("monitor", "u2", False),
- candidate("scorer", "u3", False),
- candidate("monitor", "l2", True),
- ]
- ordered = order_candidates(candidates, random.Random(42))
- monitor_ids = [c["observation"]["id"] for c in ordered["monitor"]]
- assert monitor_ids[:2] == ["l1", "l2"]
- assert sorted(monitor_ids[2:]) == ["u1", "u2"]
- assert [c["observation"]["id"] for c in ordered["scorer"]] == ["u3"]
- assert ordered == order_candidates(candidates, random.Random(42))
-
-
-def test_parse_utc_rejects_naive_timestamps() -> None:
- with pytest.raises(ValueError, match="no timezone"):
- parse_utc("2026-08-01T12:00:00")
- assert parse_utc("2026-08-01T12:00:00+12:00") == dt.datetime(2026, 8, 1, 0, 0, 0, tzinfo=dt.UTC)
-
-
[email protected]("age_days,fresh", [(0, True), (29, True), (31, False)])
-def test_ensure_dataset_fresh_enforces_consent_ttl(age_days: int, fresh: bool) -> None:
- created = (dt.datetime.now(dt.UTC) - dt.timedelta(days=age_days)).isoformat()
- dataset = GoldenDataset(created_at=created, host="https://us.posthog.com", project_id=2, cases=[])
- if fresh:
- ensure_dataset_fresh(dataset, Path("/tmp/dataset"))
- else:
- with pytest.raises(RuntimeError, match="re-run collect.py"):
- ensure_dataset_fresh(dataset, Path("/tmp/dataset"))
-
-
-def test_apply_known_freeform_tags_only_touches_freeform_classifiers() -> None:
- freeform = ClassifierScanner(prompt="x", tags=["a"], allow_freeform_tags=True)
- tagged = apply_known_freeform_tags(freeform, ["search_error"])
- assert isinstance(tagged, ClassifierScanner)
- assert tagged.known_freeform_tags == ["search_error"]
- # A fixed-vocab classifier rejects freeform output, so injecting tags there would break its validation.
- fixed = ClassifierScanner(prompt="x", tags=["a"], allow_freeform_tags=False)
- assert apply_known_freeform_tags(fixed, ["search_error"]) is fixed
- assert apply_known_freeform_tags(freeform, []) is freeform
-
-
-def test_asset_is_recorded_video_rejects_rerasterized_assets() -> None:
- asset = _VideoAsset(asset_id=1, created_at=dt.datetime(2026, 8, 1, tzinfo=dt.UTC))
- assert asset_is_recorded_video(asset, {"completed_at": "2026-08-01T00:00:01+00:00"})
- assert not asset_is_recorded_video(asset, {"completed_at": "2026-07-31T23:59:59+00:00"})
- assert not asset_is_recorded_video(asset, {"completed_at": None})
-
-
-def test_known_freeform_tags_ranks_recent_observations_only() -> None:
- now = dt.datetime.now(dt.UTC)
-
- def observation(days_ago: int, model_output: dict[str, Any] | None) -> dict[str, Any]:
- return {
- "created_at": (now - dt.timedelta(days=days_ago)).isoformat(),
- "scanner_result": {"model_output": model_output},
- }
-
- observations = [
- observation(1, {"tags_freeform": ["Search Error", "slow page"]}),
- observation(2, {"tags_freeform": ["search_error"]}),
- observation(60, {"tags_freeform": ["ancient_tag"]}),
- observation(0, None),
- ]
- assert collector._known_freeform_tags(observations) == ["search_error", "slow_page"]
-
-
-_SESSION_START = "2026-08-01T12:00:00+12:00"
-_SESSION_END = "2026-08-01T12:05:00+12:00"
-
-
-class _FakeApi:
- project_id = 2
-
- def hogql(self, query: str, values: dict[str, Any] | None = None) -> list[list[Any]]:
- if "session_replay_events" in query:
- return [["d1", _SESSION_START, _SESSION_END, 3, 4, 5, 60_000, 0, "https://example.com/a"]]
- # Field order: DEFAULT_EVENT_FIELDS + _EXTRA_FIELDS.
- return [
- [
- "$pageview",
- "2026-08-01T12:00:10+12:00",
- "",
- [],
- [],
- "w1",
- "https://example.com/a",
- None,
- "00000000-0000-0000-0000-00000000000a",
- [],
- [],
- [],
- None,
- ],
- [
- "$autocapture",
- "2026-08-01T12:01:10+12:00",
- "",
- [],
- [],
- "w1",
- "https://example.com/b",
- "click",
- "00000000-0000-0000-0000-00000000000b",
- [],
- [],
- [],
- None,
- ],
- [
- "file_uploaded",
- "2026-08-01T12:02:10+12:00",
- "",
- [],
- [],
- "w1",
- "https://example.com/b",
- None,
- "00000000-0000-0000-0000-00000000000c",
- [],
- [],
- [],
- None,
- ],
- ]
-
-
-def test_event_offsets_do_not_depend_on_local_timezone(monkeypatch: pytest.MonkeyPatch) -> None:
- def offsets_under(tz: str) -> dict[str, int]:
- monkeypatch.setenv("TZ", tz)
- time.tzset()
- inputs = build_llm_inputs(_FakeApi(), 2, "sess-1") # type: ignore[arg-type]
- assert inputs is not None
- return inputs.event_timestamps
-
- try:
- utc_offsets = offsets_under("UTC")
- nz_offsets = offsets_under("Pacific/Auckland")
- finally:
- monkeypatch.undo()
- time.tzset()
- assert utc_offsets == nz_offsets
- assert utc_offsets["00000000-0000-0000-0000-00000000000a"] == 10_000
- assert utc_offsets["00000000-0000-0000-0000-00000000000b"] == 70_000
-
-
-class _DefinitionsApi(_FakeApi):
- def __init__(self) -> None:
- self.definition_requests = 0
-
- def paginate(self, path: str, params: dict[str, Any] | None = None, max_items: int | None = None) -> Any:
- self.definition_requests += 1
- yield {"name": "file_uploaded", "description": "User uploaded a file"}
-
-
-def test_build_llm_inputs_carries_prompt_context() -> None:
- api = _DefinitionsApi()
- lookup = collector.EventDescriptionLookup(api) # type: ignore[arg-type]
- for _ in range(2):
- inputs = build_llm_inputs(api, 2, "sess-1", product_context="Acme sells anvils", event_descriptions=lookup) # type: ignore[arg-type]
- assert inputs is not None
- assert inputs.product_context == "Acme sells anvils"
- assert inputs.event_descriptions == {"file_uploaded": "User uploaded a file"}
- # The description cache spans sessions, so the second build must not refetch.
- assert api.definition_requests == 1
-
-
-class _PagedApi:
- def __init__(self, pages: list[list[list[Any]]]) -> None:
- self._pages = pages
- self._served = 0
-
- def hogql(self, query: str, values: dict[str, Any] | None = None) -> list[list[Any]]:
- page = self._pages[self._served] if self._served < len(self._pages) else []
- self._served += 1
- return page
-
-
-def test_fetch_events_caps_total_rows(monkeypatch: pytest.MonkeyPatch) -> None:
- monkeypatch.setattr(collector, "_EVENTS_PER_PAGE", 2)
- monkeypatch.setattr(collector, "_MAX_TOTAL_EVENT_ROWS", 5)
- row = ["$pageview", _SESSION_START, "u"]
- api = _PagedApi([[row, row], [row, row], [row, row], [row, row]])
- start = dt.datetime(2026, 8, 1, 0, 0, 0, tzinfo=dt.UTC)
- fetched = collector._fetch_events(api, "sess-1", start, start + dt.timedelta(minutes=5)) # type: ignore[arg-type]
- assert len(fetched.rows) == 5
- assert fetched.truncated
diff --git a/products/replay_vision/package.json b/products/replay_vision/package.json
index f4c1ccfe19d2..e235e1aaa9b6 100644
--- a/products/replay_vision/package.json
+++ b/products/replay_vision/package.json
@@ -1,7 +1,7 @@
{
"name": "@posthog/products-replay-vision",
"scripts": {
- "backend:test": "pytest -c ../../pytest.ini --rootdir ../.. backend/tests evals/test/ -v --tb=short"
+ "backend:test": "pytest -c ../../pytest.ini --rootdir ../.. backend/tests -v --tb=short"
},
"dependencies": {
"@posthog/quill": "workspace:*",