Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(evals): verify branch skills and native chart results (#122701)

Daniel Parker6723a445ca71
diff --git a/docs/published/handbook/engineering/ai/sandboxed-agents.md b/docs/published/handbook/engineering/ai/sandboxed-agents.md
index bd72346c422c..807e0e57b3eb 100644
--- a/docs/published/handbook/engineering/ai/sandboxed-agents.md
+++ b/docs/published/handbook/engineering/ai/sandboxed-agents.md
@@ -304,6 +304,10 @@ They're copied to three discovery locations during image build:
- `~/.agents/skills/` – Codex agent discovery
- `~/.claude/skills/` – Claude Code CLI discovery
+Evals build skills from the local checkout into a new output directory, then capture a snapshot. Model comparisons share one saved snapshot across their model runs, so rendered examples cannot change between models. `--skill-snapshot-report <results.json>` reuses a previous run's archived snapshot only if its source fingerprint and commit match the checkout. The fingerprint includes staged and unstaged changes plus untracked source files, so schema edits outside the skill folders also invalidate reuse. It excludes configured result files and comparison output directories, including outputs inside the checkout. Docker mounts the snapshot at all three discovery paths, replacing image-only skills as well. Modal builds use the same snapshot. Before the agent starts, the harness checks every skill file against a SHA-256 manifest in the agent's task filesystem and stops on missing, changed, or extra files. MCP delivery pins the archive hash and clears native bundled skills. Verification reads the delivery mode inside the running sandbox, including when a worker retrieves it by ID. Desktop skill-source preferences and shell-level MCP archive URLs cannot replace the eval snapshot.
+
+Run reports and PostHog eval metadata record the skill snapshot hash alongside the commit and dirty-worktree flag. The local run report includes the full file manifest's path. Model comparisons reject missing or different snapshot hashes. These checks confirm which files the agent can read; separate tool-call checks confirm whether it reads a skill.
+
For details on writing skills, see [Writing skills](/handbook/engineering/ai/writing-skills).
## Multi-turn sessions
diff --git a/products/posthog_ai/backend/services/system_prompt/service.py b/products/posthog_ai/backend/services/system_prompt/service.py
index 5ce5cae52e83..d79eb5764070 100644
--- a/products/posthog_ai/backend/services/system_prompt/service.py
+++ b/products/posthog_ai/backend/services/system_prompt/service.py
@@ -39,7 +39,8 @@ class ClaudeCodeSystemPrompt(TypedDict):
class PromptService(BaseSandboxService):
"""Provide the systemPrompt suffix for a PostHog AI sandbox Run."""
- def build(self) -> ClaudeCodeSystemPrompt:
+ @staticmethod
+ def build() -> ClaudeCodeSystemPrompt:
"""Return the PostHog AI systemPrompt as a Claude Code preset-plus-append suffix.
Called once at Run creation. The returned object goes into
diff --git a/products/posthog_ai/eval_harness/base.py b/products/posthog_ai/eval_harness/base.py
index 922aa5e489f4..93ca9a9518a9 100644
--- a/products/posthog_ai/eval_harness/base.py
+++ b/products/posthog_ai/eval_harness/base.py
@@ -4,6 +4,7 @@ import os
import time
import uuid
import asyncio
+import hashlib
import logging
import traceback
from collections.abc import Callable, Sequence
@@ -27,6 +28,7 @@ from psycopg import OperationalError as PsycopgOperationalError
from posthog.dataclasses import frozen
from posthog.exceptions import ClickHouseAtCapacity, ClickHouseQueryMemoryLimitExceeded, ClickHouseQueryTimeOut
+from products.posthog_ai.backend.services.system_prompt.service import PromptService
from products.tasks.backend.facade.agents import EVAL_INTERACTION_ORIGIN, CustomPromptSandboxContext, TurnPollTimeout
from .acp_log import ParsedLog, parse_log
@@ -248,12 +250,17 @@ class _BaseEvalRun:
def _experiment_metadata(self) -> RunMetadata:
"""The run's configuration, sent to the engine and stamped on every PostHog event."""
- return {
+ metadata: RunMetadata = {
"agent_model": self.ctx.agent_model,
"trials": self.ctx.trials,
"git_sha": self.ctx.git_sha,
"git_dirty": self.ctx.git_dirty,
}
+ if self.ctx.skills_sha256 is not None:
+ metadata["skills_sha256"] = self.ctx.skills_sha256
+ if self.ctx.skills_source_sha256 is not None:
+ metadata["skills_source_sha256"] = self.ctx.skills_source_sha256
+ return metadata
async def _task(self, input: dict[str, Any], hooks: CaseHooks) -> dict[str, Any] | None:
case_started = time.monotonic()
@@ -678,6 +685,20 @@ class _SandboxedEvalRun(_BaseEvalRun):
sandbox_context = replace(sandbox_context, interaction_origin=original_case.interaction_origin)
elif self.ctx.skill_delivery == "exec":
sandbox_context = replace(sandbox_context, interaction_origin=EVAL_INTERACTION_ORIGIN)
+ if sandbox_context.interaction_origin == "posthog_ai":
+ prompt = PromptService.build()
+ sandbox_context = replace(
+ sandbox_context,
+ system_prompt={
+ "type": prompt["type"],
+ "preset": prompt["preset"],
+ "append": prompt["append"],
+ },
+ )
+ if hooks is not None:
+ hooks.metadata["system_prompt_sha256"] = hashlib.sha256(
+ orjson.dumps(sandbox_context.system_prompt, option=orjson.OPT_SORT_KEYS)
+ ).hexdigest()
seed = (
await _run_setup_step(partial(original_case.setup, sandbox_context))
if original_case is not None and original_case.setup is not None
diff --git a/products/posthog_ai/eval_harness/harness/README.md b/products/posthog_ai/eval_harness/harness/README.md
index 4f1015d3f50c..16c82abd2d93 100644
--- a/products/posthog_ai/eval_harness/harness/README.md
+++ b/products/posthog_ai/eval_harness/harness/README.md
@@ -77,6 +77,10 @@ The final summary renders after teardown, logging is flushed, and the transcript
The comparison checks every configuration first with `--check`, then forwards child output into its combined transcript.
`--results-file` writes a validated run report using the reporter's existing scores, status counts, and case selection.
The comparison retains these reports after each model and checks that case and scorer selections match.
+It reuses the first model's archived branch-skill snapshot for later models and checks that their snapshot hashes match.
+Saved snapshots include a source fingerprint. Reuse stops if the branch's skill sources or commit changed.
+With `--keep-sandbox-containers`, mounted skill files remain under `logs/skill-snapshots/` after the run so preserved containers can still read them.
+Cases with `interaction_origin="posthog_ai"` also receive the production PostHog AI system prompt from this checkout before the task starts. The interaction origin alone does not supply that prompt. Compare skill versions only with the same client prompt and scorer versions.
No prompts, model answers, or credentials are included in the comparison report.
The reporter emits plain-text records with stable labels so terminals and agents can scan the same output.
diff --git a/products/posthog_ai/eval_harness/harness/cli.py b/products/posthog_ai/eval_harness/harness/cli.py
index 4348a3e65e3b..423dc0c29c27 100644
--- a/products/posthog_ai/eval_harness/harness/cli.py
+++ b/products/posthog_ai/eval_harness/harness/cli.py
@@ -79,6 +79,8 @@ class HarnessOptions:
comparison_dir: Path | None = None
results_file: Path | None = None
check_only: bool = False
+ skill_snapshot_report: Path | None = None
+ skill_snapshot_output_dir: Path | None = None
def _default_case_timeout() -> int:
@@ -121,6 +123,7 @@ def build_parser() -> argparse.ArgumentParser:
help="Where sandboxes run (default: docker). 'modal' exposes this host via Tailscale Funnel so remote sandboxes can reach it.",
)
parser.add_argument("--eval", dest="case_filter", default=None, help="Only run cases whose name contains this.")
+ parser.add_argument("--skill-snapshot-output-dir", type=Path, help=argparse.SUPPRESS)
parser.add_argument(
"--agent-model",
default=None,
@@ -215,6 +218,11 @@ def build_parser() -> argparse.ArgumentParser:
)
parser.add_argument("--comparison-dir", type=Path, help="New directory for comparison reports.")
parser.add_argument("--results-file", type=Path, help="Save a machine-readable run report to this path.")
+ parser.add_argument(
+ "--skill-snapshot-report",
+ type=Path,
+ help="Reuse a saved skill snapshot after checking that its branch skill sources match this checkout.",
+ )
parser.add_argument(
"--check",
dest="check_only",
@@ -270,6 +278,7 @@ def parse_args(argv: list[str] | None = None) -> HarnessOptions:
("--max-sandboxes", args.max_sandboxes is not None),
("--keep-sandbox-containers", args.keep_sandbox_containers),
("--rebuild-sandbox-image", args.rebuild_sandbox_image),
+ ("--skill-snapshot-report", args.skill_snapshot_report is not None),
)
if is_set
)
@@ -332,4 +341,6 @@ def parse_args(argv: list[str] | None = None) -> HarnessOptions:
comparison_dir=args.comparison_dir,
results_file=args.results_file,
check_only=args.check_only,
+ skill_snapshot_report=args.skill_snapshot_report,
+ skill_snapshot_output_dir=args.skill_snapshot_output_dir,
)
diff --git a/products/posthog_ai/eval_harness/harness/comparison.py b/products/posthog_ai/eval_harness/harness/comparison.py
index d23fe6747ed0..e1c003cdec4a 100644
--- a/products/posthog_ai/eval_harness/harness/comparison.py
+++ b/products/posthog_ai/eval_harness/harness/comparison.py
@@ -37,6 +37,7 @@ class ModelComparison:
).resolve()
self.runs = [ModelRun(runtime=target.runtime, model=target.model) for target in options.comparison_models]
self.issues: list[str] = []
+ self._snapshot_report = options.skill_snapshot_report
def _command(self, target: ModelTarget, path: Path, *, check: bool) -> list[str]:
options = self.options
@@ -69,6 +70,8 @@ class ModelComparison:
("--eval", options.case_filter),
("--reasoning-effort", options.reasoning_effort),
("--fail-under", options.fail_under),
+ ("--skill-snapshot-report", self._snapshot_report if not check else None),
+ ("--skill-snapshot-output-dir", self.directory if not check else None),
):
if value is not None:
command.extend([flag, str(value)])
@@ -135,6 +138,8 @@ class ModelComparison:
return "Run report does not match the requested model or exit status."
if not report.experiments:
return "No experiment results."
+ if not report.skills_sha256 or not report.skills_source_sha256 or not report.skills_archive_path:
+ return "No verified skill snapshot. These results cannot establish which skills were available."
if not any(
score.mean is not None and score.scored_trials
for experiment in report.experiments.values()
@@ -159,6 +164,7 @@ class ModelComparison:
report.suite_ids,
report.revision,
report.worktree_dirty,
+ report.skills_sha256,
{
name: (experiment.cases, sorted(experiment.scores))
for name, experiment in report.experiments.items()
@@ -167,12 +173,18 @@ class ModelComparison:
for report in reports
]
if any(selection != selections[0] for selection in selections[1:]):
- self.issues.append("Checkout, case, or scorer selections differ. These scores are not directly comparable.")
+ self.issues.append(
+ "Checkout, skill snapshot, case, or scorer selections differ. These scores are not directly comparable."
+ )
def _text(self) -> str:
rows = [
["Metric", *(f"{run.runtime}/{run.model}" for run in self.runs)],
["Exit status", *(str(run.exit_code) if run.exit_code is not None else "not run" for run in self.runs)],
+ [
+ "Skill snapshot",
+ *((run.report.skills_sha256 or "not recorded") if run.report else "not recorded" for run in self.runs),
+ ],
[
"Wall time",
*(f"{run.wall_time_seconds:.1f}s" if run.exit_code is not None else "n/a" for run in self.runs),
@@ -284,6 +296,14 @@ class ModelComparison:
report=self._read_report(path),
)
self.runs[index] = run.model_copy(update={"issue": self._validate_run(run)})
+ if (
+ self._snapshot_report is None
+ and run.report is not None
+ and run.report.skills_archive_path
+ and run.report.skills_source_sha256
+ and run.report.skills_sha256
+ ):
+ self._snapshot_report = path
self._save()
if code == 130:
return 130
diff --git a/products/posthog_ai/eval_harness/harness/context.py b/products/posthog_ai/eval_harness/harness/context.py
index 106cd46a8102..b90f799996e5 100644
--- a/products/posthog_ai/eval_harness/harness/context.py
+++ b/products/posthog_ai/eval_harness/harness/context.py
@@ -97,3 +97,6 @@ class EvalContext:
posthog_trace_project_id: int | None = None
infrastructure_retries: int = DEFAULT_INFRA_RETRIES
+
+ skills_sha256: str | None = None
+ skills_source_sha256: str | None = None
diff --git a/products/posthog_ai/eval_harness/harness/lifecycle.py b/products/posthog_ai/eval_harness/harness/lifecycle.py
index 91e58fb96614..03d0f4d00899 100644
--- a/products/posthog_ai/eval_harness/harness/lifecycle.py
+++ b/products/posthog_ai/eval_harness/harness/lifecycle.py
@@ -4,12 +4,15 @@ import os
import time
import atexit
import asyncio
+import hashlib
import logging
+import tempfile
import subprocess
from collections.abc import Sequence
from contextlib import AsyncExitStack, ExitStack
from dataclasses import replace
from functools import partial
+from pathlib import Path
from typing import TYPE_CHECKING
from unittest.mock import patch
@@ -23,6 +26,11 @@ from posthoganalytics import Posthog
from posthog.ph_client import get_client
from products.tasks.backend.constants import MCP_EXEC_SKILLS_FEATURE_FLAG
+from products.tasks.backend.logic.services.local_skills import (
+ ENV_EVAL_SKILLS_MANIFEST,
+ ENV_LOCAL_SKILLS_HOST_PATH,
+ LocalSkillsCache,
+)
from products.tasks.backend.temporal.process_task.utils import get_reasoning_effort_error
from ..engines.base import EvalEngine
@@ -46,6 +54,8 @@ from .ports import AI_GATEWAY_REDIS_URL, DJANGO_LIVE_PORT
from .providers import PreflightError, SandboxProviderStrategy, build_provider
from .reporting import ProgressReporter, SuiteRunResult
from .requirements import Infra, infra_union
+from .run_report import RunReport
+from .skill_snapshot import SkillSnapshot
if TYPE_CHECKING:
from .dashboard import EvalDashboard
@@ -56,7 +66,6 @@ from .services import (
ensure_ai_gateway_binary,
ensure_personhog_binaries,
maintain_ai_gateway_mint_key,
- package_local_skills_archive,
provision_ai_gateway_mint_key,
start_ai_gateway,
start_mcp_server,
@@ -126,6 +135,7 @@ class SandboxedEvalHarness:
self._demo_data: SandboxedDemoData | None = None
self._offline_delivery: OfflineEvalDelivery | None = None
self._ai_gateway_mint_key: str | None = None
+ self._skills_snapshot: SkillSnapshot | None = None
# Capture the revision before any eval starts so every result describes the same run.
self._git_sha = _git("rev-parse", "HEAD")
self._git_dirty = _worktree_dirty()
@@ -140,6 +150,13 @@ class SandboxedEvalHarness:
print(f"{suite.id} [{suite.kind.value}]") # noqa: T201
return 0
+ if (
+ self.options.skill_snapshot_report is not None
+ and self.options.results_file is not None
+ and self.options.skill_snapshot_report.resolve() == self.options.results_file.resolve()
+ ):
+ raise PreflightError("--skill-snapshot-report and --results-file must use different paths")
+
# Multi-turn cases poll once per turn, each poll with its own budget, so
# their per-case budget must scale with the turn count. Multi-turn modules
# declare the marker at module level (see discovery._collect_suites).
@@ -349,14 +366,69 @@ class SandboxedEvalHarness:
reporter.setup_started("build skills and start MCP server")
assert self._live_server is not None
- # Both delivery modes use rendered skills from this checkout, never a
- # previously published bundle. DockerSandbox bind-mounts the built skills;
- # ModalSandbox bakes them into the image it builds from the local context,
- # so it wants no host path.
- skills_dir = build_local_skills(set_bind_mount_env=self.options.provider == "docker")
+ # A private build directory keeps installed bundles and concurrent builds out of the run.
+ output_paths = tuple(
+ path
+ for path in (
+ self.options.results_file,
+ self.options.skill_snapshot_report,
+ self.options.skill_snapshot_output_dir,
+ )
+ if path is not None
+ )
+ source_sha256 = SkillSnapshot.source_fingerprint(
+ Path(settings.BASE_DIR), LocalSkillsCache().source_sha256, excluded_paths=output_paths
+ )
+ if self.options.skill_snapshot_report is not None:
+ saved = RunReport.model_validate_json(self.options.skill_snapshot_report.read_text())
+ if (
+ saved.skills_source_sha256 != source_sha256
+ or saved.revision != self._git_sha
+ or not saved.skills_archive_path
+ or not saved.skills_sha256
+ ):
+ raise RuntimeError("Saved skill snapshot does not match this checkout's branch skill sources")
+ self._skills_snapshot = SkillSnapshot.restore(
+ Path(saved.skills_archive_path),
+ self._stack,
+ expected_sha256=saved.skills_sha256,
+ source_sha256=source_sha256,
+ retain=self.options.keep_sandbox_containers,
+ )
+ else:
+ skill_build_dir = Path(
+ self._stack.enter_context(tempfile.TemporaryDirectory(prefix="posthog-eval-build-"))
+ )
+ skills_dir = build_local_skills(set_bind_mount_env=False, output_dir=skill_build_dir)
+ self._skills_snapshot = SkillSnapshot.capture(
+ skills_dir,
+ self._stack,
+ source_sha256=source_sha256,
+ retain=self.options.keep_sandbox_containers,
+ )
+ if (
+ SkillSnapshot.source_fingerprint(
+ Path(settings.BASE_DIR), LocalSkillsCache().source_sha256, excluded_paths=output_paths
+ )
+ != source_sha256
+ ):
+ raise RuntimeError("Branch skill sources changed while preparing the eval snapshot")
+ skills_dir = self._skills_snapshot.directory
+ self._stack.enter_context(
+ patch.dict(
+ os.environ,
+ {
+ ENV_LOCAL_SKILLS_HOST_PATH: str(skills_dir),
+ ENV_EVAL_SKILLS_MANIFEST: str(self._skills_snapshot.manifest_path),
+ },
+ )
+ )
+ reporter.set_skill_snapshot(self._skills_snapshot)
skill_archive_url: str | None = None
+ skill_archive_sha256: str | None = None
if self.options.skill_delivery == "exec":
- skill_archive = package_local_skills_archive(skills_dir)
+ skill_archive = self._skills_snapshot.archive_path
+ skill_archive_sha256 = hashlib.sha256(skill_archive.read_bytes()).hexdigest()
skill_archive_url, stop_skill_archive = start_skill_archive_server(skill_archive)
self._stack.callback(stop_skill_archive)
self._stack.callback(
@@ -364,6 +436,7 @@ class SandboxedEvalHarness:
self._live_server.url,
skill_archive_url,
exec_skills_enabled=self.options.skill_delivery == "exec",
+ skill_archive_sha256=skill_archive_sha256,
)
)
@@ -505,6 +578,8 @@ class SandboxedEvalHarness:
trials=self.options.trials,
git_sha=self._git_sha,
git_dirty=self._git_dirty,
+ skills_sha256=self._skills_snapshot.sha256 if self._skills_snapshot is not None else None,
+ skills_source_sha256=self._skills_snapshot.source_sha256 if self._skills_snapshot is not None else None,
offline_delivery=self._offline_delivery,
posthog_trace_host="https://us.posthog.com",
posthog_trace_project_id=settings.LLM_ANALYTICS_INTERNAL_TEAM_ID,
diff --git a/products/posthog_ai/eval_harness/harness/reporting.py b/products/posthog_ai/eval_harness/harness/reporting.py
index cf8b48a9d84a..71b821804d2d 100644
--- a/products/posthog_ai/eval_harness/harness/reporting.py
+++ b/products/posthog_ai/eval_harness/harness/reporting.py
@@ -24,6 +24,7 @@ if TYPE_CHECKING:
from .cli import HarnessOptions
from .dashboard import EvalDashboard
from .discovery import EvalSuite
+ from .skill_snapshot import SkillSnapshot
EVAL_RESULTS_JSONL = "eval_results.jsonl"
"""Machine-readable per-experiment summary export, opt-in via ``EXPORT_EVAL_RESULTS``."""
@@ -83,6 +84,7 @@ class ProgressReporter:
) -> None:
self._revision = revision
self._worktree_dirty = worktree_dirty
+ self._skills_snapshot: SkillSnapshot | None = None
self._total_suites = total_suites
self._verbose = verbose
self._lock = asyncio.Lock()
@@ -370,6 +372,10 @@ class ProgressReporter:
progress = f"[{self._case_counts[experiment_name]}/{total} cases]" if total is not None else ""
await asyncio.to_thread(_emit, f"EXPERIMENT DONE {experiment_name} {progress}")
+ def set_skill_snapshot(self, snapshot: SkillSnapshot) -> None:
+ self._skills_snapshot = snapshot
+ _emit(f"SKILLS SNAPSHOT sha256={snapshot.sha256} manifest={snapshot.manifest_path}")
+
def write_run_report(
self, options: HarnessOptions, suites: Sequence[EvalSuite], *, exit_code: int, duration_seconds: float
) -> None:
@@ -408,6 +414,12 @@ class ProgressReporter:
experiments=experiments,
revision=self._revision,
worktree_dirty=self._worktree_dirty,
+ skills_sha256=self._skills_snapshot.sha256 if self._skills_snapshot is not None else None,
+ skills_manifest_path=str(self._skills_snapshot.manifest_path)
+ if self._skills_snapshot is not None
+ else None,
+ skills_source_sha256=self._skills_snapshot.source_sha256 if self._skills_snapshot is not None else None,
+ skills_archive_path=str(self._skills_snapshot.archive_path) if self._skills_snapshot is not None else None,
)
options.results_file.parent.mkdir(parents=True, exist_ok=True)
temporary = options.results_file.with_suffix(".tmp")
diff --git a/products/posthog_ai/eval_harness/harness/run_report.py b/products/posthog_ai/eval_harness/harness/run_report.py
index 9ffe98df9ceb..0b54b55f7a99 100644
--- a/products/posthog_ai/eval_harness/harness/run_report.py
+++ b/products/posthog_ai/eval_harness/harness/run_report.py
@@ -39,4 +39,8 @@ class RunReport(BaseModel):
duration_seconds: float
revision: str | None = None
worktree_dirty: bool | None = None
+ skills_sha256: str | None = None
+ skills_manifest_path: str | None = None
+ skills_source_sha256: str | None = None
+ skills_archive_path: str | None = None
experiments: dict[str, ExperimentReport] = Field(default_factory=dict)
diff --git a/products/posthog_ai/eval_harness/harness/services.py b/products/posthog_ai/eval_harness/harness/services.py
index 52915ef2ce85..211133d9ca33 100644
--- a/products/posthog_ai/eval_harness/harness/services.py
+++ b/products/posthog_ai/eval_harness/harness/services.py
@@ -288,6 +288,7 @@ def start_mcp_server(
skill_archive_url: str | None,
*,
exec_skills_enabled: bool,
+ skill_archive_sha256: str | None = None,
) -> Callable[[], None]:
"""Start the MCP server as a subprocess for the eval session.
@@ -300,6 +301,8 @@ def start_mcp_server(
"""
if exec_skills_enabled and skill_archive_url is None:
raise ValueError("skill_archive_url is required when exec skills are enabled")
+ if exec_skills_enabled and skill_archive_sha256 is None:
+ raise ValueError("skill_archive_sha256 is required when exec skills are enabled")
mcp_dir = Path(settings.BASE_DIR) / "services" / "mcp"
if not (mcp_dir / "node_modules").exists():
@@ -341,8 +344,12 @@ def start_mcp_server(
# Never let a shell-level archive URL contaminate the bundled-skills baseline.
env.pop("POSTHOG_MCP_SKILLS_URL", None)
+ env.pop("POSTHOG_MCP_SKILLS_SHA256", None)
+ env.pop("POSTHOG_MCP_LOCAL_SKILLS_URL", None)
if exec_skills_enabled and skill_archive_url is not None:
env["POSTHOG_MCP_SKILLS_URL"] = skill_archive_url
+ assert skill_archive_sha256 is not None
+ env["POSTHOG_MCP_SKILLS_SHA256"] = skill_archive_sha256
logger.info(
"Starting MCP server (Hono runtime) on port %d (API: %s, exec skills: %s)",
@@ -488,12 +495,10 @@ def start_personhog() -> Callable[[], None]:
return stop
-def build_local_skills(*, set_bind_mount_env: bool) -> Path:
- """Build local skills once per run and return the built dist dir.
+def build_local_skills(*, set_bind_mount_env: bool, output_dir: Path | None = None) -> Path:
+ """Build rendered skills from this checkout.
- Uses a content-hash cache so repeat runs skip the build when nothing has
- changed, keeping the base image stable while letting eval authors iterate on
- skills without rebuilding it.
+ Eval runs supply a private output directory so shared builds cannot enter the snapshot.
Depends on ``DEBUG=True`` being active while the skill renderer runs — some
template helpers (e.g. HogQL example rendering) guard on that setting. The
@@ -504,13 +509,12 @@ def build_local_skills(*, set_bind_mount_env: bool) -> Path:
would pin the new source hash onto the old output, and every later run would
silently give the agent skills from another branch or edit.
"""
- cache = LocalSkillsCache()
+ cache = LocalSkillsCache(output_dir=output_dir)
+ source_hash = cache.source_sha256
dist_dir = cache.ensure_built(allow_stale=False)
+ if cache.source_sha256 != source_hash:
+ raise RuntimeError("Skill sources changed during the eval build")
if set_bind_mount_env:
- # Only ``DockerSandbox.create`` reads this to bind-mount the built skills.
- # Under the modal provider the locally built skills are instead baked into
- # the Modal image by the DEBUG ``from_dockerfile`` build, so setting it
- # there would be inert.
os.environ[ENV_LOCAL_SKILLS_HOST_PATH] = str(dist_dir)
return dist_dir
diff --git a/products/posthog_ai/eval_harness/harness/skill_snapshot.py b/products/posthog_ai/eval_harness/harness/skill_snapshot.py
new file mode 100644
index 000000000000..d8abd553d492
--- /dev/null
+++ b/products/posthog_ai/eval_harness/harness/skill_snapshot.py
@@ -0,0 +1,124 @@
+from __future__ import annotations
+
+import json
+import shutil
+import hashlib
+import zipfile
+import tempfile
+import subprocess
+from collections.abc import Sequence
+from contextlib import ExitStack
+from pathlib import Path
+from uuid import uuid4
+
+from posthog.dataclasses import frozen
+
+from products.tasks.backend.logic.services.skill_manifest import hash_skill_files
+
+from ..log_sink import LOGS_ROOT
+from .services import package_local_skills_archive
+
+
+@frozen
+class SkillSnapshot:
+ directory: Path
+ manifest_path: Path
+ sha256: str
+ source_sha256: str
+ archive_path: Path
+
+ @staticmethod
+ def source_fingerprint(base_dir: Path, skill_source_sha256: str, *, excluded_paths: Sequence[Path] = ()) -> str:
+ base_dir = base_dir.resolve()
+ excluded = tuple(path.resolve() for path in excluded_paths if path.resolve().is_relative_to(base_dir))
+ if base_dir in excluded:
+ raise ValueError("The eval output directory cannot be the checkout root")
+ digest = hashlib.sha256(skill_source_sha256.encode())
+ for staged in (False, True):
+ command = ["git", "diff", "--binary", "--no-ext-diff", "--no-textconv", "--no-color"]
+ if staged:
+ command.append("--cached")
+ command.extend(["--", ".", *(f":(exclude,literal){path.relative_to(base_dir)}" for path in excluded)])
+ digest.update(subprocess.check_output(command, cwd=base_dir))
+ digest.update(b"\0")
+ untracked = subprocess.check_output(
+ [
+ "git",
+ "ls-files",
+ "--others",
+ "--exclude-standard",
+ "-z",
+ "--",
+ "*.py",
+ "*.j2",
+ "*.json",
+ "*.toml",
+ "*.yaml",
+ "*.yml",
+ ],
+ cwd=base_dir,
+ )
+ for name in sorted(untracked.decode().split("\0")):
+ if name:
+ if any((base_dir / name).is_relative_to(path) for path in excluded):
+ continue
+ digest.update(name.encode())
+ digest.update(b"\0")
+ digest.update((base_dir / name).read_bytes())
+ digest.update(b"\0")
+ return digest.hexdigest()
+
+ @classmethod
+ def capture(cls, source: Path, stack: ExitStack, *, source_sha256: str, retain: bool = False) -> SkillSnapshot:
+ expected = hash_skill_files(source)
+ if not expected:
+ raise RuntimeError("Cannot run evals with an empty skill snapshot")
+ manifests = LOGS_ROOT / "skill-snapshots"
+ manifests.mkdir(parents=True, exist_ok=True)
+ snapshot_id = uuid4().hex
+ root = (
+ manifests / snapshot_id
+ if retain
+ else Path(stack.enter_context(tempfile.TemporaryDirectory(prefix="posthog-eval-skills-")))
+ )
+ directory = root / "skills"
+ shutil.copytree(source, directory, ignore=shutil.ignore_patterns(".*", "__pycache__"))
+ if hash_skill_files(directory) != expected or hash_skill_files(source) != expected:
+ raise RuntimeError("Skill files changed while capturing the eval snapshot")
+ for path in directory.rglob("*"):
+ if path.is_file():
+ path.chmod(0o444)
+ manifest = (json.dumps(expected, sort_keys=True, indent=2) + "\n").encode()
+ digest = hashlib.sha256(manifest).hexdigest()
+ manifest_path = manifests / f"{snapshot_id}_{digest}.json"
+ manifest_path.write_bytes(manifest)
+ manifest_path.chmod(0o444)
+ archive_path = package_local_skills_archive(directory, manifest_path.with_suffix(".zip"))
+ archive_path.chmod(0o444)
+ return cls(
+ directory=directory,
+ manifest_path=manifest_path,
+ sha256=digest,
+ source_sha256=source_sha256,
+ archive_path=archive_path,
+ )
+
+ @classmethod
+ def restore(
+ cls, archive: Path, stack: ExitStack, *, expected_sha256: str, source_sha256: str, retain: bool = False
+ ) -> SkillSnapshot:
+ directory = Path(stack.enter_context(tempfile.TemporaryDirectory(prefix="posthog-eval-restore-")))
+ with zipfile.ZipFile(archive) as bundle:
+ for entry in bundle.infolist():
+ relative = Path(entry.filename)
+ if relative.is_absolute() or ".." in relative.parts:
+ raise RuntimeError("Invalid path in the eval skill snapshot")
+ if entry.is_dir():
+ continue
+ destination = directory / relative
+ destination.parent.mkdir(parents=True, exist_ok=True)
+ destination.write_bytes(bundle.read(entry))
+ snapshot = cls.capture(directory, stack, source_sha256=source_sha256, retain=retain)
+ if snapshot.sha256 != expected_sha256:
+ raise RuntimeError("Saved eval skill snapshot does not match its recorded hash")
+ return snapshot
diff --git a/products/posthog_ai/eval_harness/test/test_cli_mcp_scorers.py b/products/posthog_ai/eval_harness/test/test_cli_mcp_scorers.py
index de14426be006..1abc17864ec0 100644
--- a/products/posthog_ai/eval_harness/test/test_cli_mcp_scorers.py
+++ b/products/posthog_ai/eval_harness/test/test_cli_mcp_scorers.py
@@ -377,6 +377,50 @@ def test_chart_scorers_grade_each_chart_path(
assert ChartFromTypedQuery()._run_eval_sync(output, expected={"chart_from_typed_query": spec}).score == typed
+ [
+ (
+ "native_metadata_without_render_note",
+ {"_meta": {"com.posthog.mcp/app_data": {"query": {"kind": "TrendsQuery"}, "results": []}}},
+ "completed",
+ 1.0,
+ ),
+ (
+ "native_structured_content",
+ {"structuredContent": {"query": {"kind": "TrendsQuery"}, "results": []}},
+ "completed",
+ 1.0,
+ ),
+ (
+ "wrong_query_kind",
+ {"_meta": {"com.posthog.mcp/app_data": {"query": {"kind": "HogQLQuery", "query": "SELECT 1"}}}},
+ "completed",
+ 0.0,
+ ),
+ (
+ "failed_native_query",
+ {"_meta": {"com.posthog.mcp/app_data": {"query": {"kind": "TrendsQuery"}}}},
+ "failed",
+ 0.0,
+ ),
+ ("desktop_note_without_native_payload", {"content": INLINE_VIEW_NOTE}, "completed", 0.0),
+ ]
+)
+def test_posthog_ai_chart_scorers_require_native_query_data(
+ _name: str, payload: dict, status: str, shown: float
+) -> None:
+ output = _chart_output(
+ [("mcp__posthog__query-trends", {"series": []}, json.dumps(payload), status)],
+ answer=f"Signups rose.\n\n{_SQL_CHART}",
+ )
+ spec = {"tool_name_any_of": ["query-trends"], "surface": "posthog_ai"}
+
+ assert ChartShown()._run_eval_sync(output, expected={"chart_shown": spec}).score == shown
+ assert ChartFromTypedQuery()._run_eval_sync(output, expected={"chart_from_typed_query": spec}).score == (
+ 1.0 if shown else None
+ )
+
+
_LONG_TABLE = "| Week | Signups |\n|---|---|\n| W1 | 10 |\n| W2 | 12 |\n| W3 | 9 |\n| W4 | 14 |"
_SUMMARY_TABLE = "| Period | Signups |\n| :--- | ---: |\n| Last 7 days | 45 |\n| Previous 7 days | 38 |"
diff --git a/products/posthog_ai/eval_harness/test/test_model_comparison.py b/products/posthog_ai/eval_harness/test/test_model_comparison.py
index c780b81e8077..c0508090cf3a 100644
--- a/products/posthog_ai/eval_harness/test/test_model_comparison.py
+++ b/products/posthog_ai/eval_harness/test/test_model_comparison.py
@@ -4,6 +4,7 @@ import os
import sys
import json
import signal
+import subprocess
from pathlib import Path
import pytest
@@ -15,12 +16,14 @@ from products.posthog_ai.eval_harness.harness.discovery import EvalSuite
from products.posthog_ai.eval_harness.harness.reporting import ProgressReporter
from products.posthog_ai.eval_harness.harness.requirements import SuiteKind
from products.posthog_ai.eval_harness.harness.run_report import ExperimentReport, RunReport, ScoreReport
+from products.posthog_ai.eval_harness.harness.skill_snapshot import SkillSnapshot
@pytest.mark.parametrize(
"scenario,expected_exit,expected_calls",
[
("success", 0, 4),
+ ("in_checkout_output", 0, 4),
("skipped", 1, 4),
("filtered_out_experiment", 0, 4),
("launch_error", 1, 1),
@@ -29,6 +32,8 @@ from products.posthog_ai.eval_harness.harness.run_report import ExperimentReport
("different_cases", 1, 4),
("different_scorers", 1, 4),
("different_checkout", 1, 4),
+ ("different_skills", 1, 4),
+ ("missing_skills", 1, 4),
("missing_report", 1, 4),
("failed_check", 1, 1),
("one_shot", 1, 1),
@@ -44,6 +49,8 @@ def test_model_comparison_preserves_settings_and_partial_results(
expected_calls: int,
) -> None:
directory = tmp_path / "comparison"
+ if scenario == "in_checkout_output":
+ subprocess.run(["git", "init", "-q", str(tmp_path)], check=True)
options = parse_args(
[
"example/suite",
@@ -69,8 +76,10 @@ def test_model_comparison_preserves_settings_and_partial_results(
]
)
calls: list[bool] = []
+ fingerprint: str | None = None
def execute(command: list[str]) -> int:
+ nonlocal fingerprint
child = parse_args(command[3:])
calls.append(child.check_only)
assert child.selectors == options.selectors
@@ -85,6 +94,20 @@ def test_model_comparison_preserves_settings_and_partial_results(
assert child.interactive is False
assert not child.comparison_models
assert child.results_file is not None
+ if scenario == "in_checkout_output" and not child.check_only:
+ assert child.skill_snapshot_output_dir == directory
+ current = SkillSnapshot.source_fingerprint(
+ tmp_path, "skills", excluded_paths=(child.skill_snapshot_output_dir,)
+ )
+ if fingerprint is None:
+ fingerprint = current
+ assert current == fingerprint
+ if (
+ not child.check_only
+ and child.agent_runtime == "codex"
+ and scenario not in {"missing_report", "missing_skills"}
+ ):
+ assert child.skill_snapshot_report == directory / "1-results.json"
if child.check_only and scenario == "failed_check":
return 2
if scenario == "launch_error":
@@ -107,6 +130,13 @@ def test_model_comparison_preserves_settings_and_partial_results(
exit_code=int(error),
duration_seconds=2,
revision="changed" if scenario == "different_checkout" and second else "example-revision",
+ skills_sha256=None
+ if scenario == "missing_skills"
+ else "b" * 64
+ if scenario == "different_skills" and second
+ else "a" * 64,
+ skills_source_sha256="b" * 64,
+ skills_archive_path=str(tmp_path / "skills.zip"),
experiments={}
if child.check_only
else {
@@ -216,6 +246,15 @@ async def test_run_report_preserves_zero_skipped_scores_and_scorer_errors(
monkeypatch.delenv("EXPORT_EVAL_RESULTS", raising=False)
options = parse_args(["example", "--trials", "3", "--results-file", str(tmp_path / "results.json")])
reporter = ProgressReporter(1, revision="example-revision", worktree_dirty=True)
+ reporter.set_skill_snapshot(
+ SkillSnapshot(
+ directory=tmp_path / "skills",
+ manifest_path=tmp_path / "manifest.json",
+ sha256="a" * 64,
+ source_sha256="b" * 64,
+ archive_path=tmp_path / "skills.zip",
+ )
+ )
await reporter.experiment_started("example", 3, tmp_path, case_names=["selected"])
for status in ("ok", "ok", "timeout"):
await reporter.case_done("example", "selected", duration_seconds=1, status=status)
@@ -261,4 +300,8 @@ async def test_run_report_preserves_zero_skipped_scores_and_scorer_errors(
assert experiment.scores["skipped"].scored_trials == 0
assert report.revision == "example-revision"
assert report.worktree_dirty is True
+ assert report.skills_sha256 == "a" * 64
+ assert report.skills_manifest_path == str(tmp_path / "manifest.json")
+ assert report.skills_source_sha256 == "b" * 64
+ assert report.skills_archive_path == str(tmp_path / "skills.zip")
assert not list(tmp_path.glob("*.tmp"))
diff --git a/products/posthog_ai/eval_harness/test/test_sandboxed_harness.py b/products/posthog_ai/eval_harness/test/test_sandboxed_harness.py
index c0bf097541b0..65936bb01346 100644
--- a/products/posthog_ai/eval_harness/test/test_sandboxed_harness.py
+++ b/products/posthog_ai/eval_harness/test/test_sandboxed_harness.py
@@ -31,8 +31,14 @@ from products.posthog_ai.eval_harness.harness.providers import (
SandboxProviderStrategy,
)
from products.posthog_ai.eval_harness.harness.reporting import ProgressReporter
-from products.tasks.backend.facade.agents import MultiTurnSession, TurnPollResult, TurnPollTimeout
+from products.tasks.backend.facade.agents import (
+ CustomPromptSandboxContext,
+ MultiTurnSession,
+ TurnPollResult,
+ TurnPollTimeout,
+)
from products.tasks.backend.facade.contracts import AgentTaskRunDTO
+from products.tasks.backend.logic.services import custom_prompt_internals
from products.tasks.backend.temporal.process_task.activities.get_task_processing_context import TaskProcessingContext
from products.tasks.backend.temporal.process_task.utils import mcp_exec_skills_env_vars
@@ -82,6 +88,29 @@ class _Provider(SandboxProviderStrategy):
_TASK_ID = UUID("00000000-0000-0000-0000-000000000001")
+ "system_prompt",
+ [None, {"type": "preset", "preset": "claude_code", "append": "Test surface instructions"}],
+)
+async def test_surface_prompt_reaches_task_run_state(
+ monkeypatch: pytest.MonkeyPatch, system_prompt: dict[str, str] | None
+) -> None:
+ task = MagicMock(id=_TASK_ID, team_id=25)
+ task.latest_run = MagicMock(id=UUID("00000000-0000-0000-0000-000000000002"))
+ create = MagicMock(return_value=task)
+ monkeypatch.setattr(custom_prompt_internals.Team.objects, "get", MagicMock(return_value=MagicMock(id=25)))
+ monkeypatch.setattr(custom_prompt_internals.Task, "create_and_run", create)
+
+ await runner.create_task_and_trigger(
+ "prompt", CustomPromptSandboxContext(team_id=25, user_id=26, system_prompt=system_prompt)
+ )
+
+ assert create.call_args.kwargs["extra_run_state"] == (
+ {"systemPrompt": system_prompt} if system_prompt is not None else {}
+ )
+
+
def _patch_runner_boundaries(monkeypatch: pytest.MonkeyPatch, handle: _FakeWorkflowHandle, poll: AsyncMock) -> None:
task_run = AgentTaskRunDTO(
task_id=_TASK_ID,
@@ -230,6 +259,17 @@ def test_eval_run_exit_status(
assert f"Status: {'PASS' if expected_exit_code == 0 else 'FAIL'}" in output
+def test_eval_run_rejects_overwriting_its_snapshot_input(tmp_path: Path) -> None:
+ report = tmp_path / "results.json"
+ report.write_text('{"saved": true}')
+ harness = SandboxedEvalHarness(parse_args(["--skill-snapshot-report", str(report), "--results-file", str(report)]))
+
+ with pytest.raises(lifecycle.PreflightError, match="must use different paths"):
+ harness.run()
+
+ assert report.read_text() == '{"saved": true}'
+
+
@pytest.mark.asyncio
@pytest.mark.parametrize("delivery", ["bundled", "exec"])
@pytest.mark.parametrize("origin", ["eval", "slack", None])
diff --git a/products/posthog_ai/eval_harness/test/test_sandboxed_harness_output.py b/products/posthog_ai/eval_harness/test/test_sandboxed_harness_output.py
index e900c5d7dc35..fb1f17bef9d8 100644
--- a/products/posthog_ai/eval_harness/test/test_sandboxed_harness_output.py
+++ b/products/posthog_ai/eval_harness/test/test_sandboxed_harness_output.py
@@ -657,6 +657,9 @@ async def test_sandboxed_eval_skill_delivery_sets_the_agent_origin(
launched_context = launch.call_args.args[1]
assert launched_context.interaction_origin == expected_origin
+ assert launched_context.system_prompt == (
+ dict(base.PromptService.build()) if expected_origin == "posthog_ai" else None
+ )
assert ctx.demo_data.make_context.call_args.kwargs["disable_bundled_skills"] == (delivery == "exec")
if has_original:
assert setup.call_args.args[0].interaction_origin == expected_origin
diff --git a/products/posthog_ai/eval_harness/test/test_sandboxed_harness_services.py b/products/posthog_ai/eval_harness/test/test_sandboxed_harness_services.py
index 8acfd27a1585..9446bad5a3aa 100644
--- a/products/posthog_ai/eval_harness/test/test_sandboxed_harness_services.py
+++ b/products/posthog_ai/eval_harness/test/test_sandboxed_harness_services.py
@@ -6,6 +6,7 @@ import asyncio
import zipfile
import itertools
import subprocess
+from contextlib import ExitStack
from functools import partial
from pathlib import Path
from types import SimpleNamespace
@@ -30,11 +31,144 @@ from products.posthog_ai.eval_harness.harness.services import (
provision_ai_gateway_mint_key,
start_mcp_server,
)
+from products.posthog_ai.eval_harness.harness.skill_snapshot import SkillSnapshot
from products.posthog_ai.eval_harness.long_lived_subprocess import (
LongLivedSubprocessManager,
SubprocessReadinessTimeout,
SubprocessStartupError,
)
+from products.tasks.backend.exceptions import ProcessTaskFatalError
+from products.tasks.backend.logic.services.local_skills import (
+ ENV_EVAL_SKILLS_MANIFEST,
+ ENV_LOCAL_SKILLS_HOST_PATH,
+ populate_skills_directory,
+ verify_eval_skills,
+)
+from products.tasks.backend.logic.services.skill_manifest import hash_skill_files, verify_skill_files
+
+
[email protected]("retain", [False, True])
+def test_skill_snapshot_keeps_branch_files_when_the_build_directory_changes(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, retain: bool
+) -> None:
+ monkeypatch.setattr("products.posthog_ai.eval_harness.harness.skill_snapshot.LOGS_ROOT", tmp_path / "logs")
+ source = tmp_path / "built"
+ skill = source / "sample" / "SKILL.md"
+ skill.parent.mkdir(parents=True)
+ skill.write_text("branch version")
+ with ExitStack() as stack:
+ snapshot = SkillSnapshot.capture(source, stack, source_sha256="source", retain=retain)
+ skill.write_text("later version")
+ assert (snapshot.directory / "sample" / "SKILL.md").read_text() == "branch version"
+ assert json.loads(snapshot.manifest_path.read_text()) == hash_skill_files(snapshot.directory)
+ restored = SkillSnapshot.restore(
+ snapshot.archive_path,
+ stack,
+ expected_sha256=snapshot.sha256,
+ source_sha256=snapshot.source_sha256,
+ retain=retain,
+ )
+ assert restored.sha256 == snapshot.sha256
+ assert (restored.directory / "sample" / "SKILL.md").read_text() == "branch version"
+ with pytest.raises(RuntimeError, match="recorded hash"):
+ SkillSnapshot.restore(
+ snapshot.archive_path, stack, expected_sha256="0" * 64, source_sha256=snapshot.source_sha256
+ )
+ assert (snapshot.directory / "sample" / "SKILL.md").is_file() is retain
+ assert (restored.directory / "sample" / "SKILL.md").is_file() is retain
+
+
+def test_skill_verification_rejects_stale_native_copies(tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None:
+ roots = [tmp_path / name for name in ("plugin", "claude", "codex")]
+ for index, root in enumerate(roots):
+ skill = root / "sample" / "SKILL.md"
+ skill.parent.mkdir(parents=True)
+ skill.write_text("branch version" if index == 0 else "image version")
+ expected = hash_skill_files(roots[0])
+ with pytest.raises(RuntimeError, match="snapshot mismatch"):
+ verify_skill_files(expected, [str(root) for root in roots])
+ manifest = tmp_path / "manifest.json"
+ manifest.write_text(json.dumps(expected))
+ monkeypatch.setenv(ENV_EVAL_SKILLS_MANIFEST, str(manifest))
+ sandbox = MagicMock()
+ sandbox.config.environment_variables = {}
+ sandbox.write_file.return_value.exit_code = 0
+ sandbox.execute_in_task_filesystem.return_value = SimpleNamespace(
+ exit_code=1, stdout="", stderr="Eval skill snapshot mismatch"
+ )
+ with pytest.raises(ProcessTaskFatalError, match="Eval skill verification failed") as failure:
+ verify_eval_skills(sandbox)
+ assert failure.value.non_retryable
+ for root in roots[1:]:
+ (root / "sample" / "SKILL.md").write_text("branch version")
+ verify_skill_files(expected, [str(root) for root in roots])
+
+
[email protected]("disabled", [False, True])
+def test_skill_verification_uses_task_filesystem_and_running_container_environment(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch, disabled: bool
+) -> None:
+ root = tmp_path / "agent-home"
+ skill = root / "sample" / "SKILL.md"
+ skill.parent.mkdir(parents=True)
+ skill.write_text("branch version")
+ manifest = tmp_path / "manifest.json"
+ manifest.write_text(json.dumps(hash_skill_files(root)))
+ if disabled:
+ skill.unlink()
+ monkeypatch.setenv(ENV_EVAL_SKILLS_MANIFEST, str(manifest))
+ monkeypatch.setattr("products.tasks.backend.logic.services.local_skills.BUNDLED_SKILLS_PATHS", (str(root),))
+ sandbox = MagicMock()
+ sandbox.config.environment_variables = {}
+ sandbox.write_file.return_value.exit_code = 0
+
+ def execute_in_task_filesystem(command: str, timeout_seconds: int) -> SimpleNamespace:
+ target, payload = sandbox.write_file.call_args.args
+ assert target.startswith("/opt/posthog/")
+ program = tmp_path / "verification.py"
+ program.write_bytes(payload)
+ result = subprocess.run(
+ ["python3", str(program)],
+ env={**os.environ, "POSTHOG_CODE_DISABLE_BUNDLED_SKILLS": "1" if disabled else "0"},
+ capture_output=True,
+ text=True,
+ timeout=timeout_seconds,
+ check=False,
+ )
+ return SimpleNamespace(exit_code=result.returncode, stdout=result.stdout, stderr=result.stderr)
+
+ sandbox.execute_in_task_filesystem.side_effect = execute_in_task_filesystem
+ verify_eval_skills(sandbox)
+ sandbox.execute.assert_not_called()
+
+
+def test_skill_source_fingerprint_detects_schema_edits_outside_skill_directories(tmp_path: Path) -> None:
+ subprocess.run(["git", "init", "-q", str(tmp_path)], check=True)
+ schema = tmp_path / "schema.py"
+ schema.write_text("version = 1\n")
+ subprocess.run(["git", "add", "schema.py"], cwd=tmp_path, check=True)
+ original = SkillSnapshot.source_fingerprint(tmp_path, "skills")
+ schema.write_text("version = 2\n")
+ changed = SkillSnapshot.source_fingerprint(tmp_path, "skills")
+ assert changed != original
+ (tmp_path / "new_schema.py").write_text("version = 3\n")
+ assert SkillSnapshot.source_fingerprint(tmp_path, "skills") != changed
+
+
+def test_eval_skill_population_uses_the_snapshot_instead_of_the_shared_build(
+ tmp_path: Path, monkeypatch: pytest.MonkeyPatch
+) -> None:
+ snapshot = tmp_path / "snapshot"
+ shared = tmp_path / "products/posthog_ai/dist/skills"
+ for root, content in ((snapshot, "snapshot version"), (shared, "shared build version")):
+ skill = root / "sample" / "SKILL.md"
+ skill.parent.mkdir(parents=True)
+ skill.write_text(content)
+ monkeypatch.setenv(ENV_EVAL_SKILLS_MANIFEST, "manifest.json")
+ monkeypatch.setenv(ENV_LOCAL_SKILLS_HOST_PATH, str(snapshot))
+ destination = tmp_path / "modal-build"
+ populate_skills_directory(destination, base_dir=tmp_path)
+ assert (destination / "sample" / "SKILL.md").read_text() == "snapshot version"
def test_package_local_skills_archive_is_deterministic_and_excludes_cache_markers(tmp_path: Path) -> None:
@@ -73,7 +207,14 @@ def test_start_mcp_server_isolates_the_selected_skill_delivery(
stop = MagicMock()
with (
- patch.dict(os.environ, {"POSTHOG_MCP_SKILLS_URL": "https://stale.example/skills.zip"}),
+ patch.dict(
+ os.environ,
+ {
+ "POSTHOG_MCP_SKILLS_URL": "https://stale.example/skills.zip",
+ "POSTHOG_MCP_LOCAL_SKILLS_URL": "https://local.example/skills.zip",
+ "POSTHOG_MCP_SKILLS_SHA256": "stale",
+ },
+ ),
patch("products.posthog_ai.eval_harness.harness.services.settings.BASE_DIR", tmp_path),
patch(
"products.posthog_ai.eval_harness.harness.services.LONG_LIVED_SUBPROCESSES.start",
@@ -84,6 +225,7 @@ def test_start_mcp_server_isolates_the_selected_skill_delivery(
"http://localhost:18000",
skill_archive_url,
exec_skills_enabled=exec_skills_enabled,
+ skill_archive_sha256="a" * 64 if exec_skills_enabled else None,
)
env = start.call_args.kwargs["env"]
@@ -95,8 +237,11 @@ def test_start_mcp_server_isolates_the_selected_skill_delivery(
assert overrides["dashboard-widgets"] is True
if skill_archive_url is None:
assert "POSTHOG_MCP_SKILLS_URL" not in env
+ assert "POSTHOG_MCP_SKILLS_SHA256" not in env
else:
assert env["POSTHOG_MCP_SKILLS_URL"] == skill_archive_url
+ assert env["POSTHOG_MCP_SKILLS_SHA256"] == "a" * 64
+ assert "POSTHOG_MCP_LOCAL_SKILLS_URL" not in env
assert result is stop
diff --git a/products/posthog_ai/evals/cli_mcp/scorers.py b/products/posthog_ai/evals/cli_mcp/scorers.py
index 6e57f9e10b3a..1c4fe42b52c0 100644
--- a/products/posthog_ai/evals/cli_mcp/scorers.py
+++ b/products/posthog_ai/evals/cli_mcp/scorers.py
@@ -1330,10 +1330,44 @@ def _render_matches_query(render: ToolCall, calls: list[ToolCall]) -> bool:
)
-def _chart_paths(output: dict | None, parser: LogParser, tools: set[str]) -> list[str]:
- """How the user saw a chart: ``render-ui``, ``inline`` (the client's UI app), or a ``chart_references`` kind."""
+def _native_query_chart(call: ToolCall) -> bool:
+ """PostHog AI's extractQueryResult widget reads metadata without an inline-view text note."""
+ try:
+ payload = json.loads(call.output)
+ except (ValueError, TypeError):
+ return False
+ if not isinstance(payload, dict) or payload.get("isError") is True:
+ return False
+ meta = payload.get("_meta")
+ data = meta.get("com.posthog.mcp/app_data") if isinstance(meta, dict) else None
+ if not isinstance(data, dict):
+ data = payload.get("structuredContent")
+ if not isinstance(data, dict):
+ data = payload
+ query = data.get("query")
+ if not isinstance(query, dict):
+ return False
+ if query.get("kind") == "InsightVizNode":
+ query = query.get("source")
+ if not isinstance(query, dict):
+ return False
+ kinds = {
+ "query-trends": "TrendsQuery",
+ "query-funnel": "FunnelsQuery",
+ "query-retention": "RetentionQuery",
+ "query-paths": "PathsQuery",
+ "query-lifecycle": "LifecycleQuery",
+ "query-stickiness": "StickinessQuery",
+ }
+ return call.name in kinds and query.get("kind") == kinds[call.name]
+
+
+def _chart_paths(output: dict | None, parser: LogParser, tools: set[str], surface: str | None = None) -> list[str]:
+ """Native widget data or the chart paths supported by the default MCP app scorer."""
paths: list[str] = []
calls = parser.get_tool_calls()
+ if surface == "posthog_ai":
+ return ["native" for call in calls if not call.is_error and call.name in tools and _native_query_chart(call)]
for call in _render_ui_calls(parser):
if not call.is_error and _rendered_tool_name(call) in tools and _render_matches_query(call, calls):
paths.append("render-ui")
@@ -1362,6 +1396,9 @@ class ChartShown(Scorer):
that the client already displays inline (``INLINE_VIEW_NOTE``), or a full-size chart link or
block tag in the answer (a saved insight or a SQL query). ``ChartFromTypedQuery``
grades which of these the agent chose. ``None`` if the spec is absent.
+
+ ``surface="posthog_ai"`` checks the native query widget payload instead. It does
+ not count Desktop links or require an MCP app's text note. Neither mode checks pixels.
"""
def _name(self) -> str:
@@ -1374,7 +1411,8 @@ class ChartShown(Scorer):
parser = _build_parser(output)
if parser is None:
return Score(name=self._name(), score=None, metadata={"reason": "No raw log"})
- paths = _chart_paths(output, parser, tools)
+ spec = _read_spec(expected, self._name()) or {}
+ paths = _chart_paths(output, parser, tools, spec.get("surface"))
if paths:
return Score(name=self._name(), score=1.0, metadata={"paths": paths})
return Score(
@@ -1392,7 +1430,7 @@ class ChartFromTypedQuery(Scorer):
An agent that answers with ``query-trends`` and then links a hand-written SQL chart shows
numbers computed by a different query, without PostHog's standard definitions such as the
project timezone or test-account filtering. Score 1.0 when any chart comes from
- ``render-ui``, the inline view, or a saved insight that holds a typed query; 0.0 when every
+ a native PostHog AI widget, ``render-ui``, the inline view, or a saved insight that holds a typed query; 0.0 when every
chart is a SQL query. ``None`` when no chart was shown (graded by ``ChartShown``), when the only
other charts are saved insights whose query kind no tool call showed, or the spec is absent.
"""
@@ -1407,7 +1445,8 @@ class ChartFromTypedQuery(Scorer):
parser = _build_parser(output)
if parser is None:
return Score(name=self._name(), score=None, metadata={"reason": "No raw log"})
- paths = _chart_paths(output, parser, tools)
+ spec = _read_spec(expected, self._name()) or {}
+ paths = _chart_paths(output, parser, tools, spec.get("surface"))
if not paths:
return Score(name=self._name(), score=None, metadata={"reason": "No chart was shown"})
if any(path not in ("sql", "saved") for path in paths):
diff --git a/products/posthog_ai/evals/offline_config.py b/products/posthog_ai/evals/offline_config.py
index 0f6fd1df4bba..df6eb0c7a211 100644
--- a/products/posthog_ai/evals/offline_config.py
+++ b/products/posthog_ai/evals/offline_config.py
@@ -134,8 +134,8 @@ _SCORERS: dict[str, dict[str, OfflineScorer]] = {
kind="boolean", version=UUID("01a15a27-735b-700b-9f5b-51533df620b3")
),
"no_markdown_data_table": OfflineScorer(kind="boolean", version=UUID("01a15a27-69b4-7bc0-b6f6-b366acc272b2")),
- "chart_from_typed_query": OfflineScorer(kind="boolean", version=UUID("01a15a27-5e27-7511-9f3e-58a57e74556d")),
- "chart_shown": OfflineScorer(kind="boolean", version=UUID("01a15a27-553d-72d2-aeb2-903e7ed54473")),
+ "chart_from_typed_query": OfflineScorer(kind="boolean", version=UUID("01a16420-8c66-7f91-9939-997516aafc3b")),
+ "chart_shown": OfflineScorer(kind="boolean", version=UUID("01a16420-30a5-770e-a4f3-16ff4f7dcb84")),
"answer_from_typed_query": OfflineScorer(kind="boolean", version=UUID("01a15a27-4bce-7e6c-b3ef-34d292489b30")),
"answer_tool_not_execute_sql": OfflineScorer(
kind="boolean", version=UUID("01a15581-3926-7c14-a907-e994586bd216")
diff --git a/products/tasks/backend/logic/services/agent_server_launcher.py b/products/tasks/backend/logic/services/agent_server_launcher.py
index ee0c41984374..f8c6a10e7778 100644
--- a/products/tasks/backend/logic/services/agent_server_launcher.py
+++ b/products/tasks/backend/logic/services/agent_server_launcher.py
@@ -37,6 +37,7 @@ from products.tasks.backend.logic.services.agentsh import (
generate_policy_yaml,
read_gh_guard_script,
)
+from products.tasks.backend.logic.services.local_skills import verify_eval_skills
from products.tasks.backend.logic.services.mcp_url import resolve_mcp_url
from products.tasks.backend.logic.services.memory_watchdog import (
MEMORY_WATCHDOG_MISSING_MARKER,
@@ -733,6 +734,7 @@ class AgentServerLaunchMixin(SandboxBase):
# Before the already-healthy shortcut: images that boot the agent server never relaunch it,
# and the agent reads its skill directories when a session starts, not when the server does.
preflight = self._agent_server_preflight(allowed_domains)
+ verify_eval_skills(self)
if preflight.reused:
return 0 if wait_for_health else None
diff --git a/products/tasks/backend/logic/services/custom_prompt_internals.py b/products/tasks/backend/logic/services/custom_prompt_internals.py
index 4d949e36d017..8dcd8083229b 100644
--- a/products/tasks/backend/logic/services/custom_prompt_internals.py
+++ b/products/tasks/backend/logic/services/custom_prompt_internals.py
@@ -161,6 +161,7 @@ class CustomPromptSandboxContext:
"""Surface the run is answering on (e.g. ``"slack"``). The agent server branches its system
prompt on this, so evals that grade surface-specific behavior must set it to exercise the
prompt the surface really ships. ``None`` leaves the run originless, like a plain task."""
+ system_prompt: dict[str, str] | None = None
mcp_exclude_tools: tuple[str, ...] = ()
"""Tool names to omit from the PostHog MCP catalog for this run (``x-posthog-exclude-tools``).
Used when a scope grant is broader than the tools this caller should advertise, e.g. hiding
@@ -337,6 +338,8 @@ async def _create_task_and_trigger(
context.posthog_mcp_scopes if context.posthog_mcp_scopes is not None else "full"
)
extra_run_state: dict[str, Any] = {}
+ if context.system_prompt is not None:
+ extra_run_state["systemPrompt"] = context.system_prompt
if context.mcp_exclude_tools:
extra_run_state["mcp_exclude_tools"] = list(context.mcp_exclude_tools)
if output_schema:
diff --git a/products/tasks/backend/logic/services/docker_sandbox.py b/products/tasks/backend/logic/services/docker_sandbox.py
index 62c95083dbfa..880a02bc651f 100644
--- a/products/tasks/backend/logic/services/docker_sandbox.py
+++ b/products/tasks/backend/logic/services/docker_sandbox.py
@@ -50,6 +50,8 @@ from .agentsh import (
read_gh_guard_script,
)
from .local_skills import (
+ BUNDLED_SKILLS_PATHS,
+ ENV_EVAL_SKILLS_MANIFEST,
ENV_LOCAL_SKILLS_HOST_PATH,
LocalSkillsCache,
bundled_skills_disabled,
@@ -549,11 +551,8 @@ class DockerSandbox(AgentServerLaunchMixin):
container_path = f"{WORKING_DIR}/repos/{org}/{repo}"
volume_args.extend(["-v", f"{local_path}:{container_path}"])
- # Opt-in bind-mount for local skills. Set by the eval harness so
- # sandboxes see the working-tree skills without rebuilding the
- # base image. Mounts per-subdirectory rather than the parent so
- # the baked-in rendered skills in the image stay visible — only
- # the specific skills the user has on disk get overlaid.
+ # Native runtimes need the same files as plugin discovery. Eval snapshots replace
+ # whole directories so image-only skills cannot enter the comparison.
local_skills_host = os.environ.get(ENV_LOCAL_SKILLS_HOST_PATH)
if (
skill_source != "local"
@@ -561,14 +560,16 @@ class DockerSandbox(AgentServerLaunchMixin):
and local_skills_host
and os.path.isdir(local_skills_host)
):
- for entry in sorted(os.listdir(local_skills_host)):
+ entries = [""] if os.environ.get(ENV_EVAL_SKILLS_MANIFEST) else sorted(os.listdir(local_skills_host))
+ for entry in entries:
if entry.startswith(".") or entry == "__pycache__":
continue
- host_skill = os.path.join(local_skills_host, entry)
+ host_skill = str(Path(local_skills_host) / entry)
if not os.path.isdir(host_skill):
continue
- container_skill = f"/scripts/plugins/posthog/skills/{entry}"
- volume_args.extend(["-v", f"{host_skill}:{container_skill}:ro"])
+ for skills_path in BUNDLED_SKILLS_PATHS:
+ container_skill = str(Path(skills_path) / entry)
+ volume_args.extend(["-v", f"{host_skill}:{container_skill}:ro"])
cap_args = ["--cap-add", "SYS_PTRACE"]
if config.template == SandboxTemplate.PI_BASE:
@@ -642,6 +643,8 @@ class DockerSandbox(AgentServerLaunchMixin):
@staticmethod
def _local_skill_source(config: SandboxConfig) -> str | None:
+ if os.environ.get(ENV_EVAL_SKILLS_MANIFEST):
+ return None
if not settings.DEBUG or config.template not in {
SandboxTemplate.DEFAULT_BASE,
SandboxTemplate.VM_BASE,
diff --git a/products/tasks/backend/logic/services/local_skills.py b/products/tasks/backend/logic/services/local_skills.py
index 94c7fc795e8f..e87730a710b8 100644
--- a/products/tasks/backend/logic/services/local_skills.py
+++ b/products/tasks/backend/logic/services/local_skills.py
@@ -21,18 +21,28 @@ from __future__ import annotations
import os
import fcntl
+import shlex
import shutil
import hashlib
import logging
from pathlib import Path
+from typing import TYPE_CHECKING
from django.conf import settings
+from pydantic import TypeAdapter
+
+from products.tasks.backend.exceptions import ProcessTaskFatalError
+
+if TYPE_CHECKING:
+ from .sandbox import SandboxBase
+
logger = logging.getLogger(__name__)
BUILT_SKILLS_RELATIVE_PATH = Path("products/posthog_ai/dist/skills")
BUILD_HASH_FILENAME = ".build-hash"
ENV_LOCAL_SKILLS_HOST_PATH = "SANDBOX_LOCAL_SKILLS_HOST_PATH"
+ENV_EVAL_SKILLS_MANIFEST = "SANDBOX_EVAL_SKILLS_MANIFEST"
ENV_DISABLE_BUNDLED_SKILLS = "POSTHOG_CODE_DISABLE_BUNDLED_SKILLS"
BUNDLED_SKILLS_PATHS: tuple[str, ...] = (
"/scripts/plugins/posthog/skills",
@@ -45,6 +55,47 @@ def bundled_skills_disabled(environment_variables: dict[str, str] | None) -> boo
return environment_variables is not None and environment_variables.get(ENV_DISABLE_BUNDLED_SKILLS) == "1"
+def verify_eval_skills(sandbox: SandboxBase) -> None:
+ manifest_path = os.environ.get(ENV_EVAL_SKILLS_MANIFEST)
+ if manifest_path is None:
+ return
+ try:
+ manifest = Path(manifest_path).read_bytes()
+ expected = TypeAdapter(dict[str, str]).validate_json(manifest)
+ except (OSError, ValueError) as error:
+ raise ProcessTaskFatalError(
+ "Cannot read the eval skill manifest", {"sandbox_id": sandbox.id}, cause=error
+ ) from error
+ if not expected:
+ raise ProcessTaskFatalError(
+ "The eval skill snapshot is empty", {"sandbox_id": sandbox.id}, cause=ValueError("Empty skill manifest")
+ )
+ digest = hashlib.sha256(manifest).hexdigest()
+ source = Path(__file__).with_name("skill_manifest.py").read_text()
+ program = (
+ source
+ + f"\nimport os\nexpected = {expected!r}\nroots = {list(BUNDLED_SKILLS_PATHS)!r}\n"
+ + f"if os.environ.get({ENV_DISABLE_BUNDLED_SKILLS!r}) == '1':\n expected = {{}}\n"
+ + f"verify_skill_files(expected, roots)\nprint({digest!r})\n"
+ )
+ target = "/opt/posthog/eval-skill-verification.py"
+ written = sandbox.write_file(target, program.encode(), timeout_seconds=60)
+ if written.exit_code != 0:
+ raise ProcessTaskFatalError(
+ "Could not stage eval skill verification",
+ {"sandbox_id": sandbox.id},
+ cause=RuntimeError(written.stderr[-2000:]),
+ )
+ result = sandbox.execute_in_task_filesystem(f"python3 {shlex.quote(target)}", timeout_seconds=60)
+ if result.exit_code != 0 or result.stdout.strip() != digest:
+ raise ProcessTaskFatalError(
+ "Eval skill verification failed",
+ {"sandbox_id": sandbox.id, "skills_sha256": digest, "stderr": result.stderr[-2000:]},
+ cause=RuntimeError(result.stderr[-2000:] or "Unexpected skill verification output"),
+ )
+ logger.info("Eval skill snapshot verified: sha256=%s sandbox=%s", digest, sandbox.id)
+
+
def _copy_directory_contents(source: Path, destination: Path) -> None:
"""Merge-copy source into destination, skipping __pycache__."""
if not source.exists():
@@ -77,7 +128,11 @@ def populate_skills_directory(destination: Path, base_dir: Path | None = None) -
is reserved for the user's local Claude Code workspace.
"""
root = base_dir or Path(settings.BASE_DIR)
- built_skills_dir = root / BUILT_SKILLS_RELATIVE_PATH
+ built_skills_dir = (
+ Path(os.environ[ENV_LOCAL_SKILLS_HOST_PATH])
+ if os.environ.get(ENV_EVAL_SKILLS_MANIFEST)
+ else root / BUILT_SKILLS_RELATIVE_PATH
+ )
if built_skills_dir.exists() and any(f for f in built_skills_dir.iterdir() if f.name != BUILD_HASH_FILENAME):
logger.info("Using pre-built skills from %s", built_skills_dir)
_copy_directory_contents(built_skills_dir, destination)
@@ -100,6 +155,10 @@ class LocalSkillsCache:
self.dist_dir = self.output_dir / "dist" / "skills"
self.hash_file = self.dist_dir / BUILD_HASH_FILENAME
+ @property
+ def source_sha256(self) -> str:
+ return self._compute_source_hash()
+
def ensure_built(self, *, allow_stale: bool = True) -> Path:
"""Build skills if source changed; return ``dist/skills``.
diff --git a/products/tasks/backend/logic/services/skill_manifest.py b/products/tasks/backend/logic/services/skill_manifest.py
new file mode 100644
index 000000000000..11357c226ede
--- /dev/null
+++ b/products/tasks/backend/logic/services/skill_manifest.py
@@ -0,0 +1,22 @@
+from __future__ import annotations
+
+import hashlib
+from pathlib import Path
+
+
+def hash_skill_files(root: Path) -> dict[str, str]:
+ files: dict[str, str] = {}
+ for path in sorted(root.rglob("*")):
+ relative = path.relative_to(root)
+ if not path.is_file() or any(part.startswith(".") or part == "__pycache__" for part in relative.parts):
+ continue
+ files[relative.as_posix()] = hashlib.sha256(path.read_bytes()).hexdigest()
+ return files
+
+
+def verify_skill_files(expected: dict[str, str], roots: list[str]) -> None:
+ for root in roots:
+ actual = hash_skill_files(Path(root))
+ changed = sorted(name for name in expected.keys() | actual.keys() if expected.get(name) != actual.get(name))
+ if changed:
+ raise RuntimeError(f"Eval skill snapshot mismatch at {root}: {', '.join(changed[:10])}")
diff --git a/products/tasks/backend/logic/services/test_docker_sandbox.py b/products/tasks/backend/logic/services/test_docker_sandbox.py
index 659ea3b2f34c..0b4989c9e435 100644
--- a/products/tasks/backend/logic/services/test_docker_sandbox.py
+++ b/products/tasks/backend/logic/services/test_docker_sandbox.py
@@ -16,13 +16,18 @@ from products.tasks.backend.logic.services import sandbox as sandbox_module
from products.tasks.backend.logic.services.agent_server_launcher import (
AGENT_SERVER_LAUNCH_CAPABILITIES,
AGENT_SERVER_PREFLIGHT_CAPABILITY_PREFIX,
+ AgentServerPreflight,
)
from products.tasks.backend.logic.services.docker_sandbox import (
DockerSandbox,
_base_dockerfile_path,
_pinned_agent_version,
)
-from products.tasks.backend.logic.services.local_skills import ENV_DISABLE_BUNDLED_SKILLS, ENV_LOCAL_SKILLS_HOST_PATH
+from products.tasks.backend.logic.services.local_skills import (
+ ENV_DISABLE_BUNDLED_SKILLS,
+ ENV_EVAL_SKILLS_MANIFEST,
+ ENV_LOCAL_SKILLS_HOST_PATH,
+)
from products.tasks.backend.logic.services.sandbox import (
ExecutionResult,
SandboxConfig,
@@ -658,6 +663,38 @@ class TestDockerSandboxUnit:
args_str = " ".join(docker_args)
assert f"-v {tmp_path}:/tmp/workspace/repos/posthog/posthog" in args_str
+ @patch("products.tasks.backend.logic.services.docker_sandbox.subprocess.run")
+ @pytest.mark.parametrize("strict", [False, True])
+ def test_create_mounts_local_skills_for_all_agent_discovery_paths(
+ self, mock_run: MagicMock, tmp_path: Path, strict: bool
+ ) -> None:
+ mock_run.return_value = MagicMock(stdout="abc123container", returncode=0)
+ skill_dir = tmp_path / "example-skill"
+ skill_dir.mkdir()
+
+ with (
+ patch.dict(
+ os.environ,
+ {
+ ENV_LOCAL_SKILLS_HOST_PATH: str(tmp_path),
+ **(
+ {ENV_EVAL_SKILLS_MANIFEST: "manifest.json", "POSTHOG_DESKTOP_SKILLS": "local"} if strict else {}
+ ),
+ },
+ ),
+ patch.object(DockerSandbox, "_get_image", return_value="posthog-sandbox-base"),
+ ):
+ DockerSandbox.create(SandboxConfig(name="test-sandbox"))
+
+ docker_args = mock_run.call_args_list[-1][0][0]
+ for discovery_path in (
+ "/scripts/plugins/posthog/skills",
+ "/root/.claude/skills",
+ "/root/.agents/skills",
+ ):
+ expected = f"{tmp_path}:{discovery_path}:ro" if strict else f"{skill_dir}:{discovery_path}/example-skill:ro"
+ assert expected in docker_args
+
@patch("products.tasks.backend.logic.services.docker_sandbox.subprocess.run")
def test_create_skips_local_skill_mounts_when_bundled_skills_disabled(self, mock_run, tmp_path) -> None:
mock_run.return_value = MagicMock(stdout="abc123container", returncode=0)
@@ -700,6 +737,23 @@ class TestDockerSandboxUnit:
)
assert clear_index < launch_index
+ def test_start_agent_server_rejects_unverified_skills_before_reusing_a_server(self) -> None:
+ sandbox = DockerSandbox.__new__(DockerSandbox)
+ with (
+ patch.object(sandbox, "_validate_agent_server_launch"),
+ patch.object(
+ sandbox,
+ "_agent_server_preflight",
+ return_value=AgentServerPreflight(reused=True, capabilities=frozenset()),
+ ),
+ patch(
+ "products.tasks.backend.logic.services.agent_server_launcher.verify_eval_skills",
+ side_effect=RuntimeError("Snapshot mismatch"),
+ ),
+ pytest.raises(RuntimeError, match="Snapshot mismatch"),
+ ):
+ sandbox.start_agent_server(None, "task", "run")
+
def test_start_agent_server_without_domains_skips_agentsh(self):
sandbox = DockerSandbox.__new__(DockerSandbox)
sandbox._container_id = "abc123"
diff --git a/services/mcp/src/hono/cache/SkillArchiveCache.ts b/services/mcp/src/hono/cache/SkillArchiveCache.ts
index 2a065a6d283c..5d86f246a15f 100644
--- a/services/mcp/src/hono/cache/SkillArchiveCache.ts
+++ b/services/mcp/src/hono/cache/SkillArchiveCache.ts
@@ -32,6 +32,7 @@ export type SkillArchiveFetchResult =
export interface SkillArchiveCacheOptions extends SharedBlobCacheOptions {
archiveUrl?: string
+ expectedArchiveSha?: string
fetchArchive?: (url: string, etag?: string) => Promise<SkillArchiveFetchResult>
/**
* Rejects downloaded bytes before they reach Redis. A throw keeps the previous
@@ -57,10 +58,11 @@ export class SkillArchiveCache extends SharedBlobCache {
constructor(redis: RedisLike, opts: SkillArchiveCacheOptions = {}) {
const archiveUrl = opts.archiveUrl ?? DEFAULT_SKILL_ARCHIVE_URL
- const namespace =
+ const urlNamespace =
archiveUrl === DEFAULT_SKILL_ARCHIVE_URL
? NAMESPACE
: `${NAMESPACE}:${createHash('sha256').update(archiveUrl).digest('hex').slice(0, 16)}`
+ const namespace = opts.expectedArchiveSha ? `${urlNamespace}:${opts.expectedArchiveSha}` : urlNamespace
super(redis, namespace, { cacheTtlSeconds: DEFAULT_ARCHIVE_TTL_SECONDS, ...opts })
this.archiveUrl = archiveUrl
this.fetchArchive = opts.fetchArchive ?? downloadArchive
diff --git a/services/mcp/src/hono/constants.ts b/services/mcp/src/hono/constants.ts
index 82d1ad61ef49..194fd1cda1d2 100644
--- a/services/mcp/src/hono/constants.ts
+++ b/services/mcp/src/hono/constants.ts
@@ -40,6 +40,7 @@ export function getEnv(): Env {
POSTHOG_ANALYTICS_API_KEY: process.env.POSTHOG_ANALYTICS_API_KEY || undefined,
POSTHOG_ANALYTICS_HOST: process.env.POSTHOG_ANALYTICS_HOST || undefined,
POSTHOG_MCP_SKILLS_URL: process.env.POSTHOG_MCP_SKILLS_URL || undefined,
+ POSTHOG_MCP_SKILLS_SHA256: process.env.POSTHOG_MCP_SKILLS_SHA256 || undefined,
POSTHOG_MCP_LOCAL_SKILLS_URL: process.env.POSTHOG_MCP_LOCAL_SKILLS_URL || undefined,
...extras,
}
diff --git a/services/mcp/src/hono/dispatcher.ts b/services/mcp/src/hono/dispatcher.ts
index bdaee4edde91..a00318a248c4 100644
--- a/services/mcp/src/hono/dispatcher.ts
+++ b/services/mcp/src/hono/dispatcher.ts
@@ -165,7 +165,10 @@ class McpDispatcher {
this.resourceCatalog = new ResourceCatalog(env, redis)
this.stateResolver = new RequestStateResolver(catalog, redis, env)
this.instructionsBuilder = new InstructionsBuilder(GUIDELINES)
- this.skillCatalogService = new SkillCatalogService(redis, { archiveUrl: env.POSTHOG_MCP_SKILLS_URL })
+ this.skillCatalogService = new SkillCatalogService(redis, {
+ archiveUrl: env.POSTHOG_MCP_SKILLS_URL,
+ expectedArchiveSha: env.POSTHOG_MCP_SKILLS_SHA256,
+ })
this.toolExecutor = new ToolExecutor(catalog, this.instructionsBuilder, this.skillCatalogService)
this.interviewRecruitment = new InterviewRecruitment(redis)
}
diff --git a/services/mcp/src/hono/skill-catalog-service.ts b/services/mcp/src/hono/skill-catalog-service.ts
index d441115a130c..d2f05af3ca85 100644
--- a/services/mcp/src/hono/skill-catalog-service.ts
+++ b/services/mcp/src/hono/skill-catalog-service.ts
@@ -1,6 +1,7 @@
import { SkillCatalog } from '@/skills/skill-catalog'
import type { RedisLike } from './cache/RedisCache'
+import { hashBytes } from './cache/SharedBlobCache'
import { SkillArchiveCache, type SkillArchiveCacheOptions } from './cache/SkillArchiveCache'
import {
skillArchiveEventsTotal,
@@ -27,6 +28,8 @@ export interface SkillCatalogServiceOptions extends SkillArchiveCacheOptions {
type LoadPhase = 'warmup' | 'poll'
+class SkillArchiveIntegrityError extends Error {}
+
/**
* Holds the parsed product skill catalog for this pod.
*
@@ -36,6 +39,7 @@ type LoadPhase = 'warmup' | 'poll'
* bad archive keeps the last catalog that parsed.
*/
export class SkillCatalogService {
+ private readonly expectedArchiveSha: string | undefined
private readonly archiveCache: SkillArchiveCache
private readonly pollIntervalMs: number
private readonly pollJitterMs: number
@@ -49,12 +53,20 @@ export class SkillCatalogService {
private polling = false
constructor(redis: RedisLike, opts: SkillCatalogServiceOptions = {}) {
- const { pollIntervalMs, pollJitterMs, warmupTimeoutMs, warmupRetryMs, ...cacheOptions } = opts
+ const { expectedArchiveSha, pollIntervalMs, pollJitterMs, warmupTimeoutMs, warmupRetryMs, ...cacheOptions } =
+ opts
+ this.expectedArchiveSha = expectedArchiveSha
+ const validateArchive =
+ cacheOptions.validateArchive ?? ((bytes: Uint8Array) => void SkillCatalog.fromZip(bytes))
this.archiveCache = new SkillArchiveCache(redis, {
...cacheOptions,
+ expectedArchiveSha,
// Parse before the bytes can reach Redis: a release that does not parse
// never becomes the fleet's shared copy.
- validateArchive: cacheOptions.validateArchive ?? ((bytes) => void SkillCatalog.fromZip(bytes)),
+ validateArchive: (bytes) => {
+ this.verifyArchive(bytes)
+ validateArchive(bytes)
+ },
})
this.pollIntervalMs = pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS
this.pollJitterMs = pollJitterMs ?? DEFAULT_POLL_JITTER_MS
@@ -81,7 +93,13 @@ export class SkillCatalogService {
return
} catch (error) {
skillArchiveEventsTotal.inc({ phase: 'warmup', result: 'error' })
+ if (error instanceof SkillArchiveIntegrityError) {
+ throw error
+ }
if (Date.now() + this.warmupRetryMs >= deadline) {
+ if (this.expectedArchiveSha) {
+ throw error
+ }
console.error('[SkillCatalogService] warmup failed; starting without product skills:', error)
return
}
@@ -151,7 +169,14 @@ export class SkillCatalogService {
this.install(changed.bytes, changed.sha, 'poll')
}
+ private verifyArchive(bytes: Uint8Array): void {
+ if (this.expectedArchiveSha && hashBytes(bytes) !== this.expectedArchiveSha) {
+ throw new SkillArchiveIntegrityError('Skill archive does not match the expected SHA-256')
+ }
+ }
+
private install(bytes: Uint8Array, sha: string, phase: LoadPhase): void {
+ this.verifyArchive(bytes)
if (sha === this.archiveSha) {
return
}
diff --git a/services/mcp/src/tools/types.ts b/services/mcp/src/tools/types.ts
index 33f7d5735c30..b02fd8dd8fdf 100644
--- a/services/mcp/src/tools/types.ts
+++ b/services/mcp/src/tools/types.ts
@@ -120,6 +120,7 @@ export type Env = {
POSTHOG_ANALYTICS_HOST: string | undefined
/** Override the published product skills archive, primarily for local development. */
POSTHOG_MCP_SKILLS_URL?: string | undefined
+ POSTHOG_MCP_SKILLS_SHA256?: string | undefined
POSTHOG_MCP_LOCAL_SKILLS_URL?: string | undefined
}
diff --git a/services/mcp/tests/unit/skill-archive-cache.test.ts b/services/mcp/tests/unit/skill-archive-cache.test.ts
index 889eda787266..0fa17c740fed 100644
--- a/services/mcp/tests/unit/skill-archive-cache.test.ts
+++ b/services/mcp/tests/unit/skill-archive-cache.test.ts
@@ -2,6 +2,7 @@ import { strToU8, zipSync } from 'fflate'
import { describe, expect, it, vi } from 'vitest'
import type { RedisLike } from '@/hono/cache/RedisCache'
+import { hashBytes } from '@/hono/cache/SharedBlobCache'
import { SkillArchiveCache, type SkillArchiveFetchResult } from '@/hono/cache/SkillArchiveCache'
import { skillArchiveLastValidatedTimestampSeconds } from '@/hono/metrics'
import { SkillCatalogService } from '@/hono/skill-catalog-service'
@@ -341,6 +342,37 @@ describe('SkillCatalogService', () => {
return vi.spyOn(console, 'error').mockImplementation(() => undefined)
}
+ it.each([true, false])('checks the pinned archive before startup (matching=%s)', async (matching) => {
+ const bytes = makeArchive()
+ const service = new SkillCatalogService(makeRedis(), {
+ fetchArchive: async () => downloaded(bytes),
+ expectedArchiveSha: matching ? hashBytes(bytes) : '0'.repeat(64),
+ })
+ if (matching) {
+ await service.warmup()
+ expect(service.getCatalog()?.size).toBe(1)
+ } else {
+ await expect(service.warmup()).rejects.toThrow('does not match the expected SHA-256')
+ expect(service.getCatalog()).toBeUndefined()
+ }
+ })
+
+ it('loads a new pin at the same URL without reusing an older cached archive', async () => {
+ const redis = makeRedis()
+ const firstBytes = makeArchive('first')
+ const secondBytes = makeArchive('second')
+ const fetchArchive = vi.fn(async () => downloaded(firstBytes))
+ const first = new SkillCatalogService(redis, { fetchArchive, expectedArchiveSha: hashBytes(firstBytes) })
+ await first.warmup()
+ fetchArchive.mockResolvedValue(downloaded(secondBytes))
+ const second = new SkillCatalogService(redis, { fetchArchive, expectedArchiveSha: hashBytes(secondBytes) })
+
+ await second.warmup()
+
+ expect(second.getCatalog()?.listNames()).toEqual(['second'])
+ expect(fetchArchive).toHaveBeenCalledTimes(2)
+ })
+
it('reports upstream validation time across restarts, unchanged releases and refresh failures', async () => {
vi.useFakeTimers()
const timestamp = vi.spyOn(skillArchiveLastValidatedTimestampSeconds, 'set')
@@ -395,6 +427,39 @@ describe('SkillCatalogService', () => {
consoleError.mockRestore()
})
+ it.each([true, false])(
+ 'retries pinned archive fetch failures within the startup budget (recovers=%s)',
+ async (recovers) => {
+ vi.useFakeTimers()
+ const consoleError = quietConsole()
+ try {
+ const bytes = makeArchive()
+ let attempts = 0
+ const fetchArchive = vi.fn(async () => {
+ attempts += 1
+ if (attempts === 1 || !recovers) {
+ throw new Error('offline')
+ }
+ return downloaded(bytes)
+ })
+ const service = new SkillCatalogService(makeRedis(), {
+ fetchArchive,
+ expectedArchiveSha: hashBytes(bytes),
+ warmupTimeoutMs: 50,
+ warmupRetryMs: 10,
+ })
+ const outcome = service.warmup().catch((error: unknown) => error)
+ await vi.runAllTimersAsync()
+ expect(await outcome).toEqual(recovers ? undefined : new Error('offline'))
+ expect(fetchArchive.mock.calls.length).toBeGreaterThan(1)
+ expect(service.getCatalog()?.size).toBe(recovers ? 1 : undefined)
+ } finally {
+ consoleError.mockRestore()
+ vi.useRealTimers()
+ }
+ }
+ )
+
it('loads the catalog at warmup and adopts a newer archive another pod published', async () => {
const redis = makeRedis()
const first = new SkillCatalogService(redis, {