Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(signals): support scout runs in the eval harness (#121949)

Georgiy Tarasovfd9f7230e11f
diff --git a/products/posthog_ai/eval_harness/README.md b/products/posthog_ai/eval_harness/README.md
index 5fd455f63bff..aa9ee133a25a 100644
--- a/products/posthog_ai/eval_harness/README.md
+++ b/products/posthog_ai/eval_harness/README.md
@@ -18,6 +18,12 @@ Two other docs sit next to this one:
- [`../evals/AGENTS.md`](../evals/AGENTS.md) is the Hedgebox dataset reference. Read it before writing eval cases: your `expected` values and scorers must match that taxonomy exactly.
- [`harness/AGENTS.md`](harness/AGENTS.md) lists the invariants to preserve when changing the harness itself.
+## Custom scout runners
+
+Product suites can supply `case_runner=` to `SandboxedEval` with the `SandboxedCaseRunner` protocol. The shared harness retains isolation, setup hooks, concurrency slots, timeout handling, logs, and scoring; the runner owns its task workflow cleanup and provider registration. Returning an `AgentArtifacts` subclass preserves product-specific fields in local artifacts and scorer input. Scout runners collect structured closeout fields from persisted run metadata, including when a visible report takes precedence over the agent's claimed outcome.
+
+Test cases should keep the default `SandboxedEvalCase.inherit_demo_data=True` to inherit Hedgebox data. Setting it to `False` is an explicit opt-in only for cases that intentionally need an empty isolated project, such as testing abstention with insufficient data. The setup hook must seed any data the case needs.
+
## Running
```bash
@@ -128,6 +134,7 @@ git clone [email protected]:PostHog/ai-gateway.git ~/Development/ai-gateway
```
The gateway runs on `:18080` with its own Postgres database and Valkey DB index in the clone's compose containers, so it does not touch a dev-stack gateway's ledger or blobs.
+Sandboxed agents and server-side tool judges, including the scout report safety judge, both use this isolated gateway.
It uses `LLM_GATEWAY_ANTHROPIC_API_KEY` and `LLM_GATEWAY_OPENAI_API_KEY` as its provider keys.
The harness adds test credits to the master demo team in this separate ledger. Scoped tokens require a positive balance and cannot use an overspend allowance.
Each run gets a capped `phe_` token minted with a key on that team, so spend is per run and never reaches a real wallet. The harness refreshes the mint key during long runs. It retries temporary database or Redis errors until the credential expires. Retry delays increase to at most one minute.
diff --git a/products/posthog_ai/eval_harness/base.py b/products/posthog_ai/eval_harness/base.py
index 93ca9a9518a9..e7d77f2bbb23 100644
--- a/products/posthog_ai/eval_harness/base.py
+++ b/products/posthog_ai/eval_harness/base.py
@@ -42,7 +42,7 @@ from .harness.trial_stats import trial_stats
from .log_parser import describe_tool_use
from .log_sink import append_case_scores, build_case_dir, write_case_logs
from .offline_results import OfflineEvalPublisher, OfflineEvalSettings, OfflineEvalSuite, OfflineUploadError
-from .runner import AgentNeverRanError, EvalCaseResult, agent_never_ran, run_eval_case
+from .runner import AgentNeverRanError, EvalCaseResult, SandboxedCaseRunner, agent_never_ran, run_eval_case
from .scorers import ExitCodeZero, wrap_scorers
from .trace_events import RunMetadata, emit_trace_events, emit_trace_root
@@ -623,6 +623,7 @@ class _SandboxedEvalRun(_BaseEvalRun):
no_send_logs: bool,
offline_suite: OfflineEvalSuite | None = None,
send_to_posthog: bool | None = None,
+ case_runner: SandboxedCaseRunner | None = None,
) -> None:
if any(isinstance(scorer, ExitCodeZero) for scorer in scorers):
raise ValueError("ExitCodeZero is added by the sandboxed eval harness; remove it from scorers")
@@ -646,6 +647,7 @@ class _SandboxedEvalRun(_BaseEvalRun):
self._provider_strategy = ctx.provider_strategy
self._demo_data = ctx.demo_data
self._sandbox_slots = ctx.sandbox_slots
+ self._case_runner = case_runner
def _case_input(self, case: BaseEvalCase) -> dict[str, Any]:
assert isinstance(case, SandboxedEvalCase)
@@ -679,6 +681,7 @@ class _SandboxedEvalRun(_BaseEvalRun):
self.ctx.skill_delivery == "exec"
or bool(original_case and original_case.disable_bundled_skills)
),
+ inherit_demo_data=original_case.inherit_demo_data if original_case is not None else True,
),
)
if original_case is not None and original_case.interaction_origin:
@@ -748,7 +751,7 @@ class _SandboxedEvalRun(_BaseEvalRun):
with suppress(Exception):
await self._set_case_phase(hooks, "agent")
result = await asyncio.wait_for(
- run_eval_case(
+ (self._case_runner or run_eval_case)(
eval_case,
prepared.context,
provider=self._provider_strategy,
@@ -896,6 +899,7 @@ async def SandboxedEval(
no_send_logs: bool = True,
offline_suite: OfflineEvalSuite | None = None,
send_to_posthog: bool | None = None,
+ case_runner: SandboxedCaseRunner | None = None,
) -> ExperimentResult:
"""Run a sandboxed agent evaluation suite via Braintrust.
@@ -921,6 +925,7 @@ async def SandboxedEval(
no_send_logs=no_send_logs,
offline_suite=offline_suite,
send_to_posthog=send_to_posthog,
+ case_runner=case_runner,
)
return await run.run()
diff --git a/products/posthog_ai/eval_harness/config.py b/products/posthog_ai/eval_harness/config.py
index 6777c75ab3e6..949079e5a107 100644
--- a/products/posthog_ai/eval_harness/config.py
+++ b/products/posthog_ai/eval_harness/config.py
@@ -52,6 +52,12 @@ class SandboxedEvalCase(BaseEvalCase):
repo_fixture: str = ""
"""Name of the repo fixture (informational, for tracking)."""
+ inherit_demo_data: bool = True
+ """Keep True for normal cases. Opt into False only when the case intentionally requires an empty project.
+
+ For example, insufficient-data cases must avoid inherited Hedgebox data and seed only their required sample.
+ """
+
disable_bundled_skills: bool = False
"""Remove skills baked into the sandbox image for this case.
diff --git a/products/posthog_ai/eval_harness/data_setup.py b/products/posthog_ai/eval_harness/data_setup.py
index 7cbd03cc9180..a11b1e267243 100644
--- a/products/posthog_ai/eval_harness/data_setup.py
+++ b/products/posthog_ai/eval_harness/data_setup.py
@@ -5,6 +5,7 @@ from __future__ import annotations
import uuid
import logging
import datetime
+from typing import TYPE_CHECKING
from django.conf import settings
from django.db import transaction
@@ -22,6 +23,8 @@ from products.demo.backend.facade.api import HedgeboxMatrix, MatrixManager, infe
from products.posthog_ai.backend.models.assistant import CoreMemory
from products.product_analytics.backend.facade.models import Insight
+if TYPE_CHECKING:
+ from products.posthog_ai.eval_harness.harness.django_env import NullDbBlocker
logger = logging.getLogger(__name__)
EVAL_SEED = "b1ef3c66-5f43-488a-98be-6b46d92fbcef"
@@ -166,6 +169,23 @@ def ensure_master_demo_team(django_db_blocker) -> int:
return team.id
+def create_empty_eval_project(django_db_blocker: NullDbBlocker, *, label: str) -> tuple[Team, User]:
+ suffix = uuid.uuid4().hex[:8]
+ with django_db_blocker.unblock(), transaction.atomic():
+ org = Organization.objects.create(name=f"Hedgebox ({label}-{suffix})")
+ user = User.objects.create_and_join(
+ org,
+ f"eval-{label}-{suffix}@posthog.test",
+ None,
+ EVAL_USER_FULL_NAME,
+ OrganizationMembership.Level.ADMIN,
+ theme_mode="system",
+ role_at_organization="engineering",
+ )
+ team = Team.objects.create(organization=org, is_demo=True)
+ return team, user
+
+
def copy_demo_data_to_new_team(
master_team_id: int,
django_db_blocker,
diff --git a/products/posthog_ai/eval_harness/harness/demo_data.py b/products/posthog_ai/eval_harness/harness/demo_data.py
index 34de0358ebd2..d2c937c42dfa 100644
--- a/products/posthog_ai/eval_harness/harness/demo_data.py
+++ b/products/posthog_ai/eval_harness/harness/demo_data.py
@@ -11,6 +11,7 @@ from posthog.clickhouse.client import sync_execute
from products.posthog_ai.eval_harness.data_setup import (
copy_demo_data_to_new_team,
create_core_memory,
+ create_empty_eval_project,
ensure_master_demo_team,
)
from products.tasks.backend.facade.agents import CustomPromptSandboxContext, create_skill_isolation_environment
@@ -52,8 +53,13 @@ class SandboxedDemoData:
self.reasoning_effort = reasoning_effort
self.sandbox_timeout_seconds = sandbox_timeout_seconds
- def make_context(self, case_label: str, *, disable_bundled_skills: bool = False) -> CustomPromptSandboxContext:
- _, team, user = copy_demo_data_to_new_team(self.master_team_id, self._django_db_blocker, label=case_label)
+ def make_context(
+ self, case_label: str, *, disable_bundled_skills: bool = False, inherit_demo_data: bool = True
+ ) -> CustomPromptSandboxContext:
+ if inherit_demo_data:
+ _, team, user = copy_demo_data_to_new_team(self.master_team_id, self._django_db_blocker, label=case_label)
+ else:
+ team, user = create_empty_eval_project(self._django_db_blocker, label=case_label)
create_core_memory(team, self._django_db_blocker)
sandbox_environment_id: str | None = None
if disable_bundled_skills:
diff --git a/products/posthog_ai/eval_harness/harness/lifecycle.py b/products/posthog_ai/eval_harness/harness/lifecycle.py
index 03d0f4d00899..bc5c898798f5 100644
--- a/products/posthog_ai/eval_harness/harness/lifecycle.py
+++ b/products/posthog_ai/eval_harness/harness/lifecycle.py
@@ -50,7 +50,7 @@ from .env_preflight import validate_eval_env
from .kernel_sandboxes import reclaim_kernels
from .live_server import EvalLiveServer
from .offline_delivery import OfflineEvalDelivery
-from .ports import AI_GATEWAY_REDIS_URL, DJANGO_LIVE_PORT
+from .ports import AI_GATEWAY_PORT, AI_GATEWAY_REDIS_URL, DJANGO_LIVE_PORT
from .providers import PreflightError, SandboxProviderStrategy, build_provider
from .reporting import ProgressReporter, SuiteRunResult
from .requirements import Infra, infra_union
@@ -497,10 +497,13 @@ class SandboxedEvalHarness:
# The harness never starts run_task_workflow_dispatcher, so an outbox dispatch would
# leave every case parked in QUEUED until the case's poll budget runs out.
TASKS_WORKFLOW_DISPATCH_OUTBOX_ENABLED=False,
- # Only the eval product routes to the Go gateway. The worker mints each
+ # Chat and scout evals retain their production product attribution. The worker mints each
# run a capped scoped token with this key, as it does in production.
- SANDBOX_AI_GATEWAY_PRODUCTS=EVAL_AI_PRODUCT,
+ SANDBOX_AI_GATEWAY_PRODUCTS=f"{EVAL_AI_PRODUCT},signals_scout",
SANDBOX_AI_GATEWAY_MINT_KEY=self._ai_gateway_mint_key,
+ # Tool-side judges share the eval ledger with the sandboxed agent.
+ AI_GATEWAY_URL=f"http://localhost:{AI_GATEWAY_PORT}/v1",
+ AI_GATEWAY_API_KEY=self._ai_gateway_mint_key,
**self.provider.settings_overrides(),
)
diff --git a/products/posthog_ai/eval_harness/runner.py b/products/posthog_ai/eval_harness/runner.py
index bdd8b85a0a90..7eee8315fde3 100644
--- a/products/posthog_ai/eval_harness/runner.py
+++ b/products/posthog_ai/eval_harness/runner.py
@@ -7,7 +7,7 @@ import uuid
import asyncio
import logging
from collections.abc import Awaitable, Callable
-from typing import TYPE_CHECKING
+from typing import TYPE_CHECKING, Protocol
from temporalio.client import WorkflowFailureError
@@ -125,6 +125,19 @@ class EvalCaseResult:
lines could not be split (malformed lines); scorers self-skip on it."""
+class SandboxedCaseRunner(Protocol):
+ async def __call__(
+ self,
+ case: SandboxedEvalCase,
+ context: CustomPromptSandboxContext,
+ *,
+ provider: SandboxProviderStrategy,
+ on_cleanup: Callable[[], Awaitable[None]] | None = None,
+ timeout_seconds: float | None = None,
+ on_poll_timeout: Callable[[TurnPollTimeout], None] | None = None,
+ ) -> EvalCaseResult: ...
+
+
async def _report_cleanup(on_cleanup: Callable[[], Awaitable[None]] | None) -> None:
if on_cleanup is not None:
try:
diff --git a/products/posthog_ai/eval_harness/test/test_sandboxed_harness_output.py b/products/posthog_ai/eval_harness/test/test_sandboxed_harness_output.py
index fb1f17bef9d8..93fcbb5ffc78 100644
--- a/products/posthog_ai/eval_harness/test/test_sandboxed_harness_output.py
+++ b/products/posthog_ai/eval_harness/test/test_sandboxed_harness_output.py
@@ -640,7 +640,9 @@ async def test_sandboxed_eval_skill_delivery_sets_the_agent_origin(
ctx.demo_data.make_context.return_value = sandbox_context
ctx.provider_strategy.keeps_sandboxes.return_value = False
setup = MagicMock(return_value={"seeded": True})
- case = SandboxedEvalCase(name="ordinary-case", prompt="Run the task", interaction_origin=origin, setup=setup)
+ case = SandboxedEvalCase(
+ name="ordinary-case", prompt="Run the task", interaction_origin=origin, setup=setup, inherit_demo_data=False
+ )
run = base._SandboxedEvalRun(
experiment_name="skill-delivery-test",
cases=[case] if has_original else [],
@@ -661,6 +663,7 @@ async def test_sandboxed_eval_skill_delivery_sets_the_agent_origin(
dict(base.PromptService.build()) if expected_origin == "posthog_ai" else None
)
assert ctx.demo_data.make_context.call_args.kwargs["disable_bundled_skills"] == (delivery == "exec")
+ assert ctx.demo_data.make_context.call_args.kwargs["inherit_demo_data"] == (not has_original)
if has_original:
assert setup.call_args.args[0].interaction_origin == expected_origin
assert seed == {"seeded": True}
@@ -871,15 +874,23 @@ async def test_case_setup_retries_use_fresh_data_and_report_one_outcome(
@pytest.mark.asyncio
@pytest.mark.parametrize("agent_error", [None, RuntimeError("agent failed"), asyncio.CancelledError()])
@pytest.mark.parametrize("reporting_phase", ["agent", "cleanup"])
[email protected]("custom_runner", [False, True])
async def test_kernel_cleanup_preserves_case_outcome_when_progress_fails(
- monkeypatch: pytest.MonkeyPatch, tmp_path: Path, agent_error: BaseException | None, reporting_phase: str
+ monkeypatch: pytest.MonkeyPatch,
+ tmp_path: Path,
+ agent_error: BaseException | None,
+ reporting_phase: str,
+ custom_runner: bool,
) -> None:
monkeypatch.setattr(base, "build_case_dir", MagicMock(return_value=tmp_path))
ctx = MagicMock(posthog_client=None, case_filter=None, sandbox_slots=asyncio.Semaphore(1))
ctx.per_case_timeout_seconds = 30
ctx.provider_strategy.keeps_sandboxes.return_value = False
case = SandboxedEvalCase(name="case", prompt="Run the task")
- run = base._SandboxedEvalRun("cleanup", [case], [], ctx, is_public=False, no_send_logs=True)
+ launch = AsyncMock()
+ run = base._SandboxedEvalRun(
+ "cleanup", [case], [], ctx, is_public=False, no_send_logs=True, case_runner=launch if custom_runner else None
+ )
context = CustomPromptSandboxContext(team_id=1, user_id=2)
monkeypatch.setattr(
run, "_prepare_case", AsyncMock(return_value=base._SandboxedCaseSetup(context=context, seed={}))
@@ -893,7 +904,11 @@ async def test_kernel_cleanup_preserves_case_outcome_when_progress_fails(
hooks = NullCaseHooks()
hooks.metadata["trial_id"] = "trial"
result = base.EvalCaseResult(artifacts=AgentArtifacts(exit_code=0))
- monkeypatch.setattr(base, "run_eval_case", AsyncMock(return_value=result, side_effect=agent_error))
+ launch.return_value = result
+ launch.side_effect = agent_error
+ monkeypatch.setattr(
+ base, "run_eval_case", AsyncMock(side_effect=AssertionError("Default runner used")) if custom_runner else launch
+ )
cleanup = AsyncMock()
monkeypatch.setattr(base, "reclaim_kernels", cleanup)
@@ -913,6 +928,9 @@ async def test_kernel_cleanup_preserves_case_outcome_when_progress_fails(
async def test_sandboxed_trials_keep_separate_logs_and_trace_roots(
monkeypatch: pytest.MonkeyPatch, tmp_path: Path
) -> None:
+ class ProductArtifacts(AgentArtifacts):
+ report_id: str
+
monkeypatch.delenv("EXPORT_EVAL_RESULTS", raising=False)
monkeypatch.setattr(base, "build_case_dir", lambda *_: tmp_path)
ctx = MagicMock(posthog_client=None, reporter=ProgressReporter(total_suites=1))
@@ -940,9 +958,14 @@ async def test_sandboxed_trials_keep_separate_logs_and_trace_roots(
output = await run._post_process(
SandboxedEvalCase(name="same-case", prompt="Run the task"),
hooks,
- base.EvalCaseResult(artifacts=AgentArtifacts(exit_code=0), raw_log=raw_log, trace_id=f"trace-{index}"),
+ base.EvalCaseResult(
+ artifacts=ProductArtifacts(exit_code=0, report_id=f"report-{index}"),
+ raw_log=raw_log,
+ trace_id=f"trace-{index}",
+ ),
{},
)
+ assert output["report_id"] == f"report-{index}"
results.append(
CaseResult(input={"name": "same-case"}, output=output, scores={"score": index / 2}, metadata=hooks.metadata)
)
@@ -953,6 +976,8 @@ async def test_sandboxed_trials_keep_separate_logs_and_trace_roots(
)
)
for index in range(3):
+ artifacts = json.loads((tmp_path / f"same-case.trial-trial-{index}.artifacts.json").read_text())
+ assert artifacts["report_id"] == f"report-{index}"
summary = (tmp_path / f"same-case.trial-trial-{index}.summary.txt").read_text()
assert f"trace_id: trace-{index}" in summary
assert json.loads(summary.split("=== scores ===\n")[1]) == {"score": index / 2}
diff --git a/products/signals/backend/test/test_first_report_evals.py b/products/signals/backend/test/test_first_report_evals.py
new file mode 100644
index 000000000000..e5936853eb20
--- /dev/null
+++ b/products/signals/backend/test/test_first_report_evals.py
@@ -0,0 +1,170 @@
+from __future__ import annotations
+
+import json
+import asyncio
+from datetime import UTC, datetime, timedelta
+from types import SimpleNamespace
+
+from posthog.test.base import BaseTest
+from unittest import IsolatedAsyncioTestCase
+from unittest.mock import AsyncMock, MagicMock, patch
+
+from parameterized import parameterized
+
+from products.posthog_ai.eval_harness.config import SandboxedEvalCase
+from products.posthog_ai.eval_harness.runner import WorkflowCleanupError
+from products.signals.backend.constants import FIRST_REPORT_SCOUT
+from products.signals.backend.first_report_scout_types import FirstReportScoutResult
+from products.signals.backend.models import SignalScoutRun
+from products.signals.evals import scout_runner
+from products.tasks.backend.facade.agents import CustomPromptSandboxContext
+from products.tasks.backend.models import Task, TaskRun
+
+
+class TestScoutEvalCleanup(IsolatedAsyncioTestCase):
+ @parameterized.expand([("agent_message",), ("agent_message_chunk",)])
+ def test_progress_time_uses_first_text_not_prior_tool_call(self, message_kind: str) -> None:
+ started = datetime(2026, 1, 1, tzinfo=UTC)
+ log = "invalid JSON\n" + "\n".join(
+ json.dumps(
+ {
+ "timestamp": (started + timedelta(seconds=seconds)).isoformat(),
+ "notification": {
+ "params": {
+ "update": {"sessionUpdate": kind, "content": {"type": "text", "text": "Investigating"}}
+ }
+ },
+ }
+ )
+ for seconds, kind in [(10, "tool_call"), (20, message_kind), (30, message_kind)]
+ )
+ self.assertEqual(scout_runner.first_progress_seconds(log, started), 20)
+
+ @parameterized.expand(
+ [
+ (None, True),
+ (RuntimeError("Scout failed"), False),
+ (asyncio.CancelledError(), False),
+ (None, False),
+ (None, True, "connect"),
+ (None, True, "finish"),
+ (None, True, None, True),
+ ]
+ )
+ async def test_scout_reclaims_sandbox_and_preserves_failure(
+ self,
+ error: BaseException | None,
+ cleanup_confirmed: bool,
+ cancel_during: str | None = None,
+ report_created: bool = False,
+ ) -> None:
+ task = SimpleNamespace(id="run", task_id="task", workflow_id="workflow")
+ client = MagicMock()
+ provider = MagicMock()
+ result = SimpleNamespace(status="completed", run_id="scout", last_message=None)
+ cleanup_started = asyncio.Event()
+ release_cleanup = asyncio.Event()
+ workflow_finished = asyncio.Event()
+
+ async def connect() -> MagicMock:
+ if cancel_during == "connect":
+ cleanup_started.set()
+ await release_cleanup.wait()
+ return client
+
+ async def finish(*args: object, **kwargs: object) -> bool:
+ if cancel_during == "finish":
+ cleanup_started.set()
+ await release_cleanup.wait()
+ workflow_finished.set()
+ return cleanup_confirmed
+
+ with (
+ patch.object(scout_runner, "prepare_scout", return_value="Server inventory"),
+ patch.object(scout_runner, "latest_scout_task", return_value=task),
+ patch.object(scout_runner, "arun_signals_scout", AsyncMock(return_value=result, side_effect=error)),
+ patch.object(scout_runner.tasks_api, "read_task_run_conversation_logs", return_value=""),
+ patch.object(
+ scout_runner,
+ "first_report_state",
+ return_value={"status": "report_created" if report_created else "insufficient_data"},
+ ),
+ patch.object(
+ scout_runner,
+ "collect_closeout",
+ return_value=FirstReportScoutResult(
+ outcome="insufficient_data",
+ missing_data=["More cohorts"],
+ checked_products=["product_analytics"],
+ ),
+ ),
+ patch.object(
+ scout_runner,
+ "collect_reports",
+ return_value=[
+ scout_runner.EvalReport(
+ id="report",
+ title="Upload failure",
+ summary="Check the upload handler.",
+ status="ready",
+ visible_after_seconds=20,
+ )
+ ]
+ if report_created
+ else [],
+ ),
+ patch.object(scout_runner, "async_connect", connect),
+ patch.object(scout_runner, "_finish_workflow", finish),
+ ):
+ invocation = scout_runner.run_first_report_scout(
+ SandboxedEvalCase(name="sparse", prompt="Run the scout"),
+ CustomPromptSandboxContext(team_id=1, user_id=2),
+ provider=provider,
+ on_cleanup=AsyncMock(side_effect=BrokenPipeError("Reporter disconnected")),
+ )
+ if cancel_during is not None:
+ running = asyncio.create_task(invocation)
+ await cleanup_started.wait()
+ provider.cleanup_case.assert_not_called()
+ running.cancel()
+ release_cleanup.set()
+ with self.assertRaises(asyncio.CancelledError):
+ await running
+ elif error is not None:
+ with self.assertRaises(type(error)) as raised:
+ await invocation
+ self.assertIs(raised.exception, error)
+ elif not cleanup_confirmed:
+ with self.assertRaises(WorkflowCleanupError):
+ await invocation
+ else:
+ completed = await invocation
+ self.assertEqual(completed.artifacts.model_dump()["missing_data"], ["More cohorts"])
+ self.assertEqual(completed.artifacts.model_dump()["checked_products"], ["product_analytics"])
+ self.assertEqual(
+ completed.artifacts.model_dump()["outcome"],
+ "report_created" if report_created else "insufficient_data",
+ )
+ self.assertTrue(workflow_finished.is_set())
+ provider.cleanup_case.assert_called_once_with("task")
+
+
+class TestScoutEvalCloseout(BaseTest):
+ @parameterized.expand([("report_created",), ("insufficient_data",), (None,)])
+ def test_collects_persisted_closeout(self, outcome: str | None) -> None:
+ task = Task.objects.create(team=self.team, title="First investigation")
+ task_run = TaskRun.objects.create(team=self.team, task=task, status="completed")
+ closeout = {"outcome": outcome, "checked_products": ["product_analytics"], "missing_data": ["More cohorts"]}
+ run = SignalScoutRun.objects.for_team(self.team.id).create(
+ team=self.team,
+ task_run=task_run,
+ skill_name=FIRST_REPORT_SCOUT,
+ skill_version=1,
+ metadata={"first_report_result": closeout} if outcome else {},
+ )
+ result = scout_runner.collect_closeout(self.team.id, str(run.id))
+ if outcome is None:
+ assert result is None
+ else:
+ assert result is not None
+ assert result.model_dump() == closeout
diff --git a/products/signals/evals/scout_runner.py b/products/signals/evals/scout_runner.py
new file mode 100644
index 000000000000..6542c4f84023
--- /dev/null
+++ b/products/signals/evals/scout_runner.py
@@ -0,0 +1,271 @@
+from __future__ import annotations
+
+import json
+import time
+import asyncio
+import logging
+from collections.abc import Awaitable, Callable
+from datetime import datetime
+
+from django.utils import timezone
+
+from pydantic import BaseModel, Field
+
+from posthog.models import Team, User
+from posthog.temporal.common.client import async_connect
+
+from products.posthog_ai.eval_harness.config import AgentArtifacts, SandboxedEvalCase
+from products.posthog_ai.eval_harness.harness.providers import SandboxProviderStrategy
+from products.posthog_ai.eval_harness.runner import (
+ EvalCaseResult,
+ WorkflowCleanupError,
+ _finish_workflow,
+ _parse_artifacts_from_log,
+ _report_cleanup,
+)
+from products.signals.backend.agent_runtime import AgentRuntime
+from products.signals.backend.constants import FIRST_REPORT_SCOUT
+from products.signals.backend.first_report_scout import build_first_report_context, first_report_state
+from products.signals.backend.first_report_scout_types import FirstReportScoutResult
+from products.signals.backend.models import (
+ SignalReport,
+ SignalScoutConfig,
+ SignalScoutRun,
+ SignalSourceConfig,
+ SignalTeamConfig,
+)
+from products.signals.backend.scout_harness.limits import TRIGGERED_BY_ONBOARDING
+from products.signals.backend.scout_harness.runner import arun_signals_scout
+from products.signals.backend.scout_harness.tools.profile import compute_project_profile
+from products.tasks.backend.facade import api as tasks_api
+from products.tasks.backend.facade.agents import CustomPromptSandboxContext, TurnPollTimeout
+from products.tasks.backend.facade.contracts import TaskRunDTO
+
+logger = logging.getLogger(__name__)
+
+
+class EvalReport(BaseModel):
+ id: str
+ title: str
+ summary: str
+ status: str
+ visible_after_seconds: float | None
+
+
+class ScoutArtifacts(AgentArtifacts):
+ reports: list[EvalReport] = Field(default_factory=list)
+ last_message: str | None = None
+ outcome: str = "incomplete"
+ missing_data: list[str] = Field(default_factory=list)
+ checked_products: list[str] = Field(default_factory=list)
+ context_seconds: float = 0
+ first_progress_seconds: float | None = None
+ first_report_seconds: float | None = None
+ scout_run_id: str | None = None
+
+
+def prepare_scout(context: CustomPromptSandboxContext) -> str:
+ team = Team.objects.select_related("organization").get(id=context.team_id)
+ user = User.objects.get(id=context.user_id)
+ team.organization.is_ai_data_processing_approved = True
+ team.organization.save(update_fields=["is_ai_data_processing_approved"])
+ SignalTeamConfig.objects.update_or_create(team_id=team.id, defaults={"autostart_enabled": False})
+ SignalScoutConfig.objects.for_team(team.id).create(
+ team_id=team.id,
+ skill_name=FIRST_REPORT_SCOUT,
+ enabled=False,
+ emit=True,
+ created_by=user,
+ enabled_by=user,
+ write_scopes=[],
+ )
+ SignalSourceConfig.objects.get_or_create(
+ team_id=team.id,
+ source_product="signals_scout",
+ source_type="cross_source_issue",
+ defaults={"enabled": True},
+ )
+ compute_project_profile(team=team, force=True, defer_surfaces=True)
+ inventory = build_first_report_context(team, user)
+ if not inventory.has_data():
+ raise RuntimeError("A scout eval fixture must have queryable data before launching the agent")
+ return inventory.prompt_context()
+
+
+def latest_scout_task(team_id: int) -> TaskRunDTO | None:
+ run = SignalScoutRun.objects.for_team(team_id).filter(skill_name=FIRST_REPORT_SCOUT).order_by("-created_at").first()
+ return tasks_api.get_task_run(run.task_run_id, team_id=team_id) if run is not None else None
+
+
+def collect_closeout(team_id: int, run_id: str | None) -> FirstReportScoutResult | None:
+ if run_id is None:
+ return None
+ metadata = (
+ SignalScoutRun.objects.for_team(team_id).filter(id=run_id).values_list("metadata", flat=True).first() or {}
+ )
+ result = metadata.get("first_report_result")
+ return FirstReportScoutResult.model_validate(result) if result else None
+
+
+def collect_reports(team_id: int, started_at: datetime) -> list[EvalReport]:
+ return [
+ EvalReport(
+ id=str(report.id),
+ title=report.title or "",
+ summary=report.summary or "",
+ status=report.status,
+ visible_after_seconds=(report.first_visible_at - started_at).total_seconds()
+ if report.first_visible_at
+ else None,
+ )
+ for report in SignalReport.objects.filter(team_id=team_id, created_at__gte=started_at)
+ ]
+
+
+def first_progress_seconds(raw_log: str, started_at: datetime) -> float | None:
+ for line in raw_log.splitlines():
+ try:
+ entry = json.loads(line)
+ update = entry.get("notification", {}).get("params", {}).get("update", {})
+ content = update.get("content", {})
+ if (
+ update.get("sessionUpdate") in ("agent_message", "agent_message_chunk")
+ and content.get("type") == "text"
+ and content.get("text", "").strip()
+ ):
+ timestamp = datetime.fromisoformat(entry["timestamp"])
+ return max(0, (timestamp - started_at).total_seconds())
+ except (AttributeError, KeyError, TypeError, ValueError):
+ continue
+ return None
+
+
+async def finish_scout_eval(
+ team_id: int,
+ task_run: TaskRunDTO | None,
+ *,
+ successful: bool,
+ provider: SandboxProviderStrategy,
+ on_cleanup: Callable[[], Awaitable[None]] | None,
+) -> None:
+ task_run = task_run or await asyncio.to_thread(latest_scout_task, team_id)
+ await _report_cleanup(on_cleanup)
+ if task_run is None:
+ return
+ provider.register_task(str(task_run.task_id))
+ try:
+ client = await async_connect()
+ handle = client.get_workflow_handle(task_run.workflow_id)
+ if not await _finish_workflow(
+ handle,
+ status="completed" if successful else "failed",
+ reason=None if successful else "Scout eval interrupted",
+ ):
+ if successful:
+ raise WorkflowCleanupError("Scout eval workflow cleanup could not be confirmed")
+ logger.warning("Could not confirm interrupted scout eval workflow cleanup")
+ except Exception:
+ if successful:
+ raise
+ logger.warning("Could not finish interrupted scout eval workflow", exc_info=True)
+ finally:
+ try:
+ await asyncio.to_thread(provider.cleanup_case, str(task_run.task_id))
+ except Exception:
+ if successful:
+ raise
+ logger.warning("Could not sweep interrupted scout eval sandbox", exc_info=True)
+
+
+async def run_first_report_scout(
+ case: SandboxedEvalCase,
+ context: CustomPromptSandboxContext,
+ *,
+ provider: SandboxProviderStrategy,
+ on_cleanup: Callable[[], Awaitable[None]] | None = None,
+ timeout_seconds: float | None = None,
+ on_poll_timeout: Callable[[TurnPollTimeout], None] | None = None,
+) -> EvalCaseResult:
+ if case.followups:
+ raise ValueError("The first-report scout is a single investigation and cannot accept eval followups")
+ if context.sandbox_environment_id is not None or case.interaction_origin is not None:
+ raise ValueError("Scout evals use the production environment and interaction origin")
+ started_at = timezone.now()
+ started = time.monotonic()
+ run_note = await asyncio.to_thread(prepare_scout, context)
+ context_seconds = time.monotonic() - started
+ task_run: TaskRunDTO | None = None
+ successful = False
+ run = asyncio.create_task(
+ arun_signals_scout(
+ team_id=context.team_id,
+ skill_name=FIRST_REPORT_SCOUT,
+ triggered_by=TRIGGERED_BY_ONBOARDING,
+ run_note=run_note,
+ agent_runtime=AgentRuntime(
+ runtime_adapter=context.runtime_adapter, model=context.model, reasoning_effort=context.reasoning_effort
+ ),
+ )
+ )
+ try:
+ while not run.done() and task_run is None:
+ task_run = await asyncio.to_thread(latest_scout_task, context.team_id)
+ if task_run is not None:
+ provider.register_task(str(task_run.task_id))
+ else:
+ await asyncio.wait({run}, timeout=1)
+ result = await run
+ task_run = task_run or await asyncio.to_thread(latest_scout_task, context.team_id)
+ if task_run is None:
+ raise RuntimeError(f"Scout did not start: {result.skip_reason or result.status}")
+ raw_log = (
+ await asyncio.to_thread(
+ tasks_api.read_task_run_conversation_logs, task_run.id, task_run.task_id, context.team_id
+ )
+ or ""
+ )
+ successful = result.status == "completed"
+ parsed = _parse_artifacts_from_log(raw_log, time.monotonic() - started, agent_finished=successful)
+ outcome = await asyncio.to_thread(first_report_state, context.team_id)
+ closeout = await asyncio.to_thread(collect_closeout, context.team_id, result.run_id)
+ reports = await asyncio.to_thread(collect_reports, context.team_id, started_at)
+ visible = [
+ report.visible_after_seconds
+ for report in reports
+ if report.status in ("ready", "pending_input") and report.visible_after_seconds is not None
+ ]
+ artifacts = ScoutArtifacts.model_validate(
+ {
+ **parsed.model_dump(),
+ "reports": reports,
+ "last_message": result.last_message,
+ "outcome": outcome.get("status", "incomplete"),
+ "missing_data": closeout.missing_data if closeout else [],
+ "checked_products": closeout.checked_products if closeout else [],
+ "context_seconds": context_seconds,
+ "first_progress_seconds": first_progress_seconds(raw_log, started_at),
+ "first_report_seconds": min(visible) if visible else None,
+ "scout_run_id": result.run_id,
+ }
+ )
+ if not successful:
+ artifacts.exit_code = 1
+ artifacts.stderr = result.last_message or "Scout investigation failed"
+ return EvalCaseResult(artifacts=artifacts, raw_log=raw_log, trace_id=str(task_run.id))
+ finally:
+ if not run.done():
+ run.cancel()
+ await asyncio.gather(run, return_exceptions=True)
+ completion_task = asyncio.create_task(
+ finish_scout_eval(
+ context.team_id, task_run, successful=successful, provider=provider, on_cleanup=on_cleanup
+ )
+ )
+ try:
+ await asyncio.shield(completion_task)
+ except asyncio.CancelledError:
+ try:
+ await asyncio.shield(completion_task)
+ except Exception:
+ logger.warning("Could not finish cancelled scout eval cleanup", exc_info=True)
+ raise