Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(logs): add incident cases to the logs agent eval (#122550)

Jon McCallum2d25467a4785
diff --git a/products/logs/evals/eval_agent_observability.py b/products/logs/evals/eval_agent_observability.py
index d5e258636f4f..7d35f5624e9f 100644
--- a/products/logs/evals/eval_agent_observability.py
+++ b/products/logs/evals/eval_agent_observability.py
@@ -2,6 +2,7 @@ from __future__ import annotations
import json
import uuid
+import random
from datetime import UTC, datetime, timedelta
from typing import Any
@@ -18,23 +19,39 @@ RUN_ID = "run-7f3a9c"
NEW_ERROR = "card vault handshake timed out"
SILENT = "ledger-sync"
+REQUEST_ID = "req-4b81e2"
+REQUEST_ERROR = "reservation rejected: sku locked"
+DEPLOY_VERSION = "2.4.1"
+DEPLOY_ERROR = "refund rejected: idempotency key missing"
+SLOW_ROUTE = "/api/search"
+FAILING_REGION = "ap-southeast-2"
+REGION_ERROR = "tax quote unavailable"
+
+
+def _row(team_id: int, at: datetime, service: str, body: str, severity: str = "info", **attributes: str) -> dict:
+ stamp = at.strftime("%Y-%m-%d %H:%M:%S.%f")
+ return {
+ "team_id": team_id,
+ "uuid": str(uuid.uuid4()),
+ "timestamp": stamp,
+ "observed_timestamp": stamp,
+ "body": body,
+ "severity_text": severity,
+ "service_name": service,
+ "attributes_map_str": {f"{key}__str": value for key, value in attributes.items()},
+ }
+
+
+def _insert(rows: list[dict]) -> None:
+ sync_execute("INSERT INTO logs FORMAT JSONEachRow\n" + "".join(json.dumps(row) + "\n" for row in rows))
+
def seed_observability_logs(context: CustomPromptSandboxContext) -> dict[str, Any]:
now = datetime.now(UTC).replace(second=0, microsecond=0)
spike_start = now - timedelta(minutes=40)
def log(at: datetime, service: str, body: str, severity: str = "info", **attributes: str) -> dict:
- stamp = at.strftime("%Y-%m-%d %H:%M:%S.%f")
- return {
- "team_id": context.team_id,
- "uuid": str(uuid.uuid4()),
- "timestamp": stamp,
- "observed_timestamp": stamp,
- "body": body,
- "severity_text": severity,
- "service_name": service,
- "attributes_map_str": {f"{key}__str": value for key, value in attributes.items()},
- }
+ return _row(context.team_id, at, service, body, severity, **attributes)
rows = [log(now - timedelta(minutes=m), "checkout", f"order {m} placed") for m in range(0, 24 * 60, 10)]
rows += [log(now - timedelta(days=1, minutes=m), SILENT, "ledger batch synced") for m in range(0, 600, 10)]
@@ -47,10 +64,51 @@ def seed_observability_logs(context: CustomPromptSandboxContext) -> dict[str, An
(20, "worker", "step upload failed: quota exceeded", "error"),
)
]
- sync_execute("INSERT INTO logs FORMAT JSONEachRow\n" + "".join(json.dumps(row) + "\n" for row in rows))
+ _insert(rows)
return {"spike_hhmm": spike_start.strftime("%H:%M")}
+def seed_incident_logs(context: CustomPromptSandboxContext) -> dict[str, Any]:
+ now = datetime.now(UTC).replace(second=0, microsecond=0)
+ deploy_at = now - timedelta(minutes=90)
+ rng = random.Random(7)
+
+ def log(at: datetime, service: str, body: str, severity: str = "info", **attributes: str) -> dict:
+ return _row(context.team_id, at, service, body, severity, **attributes)
+
+ rows: list[dict] = []
+ for step in range(0, 6 * 3600, 15):
+ at = now - timedelta(seconds=step)
+ recent = step < 3600
+ for service in ("api", "inventory"):
+ rows.append(log(at, service, "request handled", request_id=f"req-{rng.getrandbits(24):06x}"))
+ for route in (SLOW_ROUTE, "/api/suggest"):
+ millis = rng.randint(800, 1200) if recent and route == SLOW_ROUTE else rng.randint(30, 90)
+ rows.append(log(at, "search", f"GET {route} completed", route=route, duration_ms=str(millis)))
+ version = DEPLOY_VERSION if at >= deploy_at else "2.4.0"
+ if version == DEPLOY_VERSION and step % 45 == 0:
+ rows.append(log(at, "payments", DEPLOY_ERROR, "error", version=version))
+ else:
+ rows.append(log(at, "payments", "refund processed", version=version))
+ region = ("us-east-1", "eu-west-1", FAILING_REGION)[step // 15 % 3]
+ if region == FAILING_REGION and step < 2 * 3600:
+ rows.append(log(at, "billing", REGION_ERROR, "error", region=region))
+ else:
+ rows.append(log(at, "billing", "invoice finalized", region=region))
+
+ failed_at = now - timedelta(minutes=75)
+ rows += [
+ log(failed_at + timedelta(milliseconds=ms), service, body, severity, request_id=REQUEST_ID)
+ for ms, service, body, severity in (
+ (0, "api", "POST /api/orders received", "info"),
+ (40, "inventory", REQUEST_ERROR, "error"),
+ (55, "api", "POST /api/orders failed with status 409", "warn"),
+ )
+ ]
+ _insert(rows)
+ return {}
+
+
class AnswerMentions(Scorer):
def _name(self) -> str:
return "answer_mentions"
@@ -85,6 +143,33 @@ CASES = [
("spike_onset", "Checkout errors spiked in the last 2h. When did the spike start, in UTC?", ["seed:spike_hhmm"], 3),
]
+INCIDENT_CASES = [
+ (
+ "request_trace",
+ f"A customer says request {REQUEST_ID} failed about an hour ago. Which service caused it and why?",
+ ["inventory", "sku locked"],
+ 2,
+ ),
+ (
+ "deploy_regression",
+ "Did anything get worse after the most recent payments deploy? Name the version and what broke.",
+ [DEPLOY_VERSION, "idempotency key"],
+ 4,
+ ),
+ (
+ "latency_regression",
+ "One route in the search service got slower in the last hour. Which route is it?",
+ [SLOW_ROUTE],
+ 3,
+ ),
+ (
+ "segmented_errors",
+ "Billing errors went up in the last 2h. Is it every customer, or only some of them?",
+ [FAILING_REGION, "tax quote"],
+ 4,
+ ),
+]
+
async def eval_agent_observability(ctx: EvalContext) -> None:
await SandboxedPublicEval(
@@ -94,9 +179,10 @@ async def eval_agent_observability(ctx: EvalContext) -> None:
name=name,
prompt=prompt,
expected={"answer_mentions": terms, "call_budget": budget},
- setup=seed_observability_logs,
+ setup=setup,
)
- for name, prompt, terms, budget in CASES
+ for cases, setup in ((CASES, seed_observability_logs), (INCIDENT_CASES, seed_incident_logs))
+ for name, prompt, terms, budget in cases
],
scorers=[AnswerMentions(), CallBudget()],
ctx=ctx,