Open source alternative to Semrush and Ahrefs
TypeScript backlink-analysisgoogle-search-console-mcpkeyword-researchmcpseoseo-agentseo-auditseo-automationseo-skillsseo-toolssite-audit
feat(sam): report every turn, step, and client failure to PostHog (#573)
Ben Senescu4a4e8acfda02
diff --git a/src/client/features/sam/SamConversation.tsx b/src/client/features/sam/SamConversation.tsxindex c22df741..7d885674 100644--- a/src/client/features/sam/SamConversation.tsx+++ b/src/client/features/sam/SamConversation.tsx@@ -7,6 +7,7 @@ import { RotateCcw } from "lucide-react";import { findLast } from "remeda";import { ChatComposer } from "@/client/features/sam/ChatComposer";import { invalidateSamSessions } from "@/client/features/sam/samQueries";+import { captureClientEvent } from "@/client/lib/posthog";import {ChatMessage,humanizeToolLabel,@@ -42,6 +43,7 @@ export function SamConversation({status,stop,isRecovering,+ connectionError,} = useAgentChat({ agent, experimental_throttle: 50 });// isRecovering: the DO is settling a turn a reset interrupted (persisting@@ -52,11 +54,36 @@ export function SamConversation({messages,status,);- const sendText = (text: string) => {+ // Client-side send counts, to set against the server's sam:turn events: a+ // send with no matching turn is a message that never reached the DO.+ const sendText = (+ text: string,+ source: "composer" | "suggestion" | "edit" | "retry" = "composer",+ ) => {pinToBottom();+ captureClientEvent("sam:message_send", {+ session_id: sessionId,+ project_id: projectId,+ source,+ chars: text.length,+ });void sendMessage({ text });};+ // What the user sees as a failure: the turn-level error banner below, or+ // the socket dropping (code/reason from the close frame). The server side+ // of the same failure is the sam:turn event with status "error".+ useEffect(() => {+ if (status !== "error" && !connectionError) return;+ captureClientEvent("sam:client_error", {+ session_id: sessionId,+ project_id: projectId,+ kind: connectionError ? "connection" : "turn",+ code: connectionError?.code,+ reason: connectionError?.reason,+ });+ }, [status, connectionError, sessionId, projectId]);+// Rewind the server-side conversation to before `messageId`: the DO aborts// any in-flight turn, then deletes the message and everything after it. Sync// the local view from the server afterwards rather than slicing locally —@@ -77,8 +104,12 @@ export function SamConversation({};const undoFrom = (messageId: string) => void rewindTo(messageId);- const editAndResend = async (messageId: string, newText: string) => {- if (await rewindTo(messageId)) sendText(newText);+ const editAndResend = async (+ messageId: string,+ newText: string,+ source: "edit" | "retry" = "edit",+ ) => {+ if (await rewindTo(messageId)) sendText(newText, source);};// Retry after a failed turn goes through the same rewind-then-resend path// as edit, so the failed partial reply is deleted server-side before the@@ -91,7 +122,7 @@ export function SamConversation({.filter((part) => part.type === "text").map((part) => part.text).join("\n");- void editAndResend(lastUserMessage.id, text);+ void editAndResend(lastUserMessage.id, text, "retry");};// The DO names the session from its first message during the turn, so refresh@@ -213,7 +244,7 @@ export function SamConversation({key={question}type="button"className="rounded-full border border-base-300 bg-base-100 px-3 py-1.5 text-xs font-medium text-base-content/70 transition-colors hover:border-primary/50 hover:text-base-content"- onClick={() => sendText(question)}+ onClick={() => sendText(question, "suggestion")}>{question}</button>diff --git a/src/server/features/sam/SamChatAgent.ts b/src/server/features/sam/SamChatAgent.tsindex ba82d649..5baf6380 100644--- a/src/server/features/sam/SamChatAgent.ts+++ b/src/server/features/sam/SamChatAgent.ts@@ -29,6 +29,10 @@ import { ProjectRepository } from "@/server/features/projects/repositories/Projeimport { buildSamMcpTools } from "@/server/features/sam/samChatTools";import { buildSamSkillSource } from "@/server/features/sam/samSkills";import { buildSamSystemPrompt } from "@/server/features/sam/samSystemPrompt";+import {+ SamTelemetry,+ type SamTurnStats,+} from "@/server/features/sam/samTurnTelemetry";import { buildChatAgentModel } from "@/server/lib/openrouter";import {getEnvValueSync,@@ -38,7 +42,6 @@ import {checkUsageCreditsDepleted,trackUsageCreditSpend,} from "@/server/billing/subscription";-import { captureServerEvent } from "@/server/lib/posthog";import { getPublicOrigin } from "@/server/mcp/public-origin";import { MCP_SCOPE } from "@/lib/oauth-resource";import { AuthRepository } from "@/server/auth/repositories/AuthRepository";@@ -65,6 +68,16 @@ const SAM_BILLING_CHUNK_USD = 0.05;// estimated token count, and mid-turn once a step's input passes 90% of the// proactive ceiling — both far below the window, because the constraint is DO// memory, not the provider.+//+// The between-turn threshold is compared against the Session's built-in+// estimate (~4 chars/token over the full stored transcript). That runs high+// relative to what the model receives, because Think clips tool outputs+// older than the last four messages to 500 chars on every call, so it fires+// early — the safe side. Don't pass a tokenCounter built on the model's+// reported usage: the Session also calls the counter per single message when+// choosing what to protect, and a whole-prompt number collapses the protected+// tail to two messages. A real per-message counter means a tokenizer in the+// bundle, which costs the same heap we're protecting.const SAM_COMPACT_AFTER_TOKENS = 120_000;const SAM_MAX_INPUT_TOKENS = 160_000;@@ -130,11 +143,20 @@ export class SamChatAgent extends Think {// memory limit kills mid-way still bills what it spent — the Sep 2026// recovery loop burned ~$160/day of OpenRouter spend that never reached// onChatResponse. Deductions run off the inference loop, serialized so they- // land in order; the end-of-turn flush awaits the chain.+ // land in order; waitUntil keeps the final flush alive in the background.private turnUnbilledUsd = 0;private turnMonthlyRemaining: number | null = null;private billing: Promise<void> = Promise.resolve();+ // Turn telemetry: armed in beforeTurn, fed by the step and tool hooks,+ // reported once as `sam:turn` when the turn ends by any route (response,+ // error, or a memory-limit kill surfacing as recovery).+ private readonly telemetry = new SamTelemetry(+ () => this.name,+ () => withPgClient(() => this.loadSamContext()),+ (promise) => this.ctx.waitUntil(promise),+ );+// Mid-turn context guard, using the compaction configured in// configureSession (see SAM_MAX_INPUT_TOKENS). The reactive backstop is// deliberately off: it re-runs the whole turn from compacted history, which@@ -188,36 +210,8 @@ export class SamChatAgent extends Think {return [buildSamSkillSource()];}- // Skill activations are Think-internal tools (activate_skill), so they never- // pass through the MCP instrumentation that reports every other SAM tool- // call; mirror its event shape so both land in the same dashboards.override afterToolCall(ctx: ToolCallResultContext) {- if (ctx.toolName !== "activate_skill" || !this.samContext) return;- const input: unknown = ctx.input;- const skill =- typeof input === "object" &&- input !== null &&- "name" in input &&- typeof input.name === "string"- ? input.name- : undefined;- // ctx.waitUntil, not a bare void: the PostHog client flushes on shutdown,- // and a fire-and-forget promise on a turn's last step can be cancelled- // before that flush happens.- this.ctx.waitUntil(- captureServerEvent({- distinctId: this.samContext.row.userId,- event: "sam:skill_activated",- organizationId: this.samContext.project.organizationId,- properties: {- skill,- success: ctx.success,- duration_ms: ctx.durationMs,- project_id: this.samContext.project.id,- source: "in_app_agent",- },- }),- );+ this.telemetry.toolCall(ctx);}configureSession(session: Session): Session {@@ -246,6 +240,7 @@ export class SamChatAgent extends Think {prompt,});this.recordSpend(openRouterCostUsd(result.providerMetadata));+ this.telemetry.compaction();return result.text;}@@ -314,14 +309,16 @@ export class SamChatAgent extends Think {return { model: staticAssistantModel(text) };}- async beforeTurn(_ctx: TurnContext): Promise<TurnConfig> {+ async beforeTurn(turnCtx: TurnContext): Promise<TurnConfig> {// turnUnbilledUsd deliberately carries over: a between-turn compaction// summary can land after the previous flush, and it is the same org's// spend either way.this.turnMonthlyRemaining = null;+ const turn = this.telemetry.beginTurn(turnCtx.continuation);return withPgClient(async (): Promise<TurnConfig> => {const ctx = await this.loadSamContext();if (!ctx) {+ turn.refusal = "no_session";return this.refusalTurn("I couldn't find this chat session. Please start a new one.",);@@ -343,6 +340,7 @@ export class SamChatAgent extends Think {projectId: ctx.project.id,});if (depleted) {+ turn.refusal = "credits";return this.refusalTurn("You're out of credits. Top up to keep using SAM.",);@@ -363,6 +361,7 @@ export class SamChatAgent extends Think {organizationId,);if (hosted && !membership) {+ turn.refusal = "no_access";return this.refusalTurn("You no longer have access to this organization, so I can't continue this chat.",);@@ -381,10 +380,7 @@ export class SamChatAgent extends Think {};return {- tools: buildSamMcpTools(authContext, {- id: ctx.project.id,- domain: ctx.project.domain,- }),+ tools: buildSamMcpTools(authContext, ctx.project, turn.turnId),// SAM runs complex multi-step work in one turn (site-read intake plus// a full research chain, multi-competitor sweeps), so the step budget// is generous; cost is bounded by per-step metering and the model@@ -400,7 +396,9 @@ export class SamChatAgent extends Think {}onStepFinish(ctx: StepContext): void {- this.recordSpend(openRouterCostUsd(ctx.providerMetadata));+ const costUsd = openRouterCostUsd(ctx.providerMetadata);+ this.recordSpend(costUsd);+ this.telemetry.step(ctx, costUsd);}// Add spend to the turn's unbilled total and meter it once a chunk has@@ -408,28 +406,32 @@ export class SamChatAgent extends Think {// armed balance) are never metered.private recordSpend(costUsd: number, { flush = false } = {}): void {this.turnUnbilledUsd += costUsd;+ this.telemetry.spend(costUsd);const armed = this.turnMonthlyRemaining;if (armed === null) return;if (!flush && this.turnUnbilledUsd < SAM_BILLING_CHUNK_USD) return;const chunkUsd = this.turnUnbilledUsd;this.turnUnbilledUsd = 0;+ const turn = this.telemetry.turn;this.billing = this.billing- .then(() => this.meterSpend(chunkUsd, armed))+ .then(() => this.meterSpend(chunkUsd, armed, turn)).catch((error: unknown) => {console.error("[sam] credit metering failed", error);+ this.telemetry.captureError(error, { stage: "metering" });});}private async meterSpend(costUsd: number,armedMonthlyRemaining: number,+ turn: SamTurnStats | null,): Promise<void> {const ctx = await withPgClient(() => this.loadSamContext());if (!ctx) return;// Earlier chunks in the chain have already drawn the monthly balance// down; the value captured at enqueue time only covers a flush that// outlives its turn (onChatError), where the field has been re-armed.- const { monthlyCredits } = await trackUsageCreditSpend({+ const { monthlyCredits, topupCredits } = await trackUsageCreditSpend({customer: {userId: ctx.row.userId,userEmail: ctx.userEmail,@@ -440,15 +442,23 @@ export class SamChatAgent extends Think {creditFeature: "agent",costUsd,monthlyRemaining: this.turnMonthlyRemaining ?? armedMonthlyRemaining,- properties: { provider: "openrouter" },+ properties: { provider: "openrouter", turn_id: turn?.turnId },});- if (this.turnMonthlyRemaining !== null) {+ if (turn) turn.credits += monthlyCredits + topupCredits;+ if (this.turnMonthlyRemaining !== null)this.turnMonthlyRemaining -= monthlyCredits;- }}async onChatResponse(result: ChatResponseResult): Promise<void> {this.flushSpend();+ this.ctx.waitUntil(+ this.telemetry.report(+ result.status,+ { request_id: result.requestId, error_message: result.error },+ this.messages,+ { billing: this.billing },+ ),+ );await withPgClient(async () => {const ctx = await this.loadSamContext();@@ -485,6 +495,7 @@ export class SamChatAgent extends Think {console.error("[sam] chat turn error", ctx?.stage, error);// A stopped or failed turn still consumed what it consumed.this.flushSpend();+ this.telemetry.error(error, ctx, this.messages, this.billing);return error;}@@ -518,6 +529,14 @@ export class SamChatAgent extends Think {incidentId: ctx.incidentId,recoveryKind: ctx.recoveryKind,});+ // The DO that ran the turn is gone with its stats; this is the only+ // signal that a turn was killed, so count it as its own status.+ await this.telemetry.report(+ "interrupted",+ { incident_id: ctx.incidentId, recovery_kind: ctx.recoveryKind },+ this.messages,+ { synthesize: true },+ );// Recovery runs from the DO's startup path, and a hook that fails or// times out is re-run on the next wake; idempotent so the notice is// scheduled once per incident, not once per restart.diff --git a/src/server/features/sam/samChatTools.ts b/src/server/features/sam/samChatTools.tsindex 4046bc77..6d2ad4a6 100644--- a/src/server/features/sam/samChatTools.ts+++ b/src/server/features/sam/samChatTools.ts@@ -312,9 +312,10 @@ function scrapeTools(projectDomain: string | null): ToolSet {export function buildSamMcpTools(authContext: ToolAuthContext,project: { id: string; domain: string | null },+ turnId?: string,): ToolSet {const projectId = project.id;- const toolContext: ToolContext = { auth: authContext };+ const toolContext: ToolContext = { auth: authContext, turnId };const adaptTool = <Shape extends ZodRawShape>(definition: McpToolDefinition<Shape>,) => adaptMcpTool(definition, toolContext, projectId);diff --git a/src/server/features/sam/samTurnTelemetry.test.ts b/src/server/features/sam/samTurnTelemetry.test.tsnew file mode 100644index 00000000..493c55bf--- /dev/null+++ b/src/server/features/sam/samTurnTelemetry.test.ts@@ -0,0 +1,140 @@+import { describe, expect, it, vi } from "vitest";+import type { StepContext, ToolCallResultContext } from "@cloudflare/think";+import {+ SamTelemetry,+ SamTurnStats,+} from "@/server/features/sam/samTurnTelemetry";+import { captureServerEvent } from "@/server/lib/posthog";++vi.mock("@/server/lib/posthog", () => ({+ captureServerEvent: vi.fn(),+ captureServerError: vi.fn(),+}));++const step = (overrides: Record<string, unknown> = {}) =>+ // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- only the fields recordStep reads+ ({+ response: { modelId: "openai/gpt-5.6-luna" },+ usage: {+ inputTokens: 1000,+ outputTokens: 200,+ inputTokenDetails: { cacheReadTokens: 400 },+ outputTokenDetails: { reasoningTokens: 150 },+ },+ toolCalls: [{ toolName: "research_keywords" }],+ finishReason: "tool-calls",+ ...overrides,+ }) as unknown as StepContext;++const toolCall = (overrides: Record<string, unknown> = {}) =>+ // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- only the fields recordToolCall reads+ ({+ toolName: "research_keywords",+ success: true,+ output: { summary: "ok" },+ toolExecutionMs: 120,+ ...overrides,+ }) as unknown as ToolCallResultContext;++describe("SamTurnStats", () => {+ it("sums steps and tool calls into the turn event, counting {error} outputs as failures", () => {+ const stats = new SamTurnStats();+ stats.recordStep(step(), 0.01);+ stats.recordStep(step({ toolCalls: [], finishReason: "stop" }), 0.02);+ stats.recordSpend(0.03);+ stats.recordToolCall(toolCall());+ stats.recordToolCall(+ toolCall({ toolName: "read_pages", output: { error: "blocked" } }),+ );+ stats.recordToolCall(toolCall({ success: false, error: new Error("x") }));++ expect(stats.properties("completed")).toMatchObject({+ turn_id: stats.turnId,+ status: "completed",+ model: "openai/gpt-5.6-luna",+ steps: 2,+ input_tokens: 2000,+ output_tokens: 400,+ reasoning_tokens: 300,+ cached_input_tokens: 800,+ cost_usd: 0.03,+ tool_calls: 3,+ tool_failures: 2,+ tools: ["research_keywords", "read_pages"],+ tool_ms: 360,+ });+ });++ it("links each generation to the turn as its trace", () => {+ const stats = new SamTurnStats();+ const generation = stats.recordStep(step(), 0.01);+ expect(generation).toMatchObject({+ $ai_trace_id: stats.turnId,+ $ai_parent_id: stats.turnId,+ $ai_model: "openai/gpt-5.6-luna",+ $ai_input_tokens: 1000,+ $ai_total_cost_usd: 0.01,+ $ai_tools_called: "research_keywords",+ });+ expect(stats.traceProperties("error", "boom")).toMatchObject({+ $ai_trace_id: stats.turnId,+ $ai_is_error: true,+ $ai_error: "boom",+ });+ });+});++describe("SamTelemetry billing completion", () => {+ it.each(["completed", "error"] as const)(+ "reports a %s turn once with final credits while another turn starts",+ async (outcome) => {+ const pending: Promise<unknown>[] = [];+ const telemetry = new SamTelemetry(+ () => "session-1",+ async () => ({+ row: { userId: "user-1" },+ project: { id: "project-1", organizationId: "org-1" },+ }),+ (promise) => {+ pending.push(promise);+ },+ );+ const first = telemetry.beginTurn(false);+ first.recordStep(step(), 0.01);+ let finishBilling!: () => void;+ const billing = new Promise<void>((resolve) => {+ finishBilling = () => {+ first.credits = 7;+ resolve();+ };+ });+ if (outcome === "error") {+ telemetry.error(new Error("failed"), undefined, [], billing);+ } else {+ pending.push(telemetry.report(outcome, {}, [], { billing }));+ }+ await telemetry.report(outcome, {}, [], { billing });+ expect(captureServerEvent).not.toHaveBeenCalled();++ const second = telemetry.beginTurn(false);+ second.credits = 99;+ finishBilling();+ await Promise.all(pending);++ const events = vi+ .mocked(captureServerEvent)+ .mock.calls.map(([event]) => event);+ const turns = events.filter((event) => event.event === "sam:turn");+ expect(turns).toHaveLength(1);+ expect(turns[0]?.properties).toMatchObject({+ turn_id: first.turnId,+ status: outcome,+ credits: 7,+ });+ expect(+ events.find((event) => event.event === "$ai_trace")?.properties,+ ).toMatchObject({ $ai_trace_id: first.turnId, status: outcome });+ expect(second.reported).toBe(false);+ },+ );+});diff --git a/src/server/features/sam/samTurnTelemetry.ts b/src/server/features/sam/samTurnTelemetry.tsnew file mode 100644index 00000000..85c9016a--- /dev/null+++ b/src/server/features/sam/samTurnTelemetry.ts@@ -0,0 +1,357 @@+import type {+ ChatErrorContext,+ StepContext,+ ToolCallResultContext,+} from "@cloudflare/think";+import type { UIMessage } from "ai";+import { captureServerError, captureServerEvent } from "@/server/lib/posthog";++type SamTurnStatus =+ | "completed"+ | "aborted"+ | "error"+ | "refused"+ | "interrupted";++type SamRefusalReason = "no_session" | "credits" | "no_access";++// Tool failures reach the model as a `{ error }` output rather than a thrown+// error (samChatTools adaptMcpTool / scrapeTools), so Think reports them as+// successful calls; look at the output shape as well as the outcome.+function toolCallFailed(ctx: ToolCallResultContext): boolean {+ if (!ctx.success) return true;+ const output: unknown = ctx.output;+ return (+ typeof output === "object" &&+ output !== null &&+ "error" in output &&+ typeof output.error === "string"+ );+}++/**+ * Everything one SAM turn does, accumulated from Think's hooks and flattened+ * into the `sam:turn` event at the end. One instance per turn; `turnId` is the+ * PostHog LLM-analytics trace id, the `turn_id` on every tool call the turn+ * makes, and the join key between the two.+ */+export class SamTurnStats {+ readonly turnId = crypto.randomUUID();+ readonly startedAt = Date.now();+ private lastStepEndedAt = this.startedAt;+ continuation = false;+ refusal: SamRefusalReason | null = null;+ reported = false;+ model: string | null = null;+ steps = 0;+ inputTokens = 0;+ outputTokens = 0;+ reasoningTokens = 0;+ cachedInputTokens = 0;+ costUsd = 0;+ credits = 0;+ compactions = 0;+ toolCalls = 0;+ toolFailures = 0;+ toolMs = 0;+ private readonly tools = new Set<string>();++ /**+ * Fold one inference step in and return its `$ai_generation` body. Cost is+ * recorded separately (recordSpend) because compaction summaries spend+ * outside the step loop.+ */+ recordStep(ctx: StepContext, costUsd: number): Record<string, unknown> {+ const now = Date.now();+ const latencySeconds = (now - this.lastStepEndedAt) / 1000;+ this.lastStepEndedAt = now;+ const { usage } = ctx;+ const reasoning =+ usage.outputTokenDetails.reasoningTokens ?? usage.reasoningTokens ?? 0;+ const cached =+ usage.inputTokenDetails.cacheReadTokens ?? usage.cachedInputTokens ?? 0;+ this.model = ctx.response.modelId;+ this.steps += 1;+ this.inputTokens += usage.inputTokens ?? 0;+ this.outputTokens += usage.outputTokens ?? 0;+ this.reasoningTokens += reasoning;+ this.cachedInputTokens += cached;+ return {+ $ai_trace_id: this.turnId,+ $ai_parent_id: this.turnId,+ $ai_span_id: crypto.randomUUID(),+ $ai_span_name: `step ${this.steps}`,+ $ai_provider: "openrouter",+ $ai_model: ctx.response.modelId,+ $ai_input_tokens: usage.inputTokens ?? 0,+ $ai_output_tokens: usage.outputTokens ?? 0,+ $ai_reasoning_tokens: reasoning,+ $ai_cache_read_input_tokens: cached,+ $ai_total_cost_usd: costUsd,+ $ai_latency: latencySeconds,+ $ai_is_error: false,+ $ai_tools_called: ctx.toolCalls.map((call) => call.toolName).join(","),+ finish_reason: ctx.finishReason,+ };+ }++ recordToolCall(ctx: ToolCallResultContext): void {+ this.toolCalls += 1;+ this.toolMs += ctx.toolExecutionMs;+ this.tools.add(ctx.toolName);+ if (toolCallFailed(ctx)) this.toolFailures += 1;+ }++ recordSpend(costUsd: number): void {+ this.costUsd += costUsd;+ }++ /** The `sam:turn` event body. */+ properties(status: SamTurnStatus): Record<string, unknown> {+ return {+ turn_id: this.turnId,+ status,+ refusal_reason: this.refusal,+ continuation: this.continuation,+ model: this.model,+ steps: this.steps,+ tool_calls: this.toolCalls,+ tool_failures: this.toolFailures,+ tools: [...this.tools],+ tool_ms: this.toolMs,+ input_tokens: this.inputTokens,+ output_tokens: this.outputTokens,+ reasoning_tokens: this.reasoningTokens,+ cached_input_tokens: this.cachedInputTokens,+ cost_usd: this.costUsd,+ credits: this.credits,+ compactions: this.compactions,+ duration_ms: Date.now() - this.startedAt,+ };+ }++ /** The `$ai_trace` event body: what PostHog's LLM-analytics UI groups by. */+ traceProperties(+ status: SamTurnStatus,+ error?: string,+ ): Record<string, unknown> {+ return {+ $ai_trace_id: this.turnId,+ $ai_span_name: "sam_turn",+ $ai_provider: "openrouter",+ $ai_model: this.model,+ $ai_latency: (Date.now() - this.startedAt) / 1000,+ $ai_is_error: status === "error",+ $ai_error: error,+ status,+ };+ }+}++// The slice of the DO's session context telemetry needs: who to attribute+// events to and which org/project they belong to.+type SamIdentity = {+ row: { userId: string };+ project: { id: string; organizationId: string };+};++function skillName(input: unknown): string | undefined {+ return typeof input === "object" &&+ input !== null &&+ "name" in input &&+ typeof input.name === "string"+ ? input.name+ : undefined;+}++/**+ * The SAM Durable Object's telemetry seam: owns the turn in flight and turns+ * Think's hooks into PostHog events (`sam:turn`, `$ai_generation`/`$ai_trace`+ * for the LLM-analytics UI, `sam:skill_activated`) and error captures. Every+ * event carries `source: "in_app_agent"` plus session/project ids so it+ * filters the same way as the mcp:tool_call events the turn's tools emit.+ *+ * Captures go through `waitUntil`, not a bare void: the PostHog client+ * flushes on shutdown, and a fire-and-forget promise on a turn's last step+ * can be cancelled before that flush happens.+ */+export class SamTelemetry {+ private current: SamTurnStats | null = null;++ constructor(+ private readonly sessionId: () => string,+ // Resolved lazily (and cached by the caller): a memory-limit kill wakes a+ // fresh DO whose session row is not loaded when recovery reports.+ private readonly identity: () => Promise<SamIdentity | null>,+ private readonly waitUntil: (promise: Promise<unknown>) => void,+ ) {}++ get turn(): SamTurnStats | null {+ return this.current;+ }++ beginTurn(continuation: boolean): SamTurnStats {+ const turn = new SamTurnStats();+ turn.continuation = continuation;+ this.current = turn;+ return turn;+ }++ spend(costUsd: number): void {+ this.current?.recordSpend(costUsd);+ }++ compaction(): void {+ if (this.current) this.current.compactions += 1;+ }++ step(ctx: StepContext, costUsd: number): void {+ const turn = this.current;+ // Refusals answer through the canned model: no provider call to trace.+ if (!turn || turn.refusal) return;+ this.waitUntil(+ this.capture("$ai_generation", turn.recordStep(ctx, costUsd)),+ );+ }++ // Skill activations are Think-internal tools (activate_skill), so they never+ // pass through the MCP instrumentation that reports every other SAM tool+ // call; mirror its event shape so both land in the same dashboards.+ toolCall(ctx: ToolCallResultContext): void {+ this.current?.recordToolCall(ctx);+ if (ctx.toolName !== "activate_skill") return;+ this.waitUntil(+ this.capture("sam:skill_activated", {+ skill: skillName(ctx.input),+ success: ctx.success,+ duration_ms: ctx.durationMs,+ turn_id: this.current?.turnId,+ }),+ );+ }++ /**+ * Emit the turn's `sam:turn` event (and its `$ai_trace` when a model ran),+ * once per turn. Think fires onChatError before the response hook for a+ * failed turn, so the error path reports and the response hook then finds+ * the turn already reported. `synthesize` is for turns that end with no+ * armed stats — a parse/persist failure before beforeTurn, or a memory-+ * limit kill that reset the DO — which are reported from an empty stats+ * object so they still count. A gate refusal completes like any other+ * turn (the canned model streams it) and is reported as "refused".+ */+ async report(+ outcome: SamTurnStatus,+ extra: Record<string, unknown>,+ messages: readonly UIMessage[],+ {+ synthesize = false,+ billing,+ }: { synthesize?: boolean; billing?: Promise<void> } = {},+ ): Promise<void> {+ let turn = this.current;+ if (!turn || turn.reported) {+ if (!synthesize) return;+ turn = new SamTurnStats();+ }+ turn.reported = true;+ const status: SamTurnStatus =+ outcome === "completed" && turn.refusal ? "refused" : outcome;+ const userMessages = messages.filter((m) => m.role === "user").length;+ const properties = {+ ...turn.properties(status),+ message_count: messages.length,+ first_turn: userMessages <= 1,+ ...extra,+ credits: turn.credits,+ };+ const error = extra.error_message;+ const trace =+ turn.steps > 0+ ? turn.traceProperties(+ status,+ typeof error === "string" ? error : undefined,+ )+ : null;+ // Reserve this turn and snapshot its timing before billing settles. The+ // next turn can start while the final credit deduction is still pending.+ await billing;+ properties.credits = turn.credits;+ console.log("[sam] turn", { session_id: this.sessionId(), ...properties });+ await this.capture("sam:turn", properties);+ if (trace) {+ await this.capture("$ai_trace", trace);+ }+ }++ error(+ error: unknown,+ ctx: ChatErrorContext | undefined,+ messages: readonly UIMessage[],+ billing?: Promise<void>,+ ): void {+ const errorName = error instanceof Error ? error.name : typeof error;+ const errorMessage = error instanceof Error ? error.message : String(error);+ this.waitUntil(+ this.report(+ "error",+ {+ request_id: ctx?.requestId,+ error_stage: ctx?.stage,+ error_class: ctx?.classification,+ error_name: errorName,+ error_message: errorMessage.slice(0, 500),+ },+ messages,+ { synthesize: true, billing },+ ),+ );+ // A user-cancelled turn is an abort, not a failure worth an issue.+ if (errorName !== "AbortError") {+ this.captureError(error, {+ stage: ctx?.stage,+ classification: ctx?.classification,+ });+ }+ }++ /** Report a server-side failure to PostHog error tracking. */+ captureError(+ error: unknown,+ properties: Record<string, string | undefined>,+ ): void {+ this.waitUntil(+ this.identity().then((identity) =>+ captureServerError(+ error,+ {+ source: "sam",+ session_id: this.sessionId(),+ turn_id: this.current?.turnId,+ ...properties,+ },+ identity?.row.userId,+ ),+ ),+ );+ }++ private async capture(+ event: string,+ properties: Record<string, unknown>,+ ): Promise<void> {+ const identity = await this.identity();+ if (!identity) return;+ await captureServerEvent({+ distinctId: identity.row.userId,+ event,+ organizationId: identity.project.organizationId,+ properties: {+ source: "in_app_agent",+ session_id: this.sessionId(),+ project_id: identity.project.id,+ ...properties,+ },+ });+ }+}diff --git a/src/server/mcp/context.ts b/src/server/mcp/context.tsindex 795ab1d4..c5f7c330 100644--- a/src/server/mcp/context.ts+++ b/src/server/mcp/context.ts@@ -26,6 +26,9 @@ export type ToolAuthContext = {export type ToolContext = {auth: ToolAuthContext;+ // The in-app agent's turn id, so a tool call's telemetry can be joined to+ // the `sam:turn` event of the turn that made it. Absent for MCP transports.+ turnId?: string;};export const MCP_AUTH_CONTEXT_PROP = "openSeoAuth";diff --git a/src/server/mcp/instrumentation.ts b/src/server/mcp/instrumentation.tsindex d4745ff1..235bf9ab 100644--- a/src/server/mcp/instrumentation.ts+++ b/src/server/mcp/instrumentation.ts@@ -56,6 +56,7 @@ function captureMcpToolCall(error_code: outcome.errorCode,client_id: auth.clientId,source: auth.clientId ? "mcp_client" : "in_app_agent",+ turn_id: context.turnId,duration_ms: outcome.durationMs,project_id: outcome.projectId,row_count: outcome.rowCount,diff --git a/src/serverFunctions/sam.ts b/src/serverFunctions/sam.tsindex 4a05d986..8768c873 100644--- a/src/serverFunctions/sam.ts+++ b/src/serverFunctions/sam.ts@@ -1,10 +1,12 @@import { createServerFn } from "@tanstack/react-start";+import { waitUntil } from "cloudflare:workers";import { z } from "zod";import {requireAuthenticatedContext,requireProjectContext,} from "@/serverFunctions/middleware";import { AppError } from "@/server/lib/errors";+import { captureServerEvent } from "@/server/lib/posthog";import { SamSessionRepository } from "@/server/features/sam/SamSessionRepository";import { ProjectRepository } from "@/server/features/projects/repositories/ProjectRepository";@@ -36,6 +38,14 @@ export const createSamSession = createServerFn({ method: "POST" })if (!session) {throw new AppError("INTERNAL_ERROR", "Failed to create chat session");}+ waitUntil(+ captureServerEvent({+ distinctId: context.userId,+ event: "sam:session_create",+ organizationId: context.organizationId,+ properties: { project_id: context.projectId, session_id: session.id },+ }),+ );return { id: session.id };});@@ -64,5 +74,13 @@ export const archiveSamSession = createServerFn({ method: "POST" })throw new AppError("NOT_FOUND", "Chat session not found");}await SamSessionRepository.archiveSession(data.sessionId);+ waitUntil(+ captureServerEvent({+ distinctId: context.userId,+ event: "sam:session_archive",+ organizationId: context.organizationId,+ properties: { project_id: project.id, session_id: session.id },+ }),+ );return { ok: true };});