Skip to content

Open source alternative to Semrush and Ahrefs

feat(sam): report every turn, step, and client failure to PostHog (#573)

Ben Senescu4a4e8acfda02
diff --git a/src/client/features/sam/SamConversation.tsx b/src/client/features/sam/SamConversation.tsx
index c22df741..7d885674 100644
--- a/src/client/features/sam/SamConversation.tsx
+++ b/src/client/features/sam/SamConversation.tsx
@@ -7,6 +7,7 @@ import { RotateCcw } from "lucide-react";
import { findLast } from "remeda";
import { ChatComposer } from "@/client/features/sam/ChatComposer";
import { invalidateSamSessions } from "@/client/features/sam/samQueries";
+import { captureClientEvent } from "@/client/lib/posthog";
import {
ChatMessage,
humanizeToolLabel,
@@ -42,6 +43,7 @@ export function SamConversation({
status,
stop,
isRecovering,
+ connectionError,
} = useAgentChat({ agent, experimental_throttle: 50 });
// isRecovering: the DO is settling a turn a reset interrupted (persisting
@@ -52,11 +54,36 @@ export function SamConversation({
messages,
status,
);
- const sendText = (text: string) => {
+ // Client-side send counts, to set against the server's sam:turn events: a
+ // send with no matching turn is a message that never reached the DO.
+ const sendText = (
+ text: string,
+ source: "composer" | "suggestion" | "edit" | "retry" = "composer",
+ ) => {
pinToBottom();
+ captureClientEvent("sam:message_send", {
+ session_id: sessionId,
+ project_id: projectId,
+ source,
+ chars: text.length,
+ });
void sendMessage({ text });
};
+ // What the user sees as a failure: the turn-level error banner below, or
+ // the socket dropping (code/reason from the close frame). The server side
+ // of the same failure is the sam:turn event with status "error".
+ useEffect(() => {
+ if (status !== "error" && !connectionError) return;
+ captureClientEvent("sam:client_error", {
+ session_id: sessionId,
+ project_id: projectId,
+ kind: connectionError ? "connection" : "turn",
+ code: connectionError?.code,
+ reason: connectionError?.reason,
+ });
+ }, [status, connectionError, sessionId, projectId]);
+
// Rewind the server-side conversation to before `messageId`: the DO aborts
// any in-flight turn, then deletes the message and everything after it. Sync
// the local view from the server afterwards rather than slicing locally —
@@ -77,8 +104,12 @@ export function SamConversation({
};
const undoFrom = (messageId: string) => void rewindTo(messageId);
- const editAndResend = async (messageId: string, newText: string) => {
- if (await rewindTo(messageId)) sendText(newText);
+ const editAndResend = async (
+ messageId: string,
+ newText: string,
+ source: "edit" | "retry" = "edit",
+ ) => {
+ if (await rewindTo(messageId)) sendText(newText, source);
};
// Retry after a failed turn goes through the same rewind-then-resend path
// as edit, so the failed partial reply is deleted server-side before the
@@ -91,7 +122,7 @@ export function SamConversation({
.filter((part) => part.type === "text")
.map((part) => part.text)
.join("\n");
- void editAndResend(lastUserMessage.id, text);
+ void editAndResend(lastUserMessage.id, text, "retry");
};
// The DO names the session from its first message during the turn, so refresh
@@ -213,7 +244,7 @@ export function SamConversation({
key={question}
type="button"
className="rounded-full border border-base-300 bg-base-100 px-3 py-1.5 text-xs font-medium text-base-content/70 transition-colors hover:border-primary/50 hover:text-base-content"
- onClick={() => sendText(question)}
+ onClick={() => sendText(question, "suggestion")}
>
{question}
</button>
diff --git a/src/server/features/sam/SamChatAgent.ts b/src/server/features/sam/SamChatAgent.ts
index ba82d649..5baf6380 100644
--- a/src/server/features/sam/SamChatAgent.ts
+++ b/src/server/features/sam/SamChatAgent.ts
@@ -29,6 +29,10 @@ import { ProjectRepository } from "@/server/features/projects/repositories/Proje
import { buildSamMcpTools } from "@/server/features/sam/samChatTools";
import { buildSamSkillSource } from "@/server/features/sam/samSkills";
import { buildSamSystemPrompt } from "@/server/features/sam/samSystemPrompt";
+import {
+ SamTelemetry,
+ type SamTurnStats,
+} from "@/server/features/sam/samTurnTelemetry";
import { buildChatAgentModel } from "@/server/lib/openrouter";
import {
getEnvValueSync,
@@ -38,7 +42,6 @@ import {
checkUsageCreditsDepleted,
trackUsageCreditSpend,
} from "@/server/billing/subscription";
-import { captureServerEvent } from "@/server/lib/posthog";
import { getPublicOrigin } from "@/server/mcp/public-origin";
import { MCP_SCOPE } from "@/lib/oauth-resource";
import { AuthRepository } from "@/server/auth/repositories/AuthRepository";
@@ -65,6 +68,16 @@ const SAM_BILLING_CHUNK_USD = 0.05;
// estimated token count, and mid-turn once a step's input passes 90% of the
// proactive ceiling — both far below the window, because the constraint is DO
// memory, not the provider.
+//
+// The between-turn threshold is compared against the Session's built-in
+// estimate (~4 chars/token over the full stored transcript). That runs high
+// relative to what the model receives, because Think clips tool outputs
+// older than the last four messages to 500 chars on every call, so it fires
+// early — the safe side. Don't pass a tokenCounter built on the model's
+// reported usage: the Session also calls the counter per single message when
+// choosing what to protect, and a whole-prompt number collapses the protected
+// tail to two messages. A real per-message counter means a tokenizer in the
+// bundle, which costs the same heap we're protecting.
const SAM_COMPACT_AFTER_TOKENS = 120_000;
const SAM_MAX_INPUT_TOKENS = 160_000;
@@ -130,11 +143,20 @@ export class SamChatAgent extends Think {
// memory limit kills mid-way still bills what it spent — the Sep 2026
// recovery loop burned ~$160/day of OpenRouter spend that never reached
// onChatResponse. Deductions run off the inference loop, serialized so they
- // land in order; the end-of-turn flush awaits the chain.
+ // land in order; waitUntil keeps the final flush alive in the background.
private turnUnbilledUsd = 0;
private turnMonthlyRemaining: number | null = null;
private billing: Promise<void> = Promise.resolve();
+ // Turn telemetry: armed in beforeTurn, fed by the step and tool hooks,
+ // reported once as `sam:turn` when the turn ends by any route (response,
+ // error, or a memory-limit kill surfacing as recovery).
+ private readonly telemetry = new SamTelemetry(
+ () => this.name,
+ () => withPgClient(() => this.loadSamContext()),
+ (promise) => this.ctx.waitUntil(promise),
+ );
+
// Mid-turn context guard, using the compaction configured in
// configureSession (see SAM_MAX_INPUT_TOKENS). The reactive backstop is
// deliberately off: it re-runs the whole turn from compacted history, which
@@ -188,36 +210,8 @@ export class SamChatAgent extends Think {
return [buildSamSkillSource()];
}
- // Skill activations are Think-internal tools (activate_skill), so they never
- // pass through the MCP instrumentation that reports every other SAM tool
- // call; mirror its event shape so both land in the same dashboards.
override afterToolCall(ctx: ToolCallResultContext) {
- if (ctx.toolName !== "activate_skill" || !this.samContext) return;
- const input: unknown = ctx.input;
- const skill =
- typeof input === "object" &&
- input !== null &&
- "name" in input &&
- typeof input.name === "string"
- ? input.name
- : undefined;
- // ctx.waitUntil, not a bare void: the PostHog client flushes on shutdown,
- // and a fire-and-forget promise on a turn's last step can be cancelled
- // before that flush happens.
- this.ctx.waitUntil(
- captureServerEvent({
- distinctId: this.samContext.row.userId,
- event: "sam:skill_activated",
- organizationId: this.samContext.project.organizationId,
- properties: {
- skill,
- success: ctx.success,
- duration_ms: ctx.durationMs,
- project_id: this.samContext.project.id,
- source: "in_app_agent",
- },
- }),
- );
+ this.telemetry.toolCall(ctx);
}
configureSession(session: Session): Session {
@@ -246,6 +240,7 @@ export class SamChatAgent extends Think {
prompt,
});
this.recordSpend(openRouterCostUsd(result.providerMetadata));
+ this.telemetry.compaction();
return result.text;
}
@@ -314,14 +309,16 @@ export class SamChatAgent extends Think {
return { model: staticAssistantModel(text) };
}
- async beforeTurn(_ctx: TurnContext): Promise<TurnConfig> {
+ async beforeTurn(turnCtx: TurnContext): Promise<TurnConfig> {
// turnUnbilledUsd deliberately carries over: a between-turn compaction
// summary can land after the previous flush, and it is the same org's
// spend either way.
this.turnMonthlyRemaining = null;
+ const turn = this.telemetry.beginTurn(turnCtx.continuation);
return withPgClient(async (): Promise<TurnConfig> => {
const ctx = await this.loadSamContext();
if (!ctx) {
+ turn.refusal = "no_session";
return this.refusalTurn(
"I couldn't find this chat session. Please start a new one.",
);
@@ -343,6 +340,7 @@ export class SamChatAgent extends Think {
projectId: ctx.project.id,
});
if (depleted) {
+ turn.refusal = "credits";
return this.refusalTurn(
"You're out of credits. Top up to keep using SAM.",
);
@@ -363,6 +361,7 @@ export class SamChatAgent extends Think {
organizationId,
);
if (hosted && !membership) {
+ turn.refusal = "no_access";
return this.refusalTurn(
"You no longer have access to this organization, so I can't continue this chat.",
);
@@ -381,10 +380,7 @@ export class SamChatAgent extends Think {
};
return {
- tools: buildSamMcpTools(authContext, {
- id: ctx.project.id,
- domain: ctx.project.domain,
- }),
+ tools: buildSamMcpTools(authContext, ctx.project, turn.turnId),
// SAM runs complex multi-step work in one turn (site-read intake plus
// a full research chain, multi-competitor sweeps), so the step budget
// is generous; cost is bounded by per-step metering and the model
@@ -400,7 +396,9 @@ export class SamChatAgent extends Think {
}
onStepFinish(ctx: StepContext): void {
- this.recordSpend(openRouterCostUsd(ctx.providerMetadata));
+ const costUsd = openRouterCostUsd(ctx.providerMetadata);
+ this.recordSpend(costUsd);
+ this.telemetry.step(ctx, costUsd);
}
// Add spend to the turn's unbilled total and meter it once a chunk has
@@ -408,28 +406,32 @@ export class SamChatAgent extends Think {
// armed balance) are never metered.
private recordSpend(costUsd: number, { flush = false } = {}): void {
this.turnUnbilledUsd += costUsd;
+ this.telemetry.spend(costUsd);
const armed = this.turnMonthlyRemaining;
if (armed === null) return;
if (!flush && this.turnUnbilledUsd < SAM_BILLING_CHUNK_USD) return;
const chunkUsd = this.turnUnbilledUsd;
this.turnUnbilledUsd = 0;
+ const turn = this.telemetry.turn;
this.billing = this.billing
- .then(() => this.meterSpend(chunkUsd, armed))
+ .then(() => this.meterSpend(chunkUsd, armed, turn))
.catch((error: unknown) => {
console.error("[sam] credit metering failed", error);
+ this.telemetry.captureError(error, { stage: "metering" });
});
}
private async meterSpend(
costUsd: number,
armedMonthlyRemaining: number,
+ turn: SamTurnStats | null,
): Promise<void> {
const ctx = await withPgClient(() => this.loadSamContext());
if (!ctx) return;
// Earlier chunks in the chain have already drawn the monthly balance
// down; the value captured at enqueue time only covers a flush that
// outlives its turn (onChatError), where the field has been re-armed.
- const { monthlyCredits } = await trackUsageCreditSpend({
+ const { monthlyCredits, topupCredits } = await trackUsageCreditSpend({
customer: {
userId: ctx.row.userId,
userEmail: ctx.userEmail,
@@ -440,15 +442,23 @@ export class SamChatAgent extends Think {
creditFeature: "agent",
costUsd,
monthlyRemaining: this.turnMonthlyRemaining ?? armedMonthlyRemaining,
- properties: { provider: "openrouter" },
+ properties: { provider: "openrouter", turn_id: turn?.turnId },
});
- if (this.turnMonthlyRemaining !== null) {
+ if (turn) turn.credits += monthlyCredits + topupCredits;
+ if (this.turnMonthlyRemaining !== null)
this.turnMonthlyRemaining -= monthlyCredits;
- }
}
async onChatResponse(result: ChatResponseResult): Promise<void> {
this.flushSpend();
+ this.ctx.waitUntil(
+ this.telemetry.report(
+ result.status,
+ { request_id: result.requestId, error_message: result.error },
+ this.messages,
+ { billing: this.billing },
+ ),
+ );
await withPgClient(async () => {
const ctx = await this.loadSamContext();
@@ -485,6 +495,7 @@ export class SamChatAgent extends Think {
console.error("[sam] chat turn error", ctx?.stage, error);
// A stopped or failed turn still consumed what it consumed.
this.flushSpend();
+ this.telemetry.error(error, ctx, this.messages, this.billing);
return error;
}
@@ -518,6 +529,14 @@ export class SamChatAgent extends Think {
incidentId: ctx.incidentId,
recoveryKind: ctx.recoveryKind,
});
+ // The DO that ran the turn is gone with its stats; this is the only
+ // signal that a turn was killed, so count it as its own status.
+ await this.telemetry.report(
+ "interrupted",
+ { incident_id: ctx.incidentId, recovery_kind: ctx.recoveryKind },
+ this.messages,
+ { synthesize: true },
+ );
// Recovery runs from the DO's startup path, and a hook that fails or
// times out is re-run on the next wake; idempotent so the notice is
// scheduled once per incident, not once per restart.
diff --git a/src/server/features/sam/samChatTools.ts b/src/server/features/sam/samChatTools.ts
index 4046bc77..6d2ad4a6 100644
--- a/src/server/features/sam/samChatTools.ts
+++ b/src/server/features/sam/samChatTools.ts
@@ -312,9 +312,10 @@ function scrapeTools(projectDomain: string | null): ToolSet {
export function buildSamMcpTools(
authContext: ToolAuthContext,
project: { id: string; domain: string | null },
+ turnId?: string,
): ToolSet {
const projectId = project.id;
- const toolContext: ToolContext = { auth: authContext };
+ const toolContext: ToolContext = { auth: authContext, turnId };
const adaptTool = <Shape extends ZodRawShape>(
definition: McpToolDefinition<Shape>,
) => adaptMcpTool(definition, toolContext, projectId);
diff --git a/src/server/features/sam/samTurnTelemetry.test.ts b/src/server/features/sam/samTurnTelemetry.test.ts
new file mode 100644
index 00000000..493c55bf
--- /dev/null
+++ b/src/server/features/sam/samTurnTelemetry.test.ts
@@ -0,0 +1,140 @@
+import { describe, expect, it, vi } from "vitest";
+import type { StepContext, ToolCallResultContext } from "@cloudflare/think";
+import {
+ SamTelemetry,
+ SamTurnStats,
+} from "@/server/features/sam/samTurnTelemetry";
+import { captureServerEvent } from "@/server/lib/posthog";
+
+vi.mock("@/server/lib/posthog", () => ({
+ captureServerEvent: vi.fn(),
+ captureServerError: vi.fn(),
+}));
+
+const step = (overrides: Record<string, unknown> = {}) =>
+ // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- only the fields recordStep reads
+ ({
+ response: { modelId: "openai/gpt-5.6-luna" },
+ usage: {
+ inputTokens: 1000,
+ outputTokens: 200,
+ inputTokenDetails: { cacheReadTokens: 400 },
+ outputTokenDetails: { reasoningTokens: 150 },
+ },
+ toolCalls: [{ toolName: "research_keywords" }],
+ finishReason: "tool-calls",
+ ...overrides,
+ }) as unknown as StepContext;
+
+const toolCall = (overrides: Record<string, unknown> = {}) =>
+ // oxlint-disable-next-line typescript/no-unsafe-type-assertion -- only the fields recordToolCall reads
+ ({
+ toolName: "research_keywords",
+ success: true,
+ output: { summary: "ok" },
+ toolExecutionMs: 120,
+ ...overrides,
+ }) as unknown as ToolCallResultContext;
+
+describe("SamTurnStats", () => {
+ it("sums steps and tool calls into the turn event, counting {error} outputs as failures", () => {
+ const stats = new SamTurnStats();
+ stats.recordStep(step(), 0.01);
+ stats.recordStep(step({ toolCalls: [], finishReason: "stop" }), 0.02);
+ stats.recordSpend(0.03);
+ stats.recordToolCall(toolCall());
+ stats.recordToolCall(
+ toolCall({ toolName: "read_pages", output: { error: "blocked" } }),
+ );
+ stats.recordToolCall(toolCall({ success: false, error: new Error("x") }));
+
+ expect(stats.properties("completed")).toMatchObject({
+ turn_id: stats.turnId,
+ status: "completed",
+ model: "openai/gpt-5.6-luna",
+ steps: 2,
+ input_tokens: 2000,
+ output_tokens: 400,
+ reasoning_tokens: 300,
+ cached_input_tokens: 800,
+ cost_usd: 0.03,
+ tool_calls: 3,
+ tool_failures: 2,
+ tools: ["research_keywords", "read_pages"],
+ tool_ms: 360,
+ });
+ });
+
+ it("links each generation to the turn as its trace", () => {
+ const stats = new SamTurnStats();
+ const generation = stats.recordStep(step(), 0.01);
+ expect(generation).toMatchObject({
+ $ai_trace_id: stats.turnId,
+ $ai_parent_id: stats.turnId,
+ $ai_model: "openai/gpt-5.6-luna",
+ $ai_input_tokens: 1000,
+ $ai_total_cost_usd: 0.01,
+ $ai_tools_called: "research_keywords",
+ });
+ expect(stats.traceProperties("error", "boom")).toMatchObject({
+ $ai_trace_id: stats.turnId,
+ $ai_is_error: true,
+ $ai_error: "boom",
+ });
+ });
+});
+
+describe("SamTelemetry billing completion", () => {
+ it.each(["completed", "error"] as const)(
+ "reports a %s turn once with final credits while another turn starts",
+ async (outcome) => {
+ const pending: Promise<unknown>[] = [];
+ const telemetry = new SamTelemetry(
+ () => "session-1",
+ async () => ({
+ row: { userId: "user-1" },
+ project: { id: "project-1", organizationId: "org-1" },
+ }),
+ (promise) => {
+ pending.push(promise);
+ },
+ );
+ const first = telemetry.beginTurn(false);
+ first.recordStep(step(), 0.01);
+ let finishBilling!: () => void;
+ const billing = new Promise<void>((resolve) => {
+ finishBilling = () => {
+ first.credits = 7;
+ resolve();
+ };
+ });
+ if (outcome === "error") {
+ telemetry.error(new Error("failed"), undefined, [], billing);
+ } else {
+ pending.push(telemetry.report(outcome, {}, [], { billing }));
+ }
+ await telemetry.report(outcome, {}, [], { billing });
+ expect(captureServerEvent).not.toHaveBeenCalled();
+
+ const second = telemetry.beginTurn(false);
+ second.credits = 99;
+ finishBilling();
+ await Promise.all(pending);
+
+ const events = vi
+ .mocked(captureServerEvent)
+ .mock.calls.map(([event]) => event);
+ const turns = events.filter((event) => event.event === "sam:turn");
+ expect(turns).toHaveLength(1);
+ expect(turns[0]?.properties).toMatchObject({
+ turn_id: first.turnId,
+ status: outcome,
+ credits: 7,
+ });
+ expect(
+ events.find((event) => event.event === "$ai_trace")?.properties,
+ ).toMatchObject({ $ai_trace_id: first.turnId, status: outcome });
+ expect(second.reported).toBe(false);
+ },
+ );
+});
diff --git a/src/server/features/sam/samTurnTelemetry.ts b/src/server/features/sam/samTurnTelemetry.ts
new file mode 100644
index 00000000..85c9016a
--- /dev/null
+++ b/src/server/features/sam/samTurnTelemetry.ts
@@ -0,0 +1,357 @@
+import type {
+ ChatErrorContext,
+ StepContext,
+ ToolCallResultContext,
+} from "@cloudflare/think";
+import type { UIMessage } from "ai";
+import { captureServerError, captureServerEvent } from "@/server/lib/posthog";
+
+type SamTurnStatus =
+ | "completed"
+ | "aborted"
+ | "error"
+ | "refused"
+ | "interrupted";
+
+type SamRefusalReason = "no_session" | "credits" | "no_access";
+
+// Tool failures reach the model as a `{ error }` output rather than a thrown
+// error (samChatTools adaptMcpTool / scrapeTools), so Think reports them as
+// successful calls; look at the output shape as well as the outcome.
+function toolCallFailed(ctx: ToolCallResultContext): boolean {
+ if (!ctx.success) return true;
+ const output: unknown = ctx.output;
+ return (
+ typeof output === "object" &&
+ output !== null &&
+ "error" in output &&
+ typeof output.error === "string"
+ );
+}
+
+/**
+ * Everything one SAM turn does, accumulated from Think's hooks and flattened
+ * into the `sam:turn` event at the end. One instance per turn; `turnId` is the
+ * PostHog LLM-analytics trace id, the `turn_id` on every tool call the turn
+ * makes, and the join key between the two.
+ */
+export class SamTurnStats {
+ readonly turnId = crypto.randomUUID();
+ readonly startedAt = Date.now();
+ private lastStepEndedAt = this.startedAt;
+ continuation = false;
+ refusal: SamRefusalReason | null = null;
+ reported = false;
+ model: string | null = null;
+ steps = 0;
+ inputTokens = 0;
+ outputTokens = 0;
+ reasoningTokens = 0;
+ cachedInputTokens = 0;
+ costUsd = 0;
+ credits = 0;
+ compactions = 0;
+ toolCalls = 0;
+ toolFailures = 0;
+ toolMs = 0;
+ private readonly tools = new Set<string>();
+
+ /**
+ * Fold one inference step in and return its `$ai_generation` body. Cost is
+ * recorded separately (recordSpend) because compaction summaries spend
+ * outside the step loop.
+ */
+ recordStep(ctx: StepContext, costUsd: number): Record<string, unknown> {
+ const now = Date.now();
+ const latencySeconds = (now - this.lastStepEndedAt) / 1000;
+ this.lastStepEndedAt = now;
+ const { usage } = ctx;
+ const reasoning =
+ usage.outputTokenDetails.reasoningTokens ?? usage.reasoningTokens ?? 0;
+ const cached =
+ usage.inputTokenDetails.cacheReadTokens ?? usage.cachedInputTokens ?? 0;
+ this.model = ctx.response.modelId;
+ this.steps += 1;
+ this.inputTokens += usage.inputTokens ?? 0;
+ this.outputTokens += usage.outputTokens ?? 0;
+ this.reasoningTokens += reasoning;
+ this.cachedInputTokens += cached;
+ return {
+ $ai_trace_id: this.turnId,
+ $ai_parent_id: this.turnId,
+ $ai_span_id: crypto.randomUUID(),
+ $ai_span_name: `step ${this.steps}`,
+ $ai_provider: "openrouter",
+ $ai_model: ctx.response.modelId,
+ $ai_input_tokens: usage.inputTokens ?? 0,
+ $ai_output_tokens: usage.outputTokens ?? 0,
+ $ai_reasoning_tokens: reasoning,
+ $ai_cache_read_input_tokens: cached,
+ $ai_total_cost_usd: costUsd,
+ $ai_latency: latencySeconds,
+ $ai_is_error: false,
+ $ai_tools_called: ctx.toolCalls.map((call) => call.toolName).join(","),
+ finish_reason: ctx.finishReason,
+ };
+ }
+
+ recordToolCall(ctx: ToolCallResultContext): void {
+ this.toolCalls += 1;
+ this.toolMs += ctx.toolExecutionMs;
+ this.tools.add(ctx.toolName);
+ if (toolCallFailed(ctx)) this.toolFailures += 1;
+ }
+
+ recordSpend(costUsd: number): void {
+ this.costUsd += costUsd;
+ }
+
+ /** The `sam:turn` event body. */
+ properties(status: SamTurnStatus): Record<string, unknown> {
+ return {
+ turn_id: this.turnId,
+ status,
+ refusal_reason: this.refusal,
+ continuation: this.continuation,
+ model: this.model,
+ steps: this.steps,
+ tool_calls: this.toolCalls,
+ tool_failures: this.toolFailures,
+ tools: [...this.tools],
+ tool_ms: this.toolMs,
+ input_tokens: this.inputTokens,
+ output_tokens: this.outputTokens,
+ reasoning_tokens: this.reasoningTokens,
+ cached_input_tokens: this.cachedInputTokens,
+ cost_usd: this.costUsd,
+ credits: this.credits,
+ compactions: this.compactions,
+ duration_ms: Date.now() - this.startedAt,
+ };
+ }
+
+ /** The `$ai_trace` event body: what PostHog's LLM-analytics UI groups by. */
+ traceProperties(
+ status: SamTurnStatus,
+ error?: string,
+ ): Record<string, unknown> {
+ return {
+ $ai_trace_id: this.turnId,
+ $ai_span_name: "sam_turn",
+ $ai_provider: "openrouter",
+ $ai_model: this.model,
+ $ai_latency: (Date.now() - this.startedAt) / 1000,
+ $ai_is_error: status === "error",
+ $ai_error: error,
+ status,
+ };
+ }
+}
+
+// The slice of the DO's session context telemetry needs: who to attribute
+// events to and which org/project they belong to.
+type SamIdentity = {
+ row: { userId: string };
+ project: { id: string; organizationId: string };
+};
+
+function skillName(input: unknown): string | undefined {
+ return typeof input === "object" &&
+ input !== null &&
+ "name" in input &&
+ typeof input.name === "string"
+ ? input.name
+ : undefined;
+}
+
+/**
+ * The SAM Durable Object's telemetry seam: owns the turn in flight and turns
+ * Think's hooks into PostHog events (`sam:turn`, `$ai_generation`/`$ai_trace`
+ * for the LLM-analytics UI, `sam:skill_activated`) and error captures. Every
+ * event carries `source: "in_app_agent"` plus session/project ids so it
+ * filters the same way as the mcp:tool_call events the turn's tools emit.
+ *
+ * Captures go through `waitUntil`, not a bare void: the PostHog client
+ * flushes on shutdown, and a fire-and-forget promise on a turn's last step
+ * can be cancelled before that flush happens.
+ */
+export class SamTelemetry {
+ private current: SamTurnStats | null = null;
+
+ constructor(
+ private readonly sessionId: () => string,
+ // Resolved lazily (and cached by the caller): a memory-limit kill wakes a
+ // fresh DO whose session row is not loaded when recovery reports.
+ private readonly identity: () => Promise<SamIdentity | null>,
+ private readonly waitUntil: (promise: Promise<unknown>) => void,
+ ) {}
+
+ get turn(): SamTurnStats | null {
+ return this.current;
+ }
+
+ beginTurn(continuation: boolean): SamTurnStats {
+ const turn = new SamTurnStats();
+ turn.continuation = continuation;
+ this.current = turn;
+ return turn;
+ }
+
+ spend(costUsd: number): void {
+ this.current?.recordSpend(costUsd);
+ }
+
+ compaction(): void {
+ if (this.current) this.current.compactions += 1;
+ }
+
+ step(ctx: StepContext, costUsd: number): void {
+ const turn = this.current;
+ // Refusals answer through the canned model: no provider call to trace.
+ if (!turn || turn.refusal) return;
+ this.waitUntil(
+ this.capture("$ai_generation", turn.recordStep(ctx, costUsd)),
+ );
+ }
+
+ // Skill activations are Think-internal tools (activate_skill), so they never
+ // pass through the MCP instrumentation that reports every other SAM tool
+ // call; mirror its event shape so both land in the same dashboards.
+ toolCall(ctx: ToolCallResultContext): void {
+ this.current?.recordToolCall(ctx);
+ if (ctx.toolName !== "activate_skill") return;
+ this.waitUntil(
+ this.capture("sam:skill_activated", {
+ skill: skillName(ctx.input),
+ success: ctx.success,
+ duration_ms: ctx.durationMs,
+ turn_id: this.current?.turnId,
+ }),
+ );
+ }
+
+ /**
+ * Emit the turn's `sam:turn` event (and its `$ai_trace` when a model ran),
+ * once per turn. Think fires onChatError before the response hook for a
+ * failed turn, so the error path reports and the response hook then finds
+ * the turn already reported. `synthesize` is for turns that end with no
+ * armed stats — a parse/persist failure before beforeTurn, or a memory-
+ * limit kill that reset the DO — which are reported from an empty stats
+ * object so they still count. A gate refusal completes like any other
+ * turn (the canned model streams it) and is reported as "refused".
+ */
+ async report(
+ outcome: SamTurnStatus,
+ extra: Record<string, unknown>,
+ messages: readonly UIMessage[],
+ {
+ synthesize = false,
+ billing,
+ }: { synthesize?: boolean; billing?: Promise<void> } = {},
+ ): Promise<void> {
+ let turn = this.current;
+ if (!turn || turn.reported) {
+ if (!synthesize) return;
+ turn = new SamTurnStats();
+ }
+ turn.reported = true;
+ const status: SamTurnStatus =
+ outcome === "completed" && turn.refusal ? "refused" : outcome;
+ const userMessages = messages.filter((m) => m.role === "user").length;
+ const properties = {
+ ...turn.properties(status),
+ message_count: messages.length,
+ first_turn: userMessages <= 1,
+ ...extra,
+ credits: turn.credits,
+ };
+ const error = extra.error_message;
+ const trace =
+ turn.steps > 0
+ ? turn.traceProperties(
+ status,
+ typeof error === "string" ? error : undefined,
+ )
+ : null;
+ // Reserve this turn and snapshot its timing before billing settles. The
+ // next turn can start while the final credit deduction is still pending.
+ await billing;
+ properties.credits = turn.credits;
+ console.log("[sam] turn", { session_id: this.sessionId(), ...properties });
+ await this.capture("sam:turn", properties);
+ if (trace) {
+ await this.capture("$ai_trace", trace);
+ }
+ }
+
+ error(
+ error: unknown,
+ ctx: ChatErrorContext | undefined,
+ messages: readonly UIMessage[],
+ billing?: Promise<void>,
+ ): void {
+ const errorName = error instanceof Error ? error.name : typeof error;
+ const errorMessage = error instanceof Error ? error.message : String(error);
+ this.waitUntil(
+ this.report(
+ "error",
+ {
+ request_id: ctx?.requestId,
+ error_stage: ctx?.stage,
+ error_class: ctx?.classification,
+ error_name: errorName,
+ error_message: errorMessage.slice(0, 500),
+ },
+ messages,
+ { synthesize: true, billing },
+ ),
+ );
+ // A user-cancelled turn is an abort, not a failure worth an issue.
+ if (errorName !== "AbortError") {
+ this.captureError(error, {
+ stage: ctx?.stage,
+ classification: ctx?.classification,
+ });
+ }
+ }
+
+ /** Report a server-side failure to PostHog error tracking. */
+ captureError(
+ error: unknown,
+ properties: Record<string, string | undefined>,
+ ): void {
+ this.waitUntil(
+ this.identity().then((identity) =>
+ captureServerError(
+ error,
+ {
+ source: "sam",
+ session_id: this.sessionId(),
+ turn_id: this.current?.turnId,
+ ...properties,
+ },
+ identity?.row.userId,
+ ),
+ ),
+ );
+ }
+
+ private async capture(
+ event: string,
+ properties: Record<string, unknown>,
+ ): Promise<void> {
+ const identity = await this.identity();
+ if (!identity) return;
+ await captureServerEvent({
+ distinctId: identity.row.userId,
+ event,
+ organizationId: identity.project.organizationId,
+ properties: {
+ source: "in_app_agent",
+ session_id: this.sessionId(),
+ project_id: identity.project.id,
+ ...properties,
+ },
+ });
+ }
+}
diff --git a/src/server/mcp/context.ts b/src/server/mcp/context.ts
index 795ab1d4..c5f7c330 100644
--- a/src/server/mcp/context.ts
+++ b/src/server/mcp/context.ts
@@ -26,6 +26,9 @@ export type ToolAuthContext = {
export type ToolContext = {
auth: ToolAuthContext;
+ // The in-app agent's turn id, so a tool call's telemetry can be joined to
+ // the `sam:turn` event of the turn that made it. Absent for MCP transports.
+ turnId?: string;
};
export const MCP_AUTH_CONTEXT_PROP = "openSeoAuth";
diff --git a/src/server/mcp/instrumentation.ts b/src/server/mcp/instrumentation.ts
index d4745ff1..235bf9ab 100644
--- a/src/server/mcp/instrumentation.ts
+++ b/src/server/mcp/instrumentation.ts
@@ -56,6 +56,7 @@ function captureMcpToolCall(
error_code: outcome.errorCode,
client_id: auth.clientId,
source: auth.clientId ? "mcp_client" : "in_app_agent",
+ turn_id: context.turnId,
duration_ms: outcome.durationMs,
project_id: outcome.projectId,
row_count: outcome.rowCount,
diff --git a/src/serverFunctions/sam.ts b/src/serverFunctions/sam.ts
index 4a05d986..8768c873 100644
--- a/src/serverFunctions/sam.ts
+++ b/src/serverFunctions/sam.ts
@@ -1,10 +1,12 @@
import { createServerFn } from "@tanstack/react-start";
+import { waitUntil } from "cloudflare:workers";
import { z } from "zod";
import {
requireAuthenticatedContext,
requireProjectContext,
} from "@/serverFunctions/middleware";
import { AppError } from "@/server/lib/errors";
+import { captureServerEvent } from "@/server/lib/posthog";
import { SamSessionRepository } from "@/server/features/sam/SamSessionRepository";
import { ProjectRepository } from "@/server/features/projects/repositories/ProjectRepository";
@@ -36,6 +38,14 @@ export const createSamSession = createServerFn({ method: "POST" })
if (!session) {
throw new AppError("INTERNAL_ERROR", "Failed to create chat session");
}
+ waitUntil(
+ captureServerEvent({
+ distinctId: context.userId,
+ event: "sam:session_create",
+ organizationId: context.organizationId,
+ properties: { project_id: context.projectId, session_id: session.id },
+ }),
+ );
return { id: session.id };
});
@@ -64,5 +74,13 @@ export const archiveSamSession = createServerFn({ method: "POST" })
throw new AppError("NOT_FOUND", "Chat session not found");
}
await SamSessionRepository.archiveSession(data.sessionId);
+ waitUntil(
+ captureServerEvent({
+ distinctId: context.userId,
+ event: "sam:session_archive",
+ organizationId: context.organizationId,
+ properties: { project_id: project.id, session_id: session.id },
+ }),
+ );
return { ok: true };
});