Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(agents): trace a prompt from send to first agent output (#7541)

Wolf Mermelstein586d14fbd554
diff --git a/apps/web/src/features/block-agent/context/pending-session.ts b/apps/web/src/features/block-agent/context/pending-session.ts
index 8caede6523..48c5fd43c1 100644
--- a/apps/web/src/features/block-agent/context/pending-session.ts
+++ b/apps/web/src/features/block-agent/context/pending-session.ts
@@ -22,6 +22,7 @@
import { handleAiUsageLimitError } from '@app/features/paywall/ai-usage-limit-handling';
import { AgentSession } from '@core/agent-session/AgentSession';
+import { PromptTrace } from '@core/agent-session/prompt-telemetry';
import { refetchSoupEntity } from '@queries/soup/normalized-cache';
import { agentHarnessServiceClient } from '@service-agent-harness/client';
import type {
@@ -94,8 +95,19 @@ export function startPendingSession(
options: StartPendingSessionOptions = {}
): string {
const id = uuidv7();
+ const prompt = options.prompt?.trim() ?? '';
+ // Started before the create so the whole wait up to the first output,
+ // and every request on the way, lands in one trace.
+ const trace =
+ prompt || options.attachments?.length
+ ? new PromptTrace(id, { newSession: true })
+ : undefined;
const [sessionId, setSessionId] = createSignal<string>();
const [error, setError] = createSignal<string>();
+ const fail = (message: string, cause?: unknown) => {
+ trace?.end('failed', cause ?? new Error(message));
+ setError(message);
+ };
pending.set(id, {
sessionId,
failed: () => error() !== undefined,
@@ -104,107 +116,119 @@ export function startPendingSession(
initialInput: options.initialInput,
});
- void agentHarnessServiceClient
- .create({
- id,
- ...(options.botId ? { botId: options.botId } : {}),
- ...(options.modelOverride ? { model: options.modelOverride } : {}),
- ...(options.instructions ? { instructions: options.instructions } : {}),
- ...(options.repoUrl
- ? { repoUrl: options.repoUrl, repoBranch: options.repoBranch }
- : {}),
- } satisfies CreateAgentSessionRequest)
- .then(async (result) => {
- if (result.isErr()) {
- handleAiUsageLimitError(result.error);
- setError(
- result.error.map((error) => error.message).join(' ') ||
- 'The agent session could not be created.'
- );
- return;
- }
- // Normally the id this tab minted; an older service may mint its own.
- const created = result.value.session.id;
- void refetchSoupEntity(created, 'agentSession', { created: true });
- // Hold the block in preflight while selected settings are confirmed,
- // then adopt the session before issuing the first prompt so that prompt
- // is folded speculatively while its POST is in flight.
- const prompt = options.prompt?.trim() ?? '';
- if (
- options.modelOverride ||
- options.effortOverride ||
- prompt ||
- options.attachments?.length
- ) {
- const session = AgentSession.acquire(created);
- try {
- if (options.modelOverride || options.effortOverride) {
- await session.load();
- if (options.modelOverride) {
- await confirmSessionControl(session, {
- type: 'setModel',
- model: options.modelOverride,
- });
+ const traced = <T>(operation: () => T): T =>
+ trace ? trace.run(operation) : operation();
+
+ void traced(() =>
+ agentHarnessServiceClient
+ .create({
+ id,
+ ...(options.botId ? { botId: options.botId } : {}),
+ ...(options.modelOverride ? { model: options.modelOverride } : {}),
+ ...(options.instructions ? { instructions: options.instructions } : {}),
+ ...(options.repoUrl
+ ? { repoUrl: options.repoUrl, repoBranch: options.repoBranch }
+ : {}),
+ } satisfies CreateAgentSessionRequest)
+ .then(async (result) => {
+ if (result.isErr()) {
+ handleAiUsageLimitError(result.error);
+ fail(
+ result.error.map((error) => error.message).join(' ') ||
+ 'The agent session could not be created.'
+ );
+ return;
+ }
+ trace?.stage('created');
+ // Normally the id this tab minted; an older service may mint its own.
+ const created = result.value.session.id;
+ void refetchSoupEntity(created, 'agentSession', { created: true });
+ // Hold the block in preflight while selected settings are confirmed,
+ // then adopt the session before issuing the first prompt so that prompt
+ // is folded speculatively while its POST is in flight.
+ if (
+ options.modelOverride ||
+ options.effortOverride ||
+ prompt ||
+ options.attachments?.length
+ ) {
+ const session = AgentSession.acquire(created);
+ try {
+ if (options.modelOverride || options.effortOverride) {
+ await session.load();
+ trace?.stage('loaded');
+ if (options.modelOverride) {
+ await confirmSessionControl(session, {
+ type: 'setModel',
+ model: options.modelOverride,
+ });
+ }
+ if (options.effortOverride) {
+ const snapshot = await session.snapshot();
+ const effort = effortConfigOption(
+ snapshot.metadata.configOptions
+ );
+ if (
+ effort?.id !== options.effortOverride.configId ||
+ !effort.options.some(
+ (option) => option.value === options.effortOverride?.value
+ )
+ ) {
+ throw new Error(
+ 'The selected effort is no longer available for this model.'
+ );
+ }
+ await confirmSessionControl(session, {
+ type: 'setConfigOption',
+ ...options.effortOverride,
+ });
+ }
+ trace?.stage('configured');
}
- if (options.effortOverride) {
- const snapshot = await session.snapshot();
- const effort = effortConfigOption(
- snapshot.metadata.configOptions
+ setSessionId(created);
+ if (prompt || options.attachments?.length) {
+ const delivered = await issueSessionAction(
+ session,
+ {
+ type: 'prompt',
+ prompt,
+ ...(options.attachments?.length
+ ? { attachments: options.attachments }
+ : {}),
+ },
+ {
+ userId: options.userId ?? result.value.session.ownerId,
+ trace,
+ }
);
- if (
- effort?.id !== options.effortOverride.configId ||
- !effort.options.some(
- (option) => option.value === options.effortOverride?.value
- )
- ) {
- throw new Error(
- 'The selected effort is no longer available for this model.'
+ if (delivered.isErr()) {
+ setError(
+ delivered.error.map((error) => error.message).join(' ') ||
+ 'The first message could not be sent.'
);
}
- await confirmSessionControl(session, {
- type: 'setConfigOption',
- ...options.effortOverride,
- });
}
- }
- setSessionId(created);
- if (prompt || options.attachments?.length) {
- const delivered = await issueSessionAction(
- session,
- {
- type: 'prompt',
- prompt,
- ...(options.attachments?.length
- ? { attachments: options.attachments }
- : {}),
- },
- { userId: options.userId ?? result.value.session.ownerId }
+ } catch (error) {
+ fail(
+ error instanceof Error
+ ? error.message
+ : 'The selected settings could not be applied.',
+ error
);
- if (delivered.isErr()) {
- setError(
- delivered.error.map((error) => error.message).join(' ') ||
- 'The first message could not be sent.'
- );
- }
+ } finally {
+ session.release();
}
- } catch (error) {
- setError(
- error instanceof Error
- ? error.message
- : 'The selected settings could not be applied.'
- );
- } finally {
- session.release();
+ } else {
+ setSessionId(created);
}
- } else {
- setSessionId(created);
- }
- })
- .catch(() =>
- setError(
- 'Could not reach the agent service. Check your connection and try again.'
+ })
+ .catch((error: unknown) =>
+ fail(
+ 'Could not reach the agent service. Check your connection and try again.',
+ error
+ )
)
- );
+ );
return id;
}
diff --git a/apps/web/src/features/block-agent/context/resolve-session-id.test.ts b/apps/web/src/features/block-agent/context/resolve-session-id.test.ts
index b3873a4c66..19b1f240f4 100644
--- a/apps/web/src/features/block-agent/context/resolve-session-id.test.ts
+++ b/apps/web/src/features/block-agent/context/resolve-session-id.test.ts
@@ -112,6 +112,7 @@ vi.mock('@core/agent-session/AgentSession', () => ({
}));
const { startPendingSession } = await import('./pending-session');
+const { PromptTrace } = await import('@core/agent-session/prompt-telemetry');
const { agentHarnessServiceClient } = await import(
'@service-agent-harness/client'
);
@@ -462,6 +463,7 @@ it.each([undefined, 'explicit-user'])(
await flush();
expect(create.attribution).toHaveBeenCalledWith({
userId: userId ?? 'session-owner',
+ trace: expect.any(PromptTrace),
});
}
);
diff --git a/apps/web/src/lib/core/agent-session/AgentSession.test.ts b/apps/web/src/lib/core/agent-session/AgentSession.test.ts
index 8e5da9cf71..e9e44d0a46 100644
--- a/apps/web/src/lib/core/agent-session/AgentSession.test.ts
+++ b/apps/web/src/lib/core/agent-session/AgentSession.test.ts
@@ -52,6 +52,46 @@ const socket = vi.hoisted(() => ({
}),
}));
+type RecordedSpan = {
+ name: string;
+ attributes: Record<string, unknown>;
+ ends: number;
+};
+const telemetry = vi.hoisted(() => ({
+ spans: [] as RecordedSpan[],
+ /** The span whose `run` is on the stack, if any. */
+ active: undefined as RecordedSpan | undefined,
+}));
+vi.mock('@macro-inc/observability', () => ({
+ Telemetry: {
+ span: (name: string) => {
+ const record: RecordedSpan = { name, attributes: {}, ends: 0 };
+ telemetry.spans.push(record);
+ return {
+ setAttr: (key: string, value: unknown) => {
+ record.attributes[key] = value;
+ },
+ event: () => {},
+ error: () => {},
+ run: <T>(operation: () => T) => {
+ const outer = telemetry.active;
+ telemetry.active = record;
+ try {
+ return operation();
+ } finally {
+ telemetry.active = outer;
+ }
+ },
+ end: () => {
+ record.ends += 1;
+ },
+ };
+ },
+ },
+}));
+const promptSpans = () =>
+ telemetry.spans.filter((span) => span.name === 'agent.prompt');
+
const updates = vi.hoisted(() => ({
listeners: new Set<(event: { agentSessionId: string }) => void>(),
}));
@@ -128,6 +168,7 @@ function deferred<T>() {
beforeEach(() => {
vi.clearAllMocks();
resetSessionTurns();
+ telemetry.spans.length = 0;
socket.listeners.clear();
updates.listeners.clear();
// Instances are shared and refcounted, so a test that fails before its
@@ -856,4 +897,96 @@ describe('AgentSession', () => {
expect(logSource.watch).toHaveBeenCalledTimes(2);
live.release();
});
+
+ describe('prompt telemetry', () => {
+ it('posts a prompt inside its span and ends it at the first agent output', async () => {
+ let postedUnder: RecordedSpan | undefined;
+ harness.control.mockImplementation(
+ async (_id: string, request: { actionId: string }) => {
+ postedUnder = telemetry.active;
+ return ok({ actionId: request.actionId, status: 'sent' });
+ }
+ );
+ const live = AgentSession.acquire(SESSION);
+ await live.load();
+
+ await live.issue(
+ { type: 'prompt', prompt: 'hi' },
+ { userId: 'macro|[email protected]' }
+ );
+ await settle();
+ const actionId = (
+ harness.control.mock.calls[0][1] as { actionId: string }
+ ).actionId;
+ expect(promptSpans()).toHaveLength(1);
+ const [span] = promptSpans();
+ expect(postedUnder).toBe(span);
+ expect(span.ends).toBe(0);
+
+ fold.pushSession.mockResolvedValueOnce([
+ {
+ kind: 'update',
+ message: {
+ agentSessionId: SESSION,
+ turn: 1,
+ author: { kind: 'user', userId: 'macro|[email protected]' },
+ requestId: actionId,
+ parts: [{ kind: 'text', text: 'hi' }],
+ stop: null,
+ pending: false,
+ },
+ },
+ {
+ kind: 'new',
+ message: {
+ agentSessionId: SESSION,
+ turn: 1,
+ author: { kind: 'agent' },
+ requestId: null,
+ parts: [{ kind: 'thought', text: 'thinking' }],
+ stop: null,
+ pending: false,
+ },
+ },
+ ] satisfies FoldedStreamEvent[]);
+ AgentSession.ingest({ agentSessionId: SESSION, entries: [row(2)] });
+ await settle();
+
+ expect(span.ends).toBe(1);
+ expect(span.attributes).toMatchObject({
+ 'agent.session.id': SESSION,
+ 'agent.prompt.new_session': false,
+ 'agent.prompt.action_id': actionId,
+ 'agent.prompt.turn': 1,
+ 'agent.prompt.first_output_part': 'thought',
+ 'agent.prompt.outcome': 'output',
+ });
+ live.release();
+ expect(span.ends).toBe(1);
+ expect(span.attributes['agent.prompt.outcome']).toBe('output');
+ });
+
+ it('traces no action but a prompt', async () => {
+ const live = AgentSession.acquire(SESSION);
+ await live.load();
+
+ await live.issue({ type: 'stop' });
+ await live.issue({ type: 'setModel', model: 'a-model' });
+
+ expect(promptSpans()).toEqual([]);
+ live.release();
+ });
+
+ it('ends a prompt still waiting for output as released', async () => {
+ const live = AgentSession.acquire(SESSION);
+ await live.load();
+ await live.issue({ type: 'prompt', prompt: 'hi' });
+
+ live.release();
+
+ const [span] = promptSpans();
+ expect(span.ends).toBe(1);
+ expect(span.attributes['agent.prompt.outcome']).toBe('released');
+ });
+ });
});
diff --git a/apps/web/src/lib/core/agent-session/AgentSession.ts b/apps/web/src/lib/core/agent-session/AgentSession.ts
index 8be7e0e3ec..1beee2ce2d 100644
--- a/apps/web/src/lib/core/agent-session/AgentSession.ts
+++ b/apps/web/src/lib/core/agent-session/AgentSession.ts
@@ -54,6 +54,7 @@ import {
traceAcquire,
traceCacheWrite,
} from './load-telemetry';
+import { PromptTrace } from './prompt-telemetry';
import { publishSessionTurn } from './session-turn';
export type AgentSessionListener = (events: FoldedStreamEvent[]) => void;
@@ -206,6 +207,8 @@ export class AgentSession {
* wait in the server's queue.
*/
private turn: TurnState = 'idle';
+ /** Prompts issued here still waiting for their first output. */
+ private readonly prompts = new Set<PromptTrace>();
private setTurn(turn: TurnState | undefined): void {
const next = turn ?? 'idle';
@@ -273,13 +276,18 @@ export class AgentSession {
* nowhere else.
*
* `userId` is the caller, so the pending bubble is attributed exactly as
- * the confirmed row will be.
+ * the confirmed row will be. A prompt is traced until its first output,
+ * under `trace` when the caller started one earlier.
*/
async issue(
action: AgentAction,
- options: { userId?: string } = {}
+ options: { userId?: string; trace?: PromptTrace } = {}
): Promise<IssueResult> {
const actionId = uuidv7();
+ const trace =
+ action.type === 'prompt'
+ ? (options.trace ?? new PromptTrace(this.id, { newSession: false }))
+ : undefined;
const speculated = this.reaches(action);
if (speculated) {
// A prompt we just folded opens a turn, so the next one belongs in the
@@ -299,7 +307,9 @@ export class AgentSession {
// The harness adopts the id the client speculated under; the response
// names the id it was accepted under and `issue` reconciles the two.
const request: ControlRequest = { ...action, actionId };
- const result = await agentHarnessServiceClient.control(this.id, request);
+ const result = trace
+ ? await this.sendPrompt(trace, actionId, request)
+ : await agentHarnessServiceClient.control(this.id, request);
if (!speculated) return result;
if (result.isErr()) {
@@ -330,6 +340,41 @@ export class AgentSession {
return result;
}
+ /**
+ * Post a prompt inside its trace, and watch the fold for its output.
+ * Watched from before the POST: the confirmed row can reach the socket
+ * before the POST's answer does.
+ */
+ private async sendPrompt(
+ trace: PromptTrace,
+ actionId: string,
+ request: ControlRequest
+ ): Promise<IssueResult> {
+ trace.expect(actionId);
+ this.prompts.add(trace);
+ try {
+ const result = await trace.run(() =>
+ agentHarnessServiceClient.control(this.id, request)
+ );
+ if (result.isErr()) {
+ trace.end(
+ 'failed',
+ new Error(result.error.map((error) => error.message).join(' '))
+ );
+ } else {
+ trace.accepted(result.value.actionId, result.value.status === 'queued');
+ }
+ return result;
+ } catch (error) {
+ trace.end('failed', error);
+ throw error;
+ } finally {
+ // Released while the POST was out: nothing will observe the turn.
+ if (this.closed) trace.end('released');
+ if (trace.ended) this.prompts.delete(trace);
+ }
+ }
+
/**
* Show an action the server already holds as if it had dispatched. Nothing
* is sent: the server queued this action earlier under `actionId`, and the
@@ -420,6 +465,8 @@ export class AgentSession {
// Ended here rather than where the load notices: a fetch that never
// answers never reaches that check, and an unended span never reports.
this.trace.end('released');
+ for (const prompt of this.prompts) prompt.end('released');
+ this.prompts.clear();
this.listeners.clear();
this.unsubscribeSocket();
this.unsubscribeUpdated();
@@ -664,6 +711,10 @@ export class AgentSession {
const metadata = events.findLast((event) => event.kind === 'metadata');
if (metadata) this.setTurn(metadata.metadata.turn);
for (const listener of this.listeners) listener(events);
+ for (const prompt of this.prompts) {
+ prompt.observe(events);
+ if (prompt.ended) this.prompts.delete(prompt);
+ }
});
// A failed push must not poison the chain for every input after it.
this.chain = run.catch((error: unknown) => {
diff --git a/apps/web/src/lib/core/agent-session/prompt-telemetry.test.ts b/apps/web/src/lib/core/agent-session/prompt-telemetry.test.ts
new file mode 100644
index 0000000000..3afbcd8fbe
--- /dev/null
+++ b/apps/web/src/lib/core/agent-session/prompt-telemetry.test.ts
@@ -0,0 +1,201 @@
+/**
+ * The `agent.prompt` span read off literal fold events: the confirmed user
+ * message fixes the turn, the first agent message in it ends the span.
+ */
+
+import type { FoldedStreamEvent } from '@service-agent-fold/generated/types';
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+type RecordedSpan = {
+ name: string;
+ attributes: Record<string, unknown>;
+ events: { name: string; attributes?: Record<string, unknown> }[];
+ errors: unknown[];
+ ends: number;
+};
+
+const telemetry = vi.hoisted(() => ({ spans: [] as RecordedSpan[] }));
+
+vi.mock('@macro-inc/observability', () => ({
+ Telemetry: {
+ span: (name: string) => {
+ const record: RecordedSpan = {
+ name,
+ attributes: {},
+ events: [],
+ errors: [],
+ ends: 0,
+ };
+ telemetry.spans.push(record);
+ return {
+ setAttr: (key: string, value: unknown) => {
+ record.attributes[key] = value;
+ },
+ event: (eventName: string, attributes?: Record<string, unknown>) => {
+ record.events.push({ name: eventName, attributes });
+ },
+ error: (error: unknown) => {
+ record.errors.push(error);
+ },
+ run: <T>(operation: () => T) => operation(),
+ end: () => {
+ record.ends += 1;
+ },
+ };
+ },
+ },
+}));
+
+import { PromptTrace } from './prompt-telemetry';
+
+const SESSION = '01a0abed-279f-724c-9f49-60dbedc79b6e';
+const ACTION = '01a0abed-3000-7000-8000-000000000001';
+
+beforeEach(() => {
+ telemetry.spans.length = 0;
+});
+
+afterEach(() => {
+ vi.useRealTimers();
+});
+
+describe('PromptTrace', () => {
+ it('records the prompt from acceptance through confirmation to the first agent output', () => {
+ const trace = new PromptTrace(SESSION, { newSession: false });
+ trace.expect(ACTION);
+ trace.accepted(ACTION, false);
+
+ const speculated: FoldedStreamEvent[] = [
+ {
+ kind: 'new',
+ message: {
+ agentSessionId: SESSION,
+ turn: 1,
+ author: { kind: 'user', userId: 'macro|[email protected]' },
+ requestId: ACTION,
+ parts: [{ kind: 'text', text: 'hi' }],
+ stop: null,
+ pending: true,
+ },
+ },
+ ];
+ trace.observe(speculated);
+ const confirmed: FoldedStreamEvent[] = [
+ {
+ kind: 'update',
+ message: {
+ agentSessionId: SESSION,
+ turn: 1,
+ author: { kind: 'user', userId: 'macro|[email protected]' },
+ requestId: ACTION,
+ parts: [{ kind: 'text', text: 'hi' }],
+ stop: null,
+ pending: false,
+ },
+ },
+ ];
+ trace.observe(confirmed);
+ const output: FoldedStreamEvent[] = [
+ {
+ kind: 'new',
+ message: {
+ agentSessionId: SESSION,
+ turn: 1,
+ author: { kind: 'agent' },
+ requestId: null,
+ parts: [{ kind: 'text', text: 'hello' }],
+ stop: null,
+ pending: false,
+ },
+ },
+ ];
+ trace.observe(output);
+ trace.observe(output);
+
+ expect(telemetry.spans).toHaveLength(1);
+ const [span] = telemetry.spans;
+ expect(span.name).toBe('agent.prompt');
+ expect(span.ends).toBe(1);
+ expect(span.attributes).toMatchObject({
+ 'agent.session.id': SESSION,
+ 'agent.prompt.new_session': false,
+ 'agent.prompt.action_id': ACTION,
+ 'agent.prompt.turn': 1,
+ 'agent.prompt.first_output_part': 'text',
+ 'agent.prompt.outcome': 'output',
+ });
+ expect(span.attributes).not.toHaveProperty('agent.prompt.queued');
+ for (const stage of ['accepted', 'confirmed', 'first_output']) {
+ expect(span.attributes[`agent.prompt.${stage}_at_ms`]).toEqual(
+ expect.any(Number)
+ );
+ }
+ expect(span.attributes['agent.prompt.total_ms']).toEqual(
+ expect.any(Number)
+ );
+ expect(span.events.map((event) => event.name)).toEqual([
+ 'agent.prompt.accepted',
+ 'agent.prompt.confirmed',
+ 'agent.prompt.first_output',
+ ]);
+ expect(trace.ended).toBe(true);
+ });
+
+ it('ends with no_output when the turn stops before the agent says anything', () => {
+ const trace = new PromptTrace(SESSION, { newSession: true });
+ trace.expect(ACTION);
+
+ trace.observe([
+ {
+ kind: 'replace',
+ messages: [
+ {
+ agentSessionId: SESSION,
+ turn: 0,
+ author: { kind: 'agent' },
+ requestId: null,
+ parts: [{ kind: 'text', text: 'an earlier turn' }],
+ stop: { kind: 'end_turn' },
+ pending: false,
+ },
+ {
+ agentSessionId: SESSION,
+ turn: 1,
+ author: { kind: 'user', userId: 'macro|[email protected]' },
+ requestId: ACTION,
+ parts: [{ kind: 'text', text: 'hi' }],
+ stop: { kind: 'cancelled' },
+ pending: false,
+ },
+ ],
+ },
+ ]);
+
+ const [span] = telemetry.spans;
+ expect(span.ends).toBe(1);
+ expect(span.attributes).toMatchObject({
+ 'agent.prompt.new_session': true,
+ 'agent.prompt.turn': 1,
+ 'agent.prompt.stop': 'cancelled',
+ 'agent.prompt.outcome': 'no_output',
+ });
+ expect(span.attributes).not.toHaveProperty(
+ 'agent.prompt.first_output_at_ms'
+ );
+ });
+
+ it('reports a prompt still waiting after five minutes as stalled', () => {
+ vi.useFakeTimers();
+ const trace = new PromptTrace(SESSION, { newSession: false });
+
+ vi.advanceTimersByTime(5 * 60_000 - 1);
+ expect(telemetry.spans[0].ends).toBe(0);
+ vi.advanceTimersByTime(1);
+
+ expect(trace.ended).toBe(true);
+ expect(telemetry.spans[0].ends).toBe(1);
+ expect(telemetry.spans[0].attributes['agent.prompt.outcome']).toBe(
+ 'stalled'
+ );
+ });
+});
diff --git a/apps/web/src/lib/core/agent-session/prompt-telemetry.ts b/apps/web/src/lib/core/agent-session/prompt-telemetry.ts
new file mode 100644
index 0000000000..f66dce840e
--- /dev/null
+++ b/apps/web/src/lib/core/agent-session/prompt-telemetry.ts
@@ -0,0 +1,213 @@
+/**
+ * Telemetry for sending a prompt: from the moment a person sends it to the
+ * first agent output on screen, as one span.
+ *
+ * Without it the flow is several unrelated traces (the create, the load, a
+ * config control, the prompt control, the server's turn) and the waits in the
+ * browser between them are invisible. Every harness request made inside
+ * {@link PromptTrace.run} carries this span's traceparent, so the server's
+ * turn nests under it.
+ *
+ * Failures here are swallowed. Telemetry that can break sending a prompt is
+ * worse than no telemetry.
+ */
+
+import type { Span } from '@macro-inc/observability';
+import { Telemetry } from '@macro-inc/observability';
+import type {
+ FoldedMessage,
+ FoldedStreamEvent,
+} from '@service-agent-fold/generated/types';
+import { match } from 'ts-pattern';
+
+/** How a sent prompt ended, as far as this span is concerned. */
+export type PromptOutcome =
+ /** The agent produced its first output for the prompt's turn. */
+ | 'output'
+ /** The turn ended before the agent produced anything. */
+ | 'no_output'
+ /** The prompt was never accepted: the create, a setting, or the POST failed. */
+ | 'failed'
+ /** Every holder released the session before any output arrived. */
+ | 'released'
+ /** Still waiting at {@link STALL_THRESHOLD_MS} — reported, not concluded. */
+ | 'stalled';
+
+/** A point the prompt reached, named as its attribute and event suffix. */
+export type PromptStage =
+ /** A new session's create answered. */
+ | 'created'
+ /** A new session's log was loaded. */
+ | 'loaded'
+ /** A new session's model and effort were confirmed. */
+ | 'configured'
+ /** The control POST answered. */
+ | 'accepted'
+ /** The log confirmed the prompt: its row came back over the socket. */
+ | 'confirmed'
+ /** The first agent message in the prompt's turn was folded. */
+ | 'first_output';
+
+/**
+ * How long a prompt may wait for output before it is reported as stalled.
+ * Long because a cold sandbox boot can take minutes, and a span only
+ * exports once it ends.
+ */
+const STALL_THRESHOLD_MS = 5 * 60_000;
+
+type Attributes = Record<string, string | number | boolean>;
+
+export class PromptTrace {
+ readonly #span: Span | undefined;
+ readonly #startedAt = performance.now();
+ /** Ids the prompt may be confirmed under: the speculated one and the accepted one. */
+ readonly #actionIds = new Set<string>();
+ /** The prompt's turn, once the log confirmed it. */
+ #turn: number | undefined;
+ #ended = false;
+ #stallTimer: ReturnType<typeof setTimeout> | undefined;
+
+ constructor(sessionId: string, options: { newSession: boolean }) {
+ this.#span = start(sessionId, options.newSession);
+ this.#stallTimer = setTimeout(
+ () => this.end('stalled'),
+ STALL_THRESHOLD_MS
+ );
+ }
+
+ get ended(): boolean {
+ return this.#ended;
+ }
+
+ /** Run `operation` with this span active, so its requests join the trace. */
+ run<T>(operation: () => T): T {
+ if (!this.#span) return operation();
+ let ran = false;
+ try {
+ return this.#span.run(() => {
+ ran = true;
+ return operation();
+ });
+ } catch (error) {
+ if (ran) throw error;
+ return operation();
+ }
+ }
+
+ stage(stage: PromptStage, attributes: Attributes = {}): void {
+ if (this.#ended) return;
+ const atMs = Math.round(performance.now() - this.#startedAt);
+ this.#set(`agent.prompt.${stage}_at_ms`, atMs);
+ for (const [name, value] of Object.entries(attributes)) {
+ this.#set(`agent.prompt.${name}`, value);
+ }
+ try {
+ this.#span?.event(`agent.prompt.${stage}`, {
+ at_ms: atMs,
+ ...attributes,
+ });
+ } catch {
+ // See the module comment.
+ }
+ }
+
+ /** The control will be sent under `actionId`; its confirmation may name it. */
+ expect(actionId: string): void {
+ this.#actionIds.add(actionId);
+ }
+
+ /** The control POST answered with the id the harness accepted it under. */
+ accepted(actionId: string, queued: boolean): void {
+ this.#actionIds.add(actionId);
+ this.#set('agent.prompt.action_id', actionId);
+ if (queued) this.#set('agent.prompt.queued', true);
+ this.stage('accepted');
+ }
+
+ /** Read fold events for the prompt's confirmation and its turn's first output. */
+ observe(events: FoldedStreamEvent[]): void {
+ if (this.#ended) return;
+ try {
+ for (const event of events) {
+ for (const message of messagesOf(event)) {
+ this.#observeMessage(message);
+ if (this.#ended) return;
+ }
+ }
+ } catch {
+ // See the module comment.
+ }
+ }
+
+ /** End the span. Safe to call more than once; the first outcome stands. */
+ end(outcome: PromptOutcome, error?: unknown): void {
+ if (this.#ended) return;
+ clearTimeout(this.#stallTimer);
+ this.#stallTimer = undefined;
+ this.#set('agent.prompt.outcome', outcome);
+ this.#set(
+ 'agent.prompt.total_ms',
+ Math.round(performance.now() - this.#startedAt)
+ );
+ this.#ended = true;
+ try {
+ if (outcome === 'failed' && error !== undefined) this.#span?.error(error);
+ this.#span?.end();
+ } catch {
+ // See the module comment.
+ }
+ }
+
+ #observeMessage(message: FoldedMessage): void {
+ if (
+ this.#turn === undefined &&
+ message.author.kind === 'user' &&
+ !message.pending &&
+ message.requestId !== null &&
+ this.#actionIds.has(message.requestId)
+ ) {
+ this.#turn = message.turn;
+ this.stage('confirmed', { turn: message.turn });
+ }
+ if (this.#turn === undefined || message.turn !== this.#turn) return;
+ if (message.author.kind === 'agent') {
+ this.stage('first_output', {
+ first_output_part: message.parts[0]?.kind ?? 'none',
+ });
+ this.end('output');
+ return;
+ }
+ if (message.stop) {
+ this.#set('agent.prompt.stop', message.stop.kind);
+ this.end('no_output');
+ }
+ }
+
+ #set(name: string, value: string | number | boolean): void {
+ if (this.#ended) return;
+ try {
+ this.#span?.setAttr(name, value);
+ } catch {
+ // See the module comment.
+ }
+ }
+}
+
+function messagesOf(event: FoldedStreamEvent): FoldedMessage[] {
+ return match(event)
+ .with({ kind: 'new' }, { kind: 'update' }, (changed) => [changed.message])
+ .with({ kind: 'replace' }, (replaced) => replaced.messages)
+ .with({ kind: 'metadata' }, () => [])
+ .exhaustive();
+}
+
+function start(sessionId: string, newSession: boolean): Span | undefined {
+ try {
+ const span = Telemetry.span('agent.prompt');
+ span.setAttr('agent.session.id', sessionId);
+ span.setAttr('agent.prompt.new_session', newSession);
+ return span;
+ } catch {
+ return undefined;
+ }
+}