Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(observability): fill agent.prompt telemetry gaps (#7744)
Wolf Mermelstein5dd48ee56760
diff --git a/apps/web/src/features/agents-view/queries/composer-models.ts b/apps/web/src/features/agents-view/queries/composer-models.tsindex a170c66623..5bdb2d368b 100644--- a/apps/web/src/features/agents-view/queries/composer-models.ts+++ b/apps/web/src/features/agents-view/queries/composer-models.ts@@ -54,6 +54,8 @@ export function createComposerModels(agent: Accessor<RosterAgent | undefined>) {return {models: () => data()?.models ?? [],currentModel: () => data()?.currentModel ?? undefined,+ /** Discovery is in flight, so {@link models} is empty for now, not for good. */+ pending: () => target() !== undefined && query.isPending,message: () => {if (!agent()?.runtime.connected)return 'Connect the runtime to load models.';diff --git a/apps/web/src/features/agents-view/views/NewChatPage.test.tsx b/apps/web/src/features/agents-view/views/NewChatPage.test.tsxindex feb709abf5..81c00d4abf 100644--- a/apps/web/src/features/agents-view/views/NewChatPage.test.tsx+++ b/apps/web/src/features/agents-view/views/NewChatPage.test.tsx@@ -23,6 +23,7 @@ import { NewChatPage } from './NewChatPage';const mocks = vi.hoisted(() => ({touch: false,freePlan: false,+ modelsPending: false,openSettings: vi.fn(),capabilitiesPending: false,attachments: [] as InputAttachmentData[],@@ -117,10 +118,10 @@ vi.mock('@queries/agents/models', () => ({enabled: () => boolean) => ({get isSuccess() {- return enabled();+ return enabled() && !mocks.modelsPending;},get isPending() {- return false;+ return mocks.modelsPending;},get isError() {return false;@@ -274,6 +275,7 @@ describe('agent-led new conversation', () => {beforeEach(() => {mocks.capabilitiesPending = false;mocks.freePlan = false;+ mocks.modelsPending = false;mocks.touch = false;mocks.attachments = [];mocks.recentIds = [MACRO_CODER_BOT_ID];@@ -371,6 +373,17 @@ describe('agent-led new conversation', () => {modelOverride: 'chat-default',});});+ it('flags a send before the model catalog loads as a model fallback', () => {+ mocks.modelsPending = true;+ const send = page();+ fireEvent.click(screen.getByRole('button', { name: 'Send' }));+ expect(send).toHaveBeenCalledWith({+ prompt: 'Prompt',+ botId: undefined,+ repoUrl: undefined,+ modelFallback: true,+ });+ });it('selects a coding agent, keeps the draft, and only sends the repository to coding agents', async () => {const send = page();fireEvent.input(screen.getByRole('textbox', { name: 'Draft' }), {diff --git a/apps/web/src/features/agents-view/views/NewChatPage.tsx b/apps/web/src/features/agents-view/views/NewChatPage.tsxindex e9e4c11419..6111432c1d 100644--- a/apps/web/src/features/agents-view/views/NewChatPage.tsx+++ b/apps/web/src/features/agents-view/views/NewChatPage.tsx@@ -44,6 +44,8 @@ export type StartConversation = {repoUrl?: string;repoBranch?: string;modelOverride?: string;+ /** The catalog was still loading, so an in-memory agent runs its persona default. */+ modelFallback?: boolean;effortOverride?: { configId: string; value: string };};@@ -219,6 +221,8 @@ export function NewChatPage(props: {const repo = canSelectRepository() ? repoUrl() : undefined;if (repo) repositories.remember(repo);const model = composerModelOverride();+ const inMemory =+ persona.harness === 'macro-inmem' || persona.harness === 'in-memory';props.onStart({prompt,...(attachments.length > 0@@ -228,6 +232,9 @@ export function NewChatPage(props: {repoUrl: repo,...(repo ? { repoBranch: repoBranch() } : {}),...(model ? { modelOverride: model } : {}),+ ...(inMemory && !model && selectedCatalog.pending()+ ? { modelFallback: true }+ : {}),effortOverride: effortOverride(),});attachmentTracker.clearAttachments();diff --git a/apps/web/src/features/block-agent/context/pending-session.ts b/apps/web/src/features/block-agent/context/pending-session.tsindex cb0cc3378e..8e061a56aa 100644--- a/apps/web/src/features/block-agent/context/pending-session.ts+++ b/apps/web/src/features/block-agent/context/pending-session.ts@@ -89,6 +89,8 @@ export type StartPendingSessionOptions = {submitSurface?: PromptSubmitSurface;/** Model to run on instead of the persona's, set as the session is created. */modelOverride?: string;+ /** The composer's model catalog was still loading, so the persona default runs. */+ modelFallback?: boolean;/*** Context the surface opening the session gives the agent: its runtime reads it as* instructions, so neither the composer nor the sent prompt shows it.@@ -114,7 +116,7 @@ export type StartPendingSessionOptions = {export function startPendingSession(options: StartPendingSessionOptions = {}): string {- const warmId = takeWarmAgentSession(options);+ const { id: warmId, claim: warmClaim } = takeWarmAgentSession(options);const id = warmId ?? uuidv7();const prompt = options.prompt?.trim() ?? '';// Started before the create so the whole wait up to the first output,@@ -124,6 +126,12 @@ export function startPendingSession(? new PromptTrace(id, {newSession: true,submitSurface: options.submitSurface,+ create: {+ warmClaim,+ modelOverride: options.modelOverride !== undefined,+ modelFallback: options.modelFallback ?? false,+ effort: options.effortOverride?.value,+ },}): undefined;const [sessionId, setSessionId] = createSignal<string>();diff --git a/apps/web/src/features/block-agent/context/resolve-session-id.test.ts b/apps/web/src/features/block-agent/context/resolve-session-id.test.tsindex 085a360b34..70d1e1a668 100644--- a/apps/web/src/features/block-agent/context/resolve-session-id.test.ts+++ b/apps/web/src/features/block-agent/context/resolve-session-id.test.ts@@ -5,11 +5,14 @@* in for one being created — resolved into the one the block consumes.*/+import type { WarmClaim } from '@queries/agent-session/warm';import type { AgentAction } from '@service-agent-harness/generated/schemas';import { createRoot } from 'solid-js';import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';-const takeWarm = vi.hoisted(() => vi.fn<() => string | undefined>());+const takeWarm = vi.hoisted(() =>+ vi.fn<() => { id?: string; claim: WarmClaim }>()+);const replenishWarm = vi.hoisted(() => vi.fn<(userId: string) => void>());vi.mock('@queries/agent-session/warm', () => ({takeWarmAgentSession: takeWarm,@@ -139,6 +142,7 @@ const flush = () => new Promise((resolve) => setTimeout(resolve, 0));beforeEach(() => {takeWarm.mockReset();+ takeWarm.mockReturnValue({ claim: 'miss_none_ready' });replenishWarm.mockReset();refetchSoupEntity.mockClear();create.control.mockReset();@@ -515,7 +519,7 @@ it.each([undefined, 'explicit-user'])();it('falls back to a cold session when claiming a warm session fails', async () => {- takeWarm.mockReturnValueOnce('stale-warm');+ takeWarm.mockReturnValueOnce({ id: 'stale-warm', claim: 'hit' });const placeholder = startPendingSession();expect(placeholder).toBe('stale-warm');create.reject?.();@@ -537,7 +541,10 @@ it('falls back to a cold session when claiming a warm session fails', async () =it.each([undefined, 'ready-warm'])('replenishes only after successful creation when the reservation is %s',async (warmId) => {- takeWarm.mockReturnValueOnce(warmId);+ takeWarm.mockReturnValueOnce({+ id: warmId,+ claim: warmId ? 'hit' : 'miss_none_ready',+ });startPendingSession({ userId: 'session-owner' });await flush();expect(replenishWarm).not.toHaveBeenCalled();@@ -548,7 +555,7 @@ it.each([undefined, 'ready-warm'])();it('does not replenish when both the warm claim and fallback creation fail', async () => {- takeWarm.mockReturnValueOnce('stale-warm');+ takeWarm.mockReturnValueOnce({ id: 'stale-warm', claim: 'hit' });const placeholder = startPendingSession({ userId: 'session-owner' });create.reject?.();await vi.waitFor(() =>diff --git a/apps/web/src/lib/core/agent-session/prompt-telemetry.test.ts b/apps/web/src/lib/core/agent-session/prompt-telemetry.test.tsindex e73669d01c..2b210bd26e 100644--- a/apps/web/src/lib/core/agent-session/prompt-telemetry.test.ts+++ b/apps/web/src/lib/core/agent-session/prompt-telemetry.test.ts@@ -176,6 +176,128 @@ describe('PromptTrace', () => {);});+ it('records how a new session was asked for', () => {+ const trace = new PromptTrace(SESSION, {+ newSession: true,+ create: {+ warmClaim: 'miss_model',+ modelOverride: true,+ modelFallback: false,+ effort: 'high',+ },+ });+ trace.end('released');+ expect(telemetry.spans[0].attributes).toMatchObject({+ 'agent.prompt.warm_claim': 'miss_model',+ 'agent.prompt.model_override_set': true,+ 'agent.prompt.model_fallback': false,+ 'agent.prompt.effort_override': true,+ 'agent.prompt.effort': 'high',+ });+ });++ it('records the input delay of the event that sent the prompt', () => {+ const button = document.createElement('button');+ let trace: PromptTrace | undefined;+ button.addEventListener('click', () => {+ trace = new PromptTrace(SESSION, { newSession: false });+ });+ const click = new MouseEvent('click');+ Object.defineProperty(click, 'timeStamp', {+ value: performance.now() - 40,+ });+ button.dispatchEvent(click);+ trace?.end('released');+ expect(telemetry.spans[0].attributes['agent.prompt.input_kind']).toBe(+ 'pointer'+ );+ expect(+ telemetry.spans[0].attributes['agent.prompt.input_delay_ms']+ ).toBeGreaterThanOrEqual(40);++ new PromptTrace(SESSION, { newSession: false }).end('released');+ expect(telemetry.spans[1].attributes).not.toHaveProperty(+ 'agent.prompt.input_delay_ms'+ );+ });++ it('records when the create and prompt responses arrived, apart from when they were handled', () => {+ const observers: ((entries: PerformanceEntry[]) => void)[] = [];+ vi.stubGlobal(+ 'PerformanceObserver',+ class {+ constructor(callback: (list: PerformanceObserverEntryList) => void) {+ observers.push((entries) =>+ callback({+ getEntries: () => entries,+ } as PerformanceObserverEntryList)+ );+ }+ observe() {}+ takeRecords() {+ return [];+ }+ disconnect() {}+ }+ );+ const now = vi.spyOn(performance, 'now').mockReturnValue(1000);+ const fetched = (path: string, startTime: number, responseEnd: number) =>+ ({+ entryType: 'resource',+ initiatorType: 'fetch',+ name: `https://harness.test${path}`,+ startTime,+ responseEnd,+ }) as PerformanceResourceTiming;+ const trace = new PromptTrace(SESSION, { newSession: true });+ expect(observers).toHaveLength(1);+ const [deliver] = observers;+ deliver([+ fetched('/agent-sessions/warm', 990, 1020),+ fetched('/agent-sessions', 1005, 1049),+ fetched(`/agent-sessions/${SESSION}/log`, 1050, 1100),+ ]);+ // The effort path's configure control left before the prompt's.+ deliver([fetched(`/agent-sessions/${SESSION}/control`, 1060, 1090)]);+ now.mockReturnValue(1100);+ trace.expect(ACTION);+ deliver([fetched(`/agent-sessions/${SESSION}/control`, 1101, 1300)]);+ trace.end('released');+ expect(telemetry.spans[0].attributes).toMatchObject({+ 'agent.prompt.created_response_end_at_ms': 49,+ 'agent.prompt.accepted_response_end_at_ms': 300,+ });+ });++ it('reads a response end still buffered when the span ends', () => {+ const buffered: PerformanceEntry[] = [];+ vi.stubGlobal(+ 'PerformanceObserver',+ class {+ observe() {}+ takeRecords() {+ return buffered.splice(0);+ }+ disconnect() {+ buffered.length = 0;+ }+ }+ );+ vi.spyOn(performance, 'now').mockReturnValue(1000);+ const trace = new PromptTrace(SESSION, { newSession: true });+ buffered.push({+ entryType: 'resource',+ initiatorType: 'fetch',+ name: 'https://harness.test/agent-sessions',+ startTime: 1005,+ responseEnd: 1049,+ } as PerformanceResourceTiming);+ trace.end('failed');+ expect(+ telemetry.spans[0].attributes['agent.prompt.created_response_end_at_ms']+ ).toBe(49);+ });+it.each([['/app/md/private-document-id', 'document'],['/app/drive/md/private-document-id', 'document'],diff --git a/apps/web/src/lib/core/agent-session/prompt-telemetry.ts b/apps/web/src/lib/core/agent-session/prompt-telemetry.tsindex a0a41541d4..d23c7036bb 100644--- a/apps/web/src/lib/core/agent-session/prompt-telemetry.ts+++ b/apps/web/src/lib/core/agent-session/prompt-telemetry.ts@@ -14,6 +14,7 @@import type { Span } from '@macro-inc/observability';import { Telemetry } from '@macro-inc/observability';+import type { WarmClaim } from '@queries/agent-session/warm';import type {FoldedMessage,FoldedStreamEvent,@@ -111,11 +112,37 @@ export type PromptSubmitSurface =| 'agent_session'| 'other';+/** How a new session was asked for, to tell its slower paths apart. */+export type PromptCreate = {+ warmClaim: WarmClaim;+ /** A model was named for the create. */+ modelOverride: boolean;+ /**+ * The model catalog was still loading, so the create silently ran on the+ * persona's default instead of the composer's choice.+ */+ modelFallback: boolean;+ /** The effort confirmed before the prompt; this path loads and configures serially. */+ effort: string | undefined;+};+type PromptTraceOptions = {newSession: boolean;submitSurface?: PromptSubmitSurface;+ create?: PromptCreate;+};++/** A request whose resource-timing `responseEnd` is recorded beside its stage. */+type ResponseWatch = {+ stage: 'created' | 'accepted';+ path: RegExp;+ /** Ignore requests that started before this `performance.now()`. */+ after: number;};+const CREATE_PATH = /\/agent-sessions$/;+const CONTROL_PATH = /\/agent-sessions\/[^/]+\/control$/;+export class PromptTrace {readonly #span: Span | undefined;readonly #sessionId: string;@@ -132,10 +159,14 @@ export class PromptTrace {#mounted = false;#rendered = false;readonly #renderers = new Set<() => void>();+ #responses: ResponseWatch[] = [];+ #resourceObserver: PerformanceObserver | undefined;constructor(sessionId: string, options: PromptTraceOptions) {this.#sessionId = sessionId;this.#span = start(sessionId, options);+ this.#recordInputDelay();+ if (options.newSession) this.#watchResponse('created', CREATE_PATH);this.#stallTimer = setTimeout(() => this.end('stalled'),STALL_THRESHOLD_MS@@ -227,6 +258,7 @@ export class PromptTrace {/** The control will be sent under `actionId`; its confirmation may name it. */expect(actionId: string): void {this.#actionIds.add(actionId);+ this.#watchResponse('accepted', CONTROL_PATH);}/** The control POST answered with the id the harness accepted it under. */@@ -266,6 +298,7 @@ export class PromptTrace {this.#set('agent.prompt.renderer_attached', this.#renderers.size > 0);for (const stop of this.#renderers) stop();this.#renderers.clear();+ this.#stopWatchingResponses();if (outcome === 'hidden') this.#set('agent.prompt.hidden', true);this.#set('agent.prompt.outcome', outcome);this.#set(@@ -337,6 +370,93 @@ export class PromptTrace {);}+ /**+ * Record when the triggering keyboard or pointer event happened. The span+ * itself starts here, in the handler; the gap is main-thread input delay.+ */+ #recordInputDelay(): void {+ try {+ const event = typeof window === 'undefined' ? undefined : window.event;+ const kind =+ event instanceof KeyboardEvent+ ? 'keyboard'+ : event instanceof MouseEvent+ ? 'pointer'+ : undefined;+ if (!event || !kind) return;+ this.#set('agent.prompt.input_kind', kind);+ this.#set(+ 'agent.prompt.input_delay_ms',+ Math.max(0, Math.round(this.#startedAt - event.timeStamp))+ );+ } catch {+ // See the module comment.+ }+ }++ /**+ * Record when the request's response finished arriving, from resource+ * timing. Against `<stage>_at_ms`, when its promise resolved, the gap is+ * main-thread stall. The first matching request wins.+ */+ #watchResponse(stage: ResponseWatch['stage'], path: RegExp): void {+ try {+ if (typeof PerformanceObserver === 'undefined') return;+ this.#responses.push({ stage, path, after: performance.now() });+ if (this.#resourceObserver) return;+ this.#resourceObserver = new PerformanceObserver((list) =>+ this.#observeResources(list.getEntries())+ );+ this.#resourceObserver.observe({ type: 'resource' });+ } catch {+ // See the module comment.+ }+ }++ /** Read entries already delivered but not yet called back, then stop. */+ #stopWatchingResponses(): void {+ const observer = this.#resourceObserver;+ if (!observer) return;+ this.#resourceObserver = undefined;+ try {+ this.#matchResponses(observer.takeRecords());+ observer.disconnect();+ } catch {+ // See the module comment.+ }+ }++ #observeResources(entries: PerformanceEntryList): void {+ try {+ this.#matchResponses(entries);+ } catch {+ // See the module comment.+ }+ }++ #matchResponses(entries: PerformanceEntryList): void {+ for (const entry of entries) {+ if (!isFetchTiming(entry) || entry.responseEnd === 0) continue;+ const pathname = new URL(entry.name, location.href).pathname;+ const watch = this.#responses.find(+ (candidate) =>+ entry.startTime >= candidate.after && candidate.path.test(pathname)+ );+ if (!watch) continue;+ this.#responses = this.#responses.filter(+ (candidate) => candidate !== watch+ );+ this.#set(+ `agent.prompt.${watch.stage}_response_end_at_ms`,+ Math.round(entry.responseEnd - this.#startedAt)+ );+ }+ if (this.#responses.length === 0) {+ this.#resourceObserver?.disconnect();+ this.#resourceObserver = undefined;+ }+ }+#set(name: string, value: string | number | boolean): void {if (this.#ended) return;try {@@ -347,6 +467,15 @@ export class PromptTrace {}}+function isFetchTiming(+ entry: PerformanceEntry+): entry is PerformanceResourceTiming {+ return (+ entry.entryType === 'resource' &&+ (entry as PerformanceResourceTiming).initiatorType === 'fetch'+ );+}+function documentHidden(): boolean {return (typeof document === 'undefined' ||@@ -375,6 +504,15 @@ function start('agent.prompt.submit_surface',options.submitSurface ?? submitSurface(options.newSession));+ if (options.create) {+ const { warmClaim, modelOverride, modelFallback, effort } =+ options.create;+ span.setAttr('agent.prompt.warm_claim', warmClaim);+ span.setAttr('agent.prompt.model_override_set', modelOverride);+ span.setAttr('agent.prompt.model_fallback', modelFallback);+ span.setAttr('agent.prompt.effort_override', effort !== undefined);+ if (effort !== undefined) span.setAttr('agent.prompt.effort', effort);+ }return span;} catch {return undefined;diff --git a/apps/web/src/lib/queries/agent-session/warm.test.tsx b/apps/web/src/lib/queries/agent-session/warm.test.tsxindex c9bd58b44b..a4a2b4adb7 100644--- a/apps/web/src/lib/queries/agent-session/warm.test.tsx+++ b/apps/web/src/lib/queries/agent-session/warm.test.tsx@@ -56,8 +56,10 @@ function prepared(id: string) {function setup() {const client = new QueryClient();clients.push(client);- const take = (options: Parameters<typeof takeWarmAgentSession>[0] = {}) =>+ const claim = (options: Parameters<typeof takeWarmAgentSession>[0] = {}) =>takeWarmAgentSession({ userId: OWNER, ...options }, client);+ const take = (options: Parameters<typeof takeWarmAgentSession>[0] = {}) =>+ claim(options).id;const replenish = () => replenishWarmAgentSession(OWNER, client);const settled = () => vi.waitFor(() => expect(client.isFetching()).toBe(0));const mount = () => {@@ -80,7 +82,7 @@ function setup() {disposers.add(unmount);return unmount;};- return { take, replenish, settled, mount };+ return { claim, take, replenish, settled, mount };}it('does not warm for an owner who has no existing warming query', () => {@@ -110,23 +112,30 @@ it('retries a failed initial preparation only after successful creation', asyncit('deduplicates page mounts and preserves a reservation for the matching owner and settings', async () => {warm.mockResolvedValueOnce(prepared('warm-id'));- const { mount, settled, take, replenish } = setup();+ const { mount, settled, claim, take, replenish } = setup();mount();mount();await settled();expect(warm).toHaveBeenCalledTimes(1);- expect(take({ userId: 'other' })).toBeUndefined();- expect(take({ userId: undefined })).toBeUndefined();- expect(take({ botId: 'another-bot' })).toBeUndefined();- expect(take({ repoUrl: 'https://github.com/example/repo' })).toBeUndefined();- expect(take({ modelOverride: 'different' })).toBeUndefined();- expect(take({ instructions: 'different' })).toBeUndefined();+ expect(claim({ userId: 'other' })).toEqual({ claim: 'miss_none_ready' });+ expect(claim({ userId: undefined })).toEqual({ claim: 'not_attempted' });+ expect(claim({ botId: 'another-bot' })).toEqual({ claim: 'not_attempted' });+ expect(claim({ repoUrl: 'https://github.com/example/repo' })).toEqual({+ claim: 'not_attempted',+ });+ expect(claim({ modelOverride: 'different' })).toEqual({+ claim: 'miss_model',+ });+ expect(claim({ instructions: 'different' })).toEqual({+ claim: 'miss_other',+ });replenish();await settled();expect(warm).toHaveBeenCalledTimes(1);- expect(take({ botId: MACRO_NEW_BOT_ID, modelOverride: 'model-a' })).toBe(- 'warm-id'- );+ expect(claim({ botId: MACRO_NEW_BOT_ID, modelOverride: 'model-a' })).toEqual({+ id: 'warm-id',+ claim: 'hit',+ });expect(take()).toBeUndefined();await settled();});diff --git a/apps/web/src/lib/queries/agent-session/warm.ts b/apps/web/src/lib/queries/agent-session/warm.tsindex 0a82a2f6ea..92287fb032 100644--- a/apps/web/src/lib/queries/agent-session/warm.ts+++ b/apps/web/src/lib/queries/agent-session/warm.ts@@ -47,6 +47,18 @@ export function useWarmAgentSessionQuery(userId: Accessor<string | undefined>) {});}+/** Why a new session did or did not start on a warm reservation. */+export type WarmClaim =+ | 'hit'+ /** A reservation was ready on another model. */+ | 'miss_model'+ /** No reservation, or only an expired one. */+ | 'miss_none_ready'+ /** A reservation was ready with other instructions. */+ | 'miss_other'+ /** The session is not the owner's default agent, so none could match. */+ | 'not_attempted';+/*** Consume once, only for the owner and exact default-agent configuration.* Keep the empty cache fresh until creation finishes and releases server capacity.@@ -60,27 +72,28 @@ export function takeWarmAgentSession(repoUrl?: string;},client: QueryClient = queryClient-): string | undefined {+): { id?: string; claim: WarmClaim } {if (!options.userId ||(options.botId && options.botId !== MACRO_NEW_BOT_ID) ||options.repoUrl)- return;+ return { claim: 'not_attempted' };const key = agentSessionWarmKeys.owner(options.userId).queryKey;const entry = client.getQueryData<WarmSession | null>(key);- if (!entry) return;+ if (!entry) return { claim: 'miss_none_ready' };if (entry.expires <= Date.now()) {client.setQueryData(key, null);- return;+ return { claim: 'miss_none_ready' };}- if (options.modelOverride && options.modelOverride !== entry.model) return;+ if (options.modelOverride && options.modelOverride !== entry.model)+ return { claim: 'miss_model' };if ((options.instructions?.trim() || '') !== (entry.instructions?.trim() || ''))- return;+ return { claim: 'miss_other' };client.setQueryData(key, null);- return entry.id;+ return { id: entry.id, claim: 'hit' };}/** Prepare one replacement after creation succeeds, joining any existing request. */