Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(observability): measure agent session switches until the transcript paints (#7747)
Wolf Mermelstein5e2c992870ad
diff --git a/apps/web/src/features/block-agent/context/create-agent-session.test.tsx b/apps/web/src/features/block-agent/context/create-agent-session.test.tsxindex 3a49392b23..fdd9059d0f 100644--- a/apps/web/src/features/block-agent/context/create-agent-session.test.tsx+++ b/apps/web/src/features/block-agent/context/create-agent-session.test.tsx@@ -35,6 +35,7 @@ vi.mock('@queries/agent-session/log', () => ({AgentSessionLogUnavailable: class extends Error {},watchAgentSessionLog: () => ({cached: logSource.cached(),+ cacheMiss: () => undefined,fetched: logSource.fetched(),stop: () => {},}),diff --git a/apps/web/src/features/block-agent/context/create-agent-session.ts b/apps/web/src/features/block-agent/context/create-agent-session.tsindex 0b3d94d751..ca00d025dc 100644--- a/apps/web/src/features/block-agent/context/create-agent-session.ts+++ b/apps/web/src/features/block-agent/context/create-agent-session.ts@@ -241,7 +241,7 @@ export function createAgentSession(setMetadata(snapshot.metadata);replace(snapshot.messages);});- session.rendered(kind);+ session.rendered(kind, snapshot.messages);};/** The fetched session, shown, with a rename that landed meanwhile kept. */diff --git a/apps/web/src/features/block-agent/context/pending-session.test.tsx b/apps/web/src/features/block-agent/context/pending-session.test.tsxindex 8c810790e6..1778bdd75c 100644--- a/apps/web/src/features/block-agent/context/pending-session.test.tsx+++ b/apps/web/src/features/block-agent/context/pending-session.test.tsx@@ -35,6 +35,7 @@ vi.mock('@queries/agent-session/log', () => ({AgentSessionLogUnavailable: class extends Error {},watchAgentSessionLog: () => ({cached: Promise.resolve(undefined),+ cacheMiss: () => undefined,fetched: Promise.resolve({ bot: { id: 'bot' }, rows: [] }),stop: () => {},}),diff --git a/apps/web/src/lib/core/agent-session/AgentSession.test.ts b/apps/web/src/lib/core/agent-session/AgentSession.test.tsindex 6eca1f18d2..3eb99b8f76 100644--- a/apps/web/src/lib/core/agent-session/AgentSession.test.ts+++ b/apps/web/src/lib/core/agent-session/AgentSession.test.ts@@ -36,6 +36,7 @@ const logSource = vi.hoisted(() => {const source = {Unavailable,cached: vi.fn(),+ cacheMiss: vi.fn(),fetched: vi.fn(),forget: vi.fn(),watch: vi.fn(),@@ -50,6 +51,7 @@ const logSource = vi.hoisted(() => {source.follow = follow;return {cached: source.cached(),+ cacheMiss: source.cacheMiss,fetched: source.fetched(),stop: source.stop,};@@ -113,6 +115,10 @@ vi.mock('@macro-inc/observability', () => ({}));const promptSpans = () =>telemetry.spans.filter((span) => span.name === 'agent.prompt');+const loadSpans = () =>+ telemetry.spans.filter((span) => span.name === 'agent.session.load');+const renderSpans = () =>+ telemetry.spans.filter((span) => span.name === 'agent.session.render');const updates = vi.hoisted(() => ({listeners: new Set<(event: { agentSessionId: string }) => void>(),@@ -994,6 +1000,168 @@ describe('AgentSession', () => {live.release();});+ describe('load telemetry', () => {+ const nextPaint = () =>+ new Promise((resolve) =>+ requestAnimationFrame(() => requestAnimationFrame(resolve))+ );++ it('says why the cache missed', async () => {+ logSource.cacheMiss.mockReturnValue('network_first');+ const live = AgentSession.acquire(SESSION);+ await live.load();++ expect(loadSpans()[0].attributes).toMatchObject({+ 'agent.session.load.cache_hit': false,+ 'agent.session.load.cache_miss': 'network_first',+ });+ live.release();+ });++ it('says why a load failed', async () => {+ harness.get.mockResolvedValueOnce(err([{ code: 'HTTP_ERROR' }]));+ const live = AgentSession.acquire(SESSION);+ await expect(live.load()).rejects.toThrow();++ logSource.fetched.mockRejectedValueOnce(+ new logSource.Unavailable(SESSION, 'failed')+ );+ await expect(live.load()).rejects.toThrow();++ harness.get.mockResolvedValueOnce(err([{ code: 'UNAUTHORIZED' }]));+ await expect(live.load()).rejects.toThrow();++ fold.readSession.mockRejectedValueOnce(new Error('worker died'));+ await expect(live.load()).rejects.toThrow('worker died');++ expect(+ loadSpans().map((span) => [+ span.attributes['agent.session.load.outcome'],+ span.attributes['agent.session.load.failure'],+ ])+ ).toEqual([+ ['failed', 'session_fetch'],+ ['failed', 'log_fetch'],+ ['failed', 'access_denied'],+ ['failed', 'fold'],+ ]);+ live.release();+ });++ it('holds the render span open until the transcript has painted', async () => {+ const live = AgentSession.acquire(SESSION);+ await live.load();++ live.rendered('fetched', [+ {+ agentSessionId: SESSION,+ turn: 0,+ author: { kind: 'user', userId: null },+ requestId: null,+ parts: [{ kind: 'text', text: 'Hi' }],+ stop: null,+ pending: false,+ },+ {+ agentSessionId: SESSION,+ turn: 0,+ author: { kind: 'agent' },+ requestId: null,+ parts: [+ { kind: 'thought', text: 'hmm' },+ { kind: 'text', text: 'Hello' },+ ],+ stop: null,+ pending: false,+ },+ ]);+ const [span] = renderSpans();+ expect(span.ends).toBe(0);+ expect(span.attributes).toMatchObject({+ 'agent.session.render.kind': 'fetched',+ 'agent.session.load.transcript_messages': 2,+ 'agent.session.load.transcript_turns': 1,+ 'agent.session.load.transcript_chars': 10,+ });++ await nextPaint();++ expect(span.ends).toBe(1);+ expect(+ span.attributes['agent.session.load.painted_at_ms']+ ).toBeGreaterThanOrEqual(+ span.attributes['agent.session.render.since_load_ms'] as number+ );+ expect(span.attributes['agent.session.load.paint_ms']).toEqual(+ expect.any(Number)+ );+ live.release();+ });++ it('attributes the long animation frames before the paint', async () => {+ const observed: { deliver?: (entries: unknown[]) => void } = {};+ class FakeObserver {+ static supportedEntryTypes = ['long-animation-frame'];+ constructor(callback: (list: { getEntries: () => unknown[] }) => void) {+ observed.deliver = (entries) =>+ callback({ getEntries: () => entries });+ }+ observe() {}+ takeRecords() {+ return [];+ }+ disconnect() {}+ }+ vi.stubGlobal('PerformanceObserver', FakeObserver);+ onTestFinished(() => {+ vi.unstubAllGlobals();+ });+ const live = AgentSession.acquire(SESSION);+ await live.load();+ const now = performance.now();+ const frame = (duration: number, scripts: unknown[]) => ({+ entryType: 'long-animation-frame',+ startTime: now - duration,+ duration,+ blockingDuration: duration - 50,+ scripts,+ });+ observed.deliver?.([+ frame(80, []),+ frame(500, [+ { duration: 100, invoker: 'small', sourceURL: 'a.js' },+ {+ duration: 300,+ invoker: 'ThreadList.positionInitial',+ sourceURL: `https://app/${'x'.repeat(300)}.js`,+ },+ ]),+ ]);++ live.rendered('fetched', []);+ await nextPaint();+ const [span] = renderSpans();+ // Ended once the frames before the paint have been reported.+ expect(span.ends).toBe(0);+ await new Promise((resolve) => setTimeout(resolve, 150));++ expect(span.ends).toBe(1);+ expect(span.attributes).toMatchObject({+ 'agent.session.load.loaf_count': 2,+ 'agent.session.load.loaf_total_ms': 580,+ 'agent.session.load.loaf_max_ms': 500,+ 'agent.session.load.loaf_blocking_ms': 480,+ 'agent.session.load.loaf_top_script_invoker':+ 'ThreadList.positionInitial',+ 'agent.session.load.loaf_top_script_ms': 300,+ });+ expect(+ span.attributes['agent.session.load.loaf_top_script_source']+ ).toHaveLength(200);+ live.release();+ });+ });+describe('prompt telemetry', () => {it('posts a prompt inside its span and ends it at the first painted agent text', async () => {let postedUnder: RecordedSpan | undefined;diff --git a/apps/web/src/lib/core/agent-session/AgentSession.ts b/apps/web/src/lib/core/agent-session/AgentSession.tsindex 0b33e661ca..66dc4328bf 100644--- a/apps/web/src/lib/core/agent-session/AgentSession.ts+++ b/apps/web/src/lib/core/agent-session/AgentSession.ts@@ -48,8 +48,14 @@ import type {SessionBot,} from '@service-agent-harness/generated/schemas';import { subscribeGraphqlSoupReconnected } from '@service-storage/graphql-soup';+import { CombinedError } from '@urql/core';import { v7 as uuidv7 } from 'uuid';-import { SessionLoadTrace, traceAcquire } from './load-telemetry';+import {+ type LoadFailure,+ SessionLoadTrace,+ traceAcquire,+ transcriptSize,+} from './load-telemetry';import { frameDelivery, PromptTrace } from './prompt-telemetry';import { publishSessionTurn } from './session-turn';@@ -97,6 +103,15 @@ const inaccessible = (error: unknown) =>error instanceof AgentSessionLogUnavailable &&error.reason === 'inaccessible';+/** Why the log query failed, for the load span. */+function logFailure(error: unknown): LoadFailure {+ const cause = error instanceof Error ? error.cause : undefined;+ if (cause instanceof CombinedError) {+ return cause.networkError ? 'log_network' : 'log_graphql';+ }+ return cause === undefined ? 'log_fetch' : 'log_unreadable';+}+const accessDenied = (errors: { code: string }[]) =>errors.some((error) => error.code === 'UNAUTHORIZED' || error.code === 'FORBIDDEN'@@ -524,7 +539,8 @@ export class AgentSession {this.watch.cached.then((cached) => {this.trace.cacheRead(readStartedAt,- cached && { rows: cached.rows.length }+ cached && { rows: cached.rows.length },+ this.watch.cacheMiss());return cached;}),@@ -546,8 +562,11 @@ export class AgentSession {}/** A surface put this session's transcript on screen. */- rendered(kind: 'cached' | 'fetched'): void {- this.trace.rendered(kind);+ rendered(+ kind: 'cached' | 'fetched',+ messages: SessionFoldSnapshot['messages']+ ): void {+ this.trace.rendered(kind, transcriptSize(messages));}private startLoad(): Promise<AgentSessionRecord> {@@ -585,15 +604,19 @@ export class AgentSession {if (session.isErr()) {if (accessDenied(session.error)) {this.forgetCached();+ this.trace.failing('access_denied');throw new AgentSessionAccessDenied(this.id);}+ this.trace.failing('session_fetch');throw new Error(`agent session could not be fetched: ${this.id}`);}if (!log.ok) {if (inaccessible(log.error)) {this.forgetCached();+ this.trace.failing('access_denied');throw new AgentSessionAccessDenied(this.id);}+ this.trace.failing(logFailure(log.error));throw new Error(`agent session log could not be fetched: ${this.id}`, {cause: log.error,});@@ -602,12 +625,17 @@ export class AgentSession {const foldStartedAt = performance.now();this.settled = true;- await this.apply([{ kind: 'snapshot', rows: log.log.rows }]);- // Inputs can keep arriving while each push is in flight; drain until a- // check finds nothing, then flip ready so the next one goes straight in.- await this.drainBuffered();- this.trace.folded(foldStartedAt);- this.setTurn((await readSession(this.id)).metadata.turn);+ try {+ await this.apply([{ kind: 'snapshot', rows: log.log.rows }]);+ // Inputs can keep arriving while each push is in flight; drain until a+ // check finds nothing, then flip ready so the next one goes straight in.+ await this.drainBuffered();+ this.trace.folded(foldStartedAt);+ this.setTurn((await readSession(this.id)).metadata.turn);+ } catch (error) {+ this.trace.failing('fold');+ throw error;+ }if (this.resyncRequested) void this.resync();return { session: session.value, bot: log.log.bot };}diff --git a/apps/web/src/lib/core/agent-session/load-telemetry.ts b/apps/web/src/lib/core/agent-session/load-telemetry.tsindex 4b718797db..300db8666b 100644--- a/apps/web/src/lib/core/agent-session/load-telemetry.ts+++ b/apps/web/src/lib/core/agent-session/load-telemetry.ts@@ -15,6 +15,13 @@import type { Span } from '@macro-inc/observability';import { Telemetry } from '@macro-inc/observability';+import type { CacheMiss } from '@queries/agent-session/log';+import type { FoldedMessage } from '@service-agent-fold/generated/types';+import {+ afterLongFrames,+ observeLongFrames,+ summarizeFrames,+} from './long-frames';/** How an attempt to open a session ended. */export type LoadOutcome =@@ -27,6 +34,25 @@ export type LoadOutcome =/** Still unsettled at {@link STALL_THRESHOLD_MS} — reported, not concluded. */| 'stalled';+/** Why a load ended `failed`, bounded so failures group. */+export type LoadFailure =+ /** The viewer cannot see the session. */+ | 'access_denied'+ /** The session row could not be fetched. */+ | 'session_fetch'+ /** The log query failed in transport. */+ | 'log_network'+ /** The log query answered with GraphQL errors. */+ | 'log_graphql'+ /** The log arrived but its rows could not be read. */+ | 'log_unreadable'+ /** The log query failed for no reason it gave. */+ | 'log_fetch'+ /** The fold worker failed on the snapshot. */+ | 'fold'+ /** Nothing above; a bug in the classification. */+ | 'unknown';+/*** How long a load may go unsettled before it is reported as stalled.*@@ -41,13 +67,6 @@ export type LoadOutcome =*/const STALL_THRESHOLD_MS = 15_000;-/**- * One attempt to open a session, from the first acquisition to whatever- * ended it.- *- * Held rather than run around an operation because the operation it measures- * can be abandoned: `release` ends the span from outside the load.- *//** A stage of the load, named as its measure and attribute suffix. */export type LoadStage =/** The cached log was looked up. */@@ -63,7 +82,41 @@ export type LoadStage =/** A surface painted the cached transcript. */| 'warm_render'/** A surface painted the fetched transcript. */- | 'render';+ | 'render'+ /** The cached transcript laid out and a frame painted it. */+ | 'warm_painted'+ /** The fetched transcript laid out and a frame painted it. */+ | 'painted';++/** What a surface rendered, so slow paints can be grouped by size. */+export type TranscriptSize = {+ messages: number;+ turns: number;+ /** Text and thought characters: roughly the markdown laid out. */+ chars: number;+};++export function transcriptSize(+ messages: readonly FoldedMessage[]+): TranscriptSize {+ const turns = new Set<number>();+ let chars = 0;+ for (const message of messages) {+ turns.add(message.turn);+ for (const part of message.parts) {+ if (part.kind === 'text' || part.kind === 'thought') {+ chars += part.text.length;+ }+ }+ }+ return { messages: messages.length, turns: turns.size, chars };+}++/**+ * How long a render waits for its paint before the span is ended without+ * one: a hidden tab runs no animation frames.+ */+const PAINT_TIMEOUT_MS = 5_000;/*** One attempt to open a session, from the first acquisition to whatever@@ -85,7 +138,9 @@ export class SessionLoadTrace {readonly #sessionId: string;readonly #startedAt = performance.now();readonly #recorded: Record<string, string | number | boolean> = {};+ readonly #frames = observeLongFrames();#ended = false;+ #failure: LoadFailure | undefined;#stallTimer: ReturnType<typeof setTimeout> | undefined;constructor(sessionId: string) {@@ -95,6 +150,8 @@ export class SessionLoadTrace {() => this.end('stalled'),STALL_THRESHOLD_MS);+ // A paint later than this is not a load any more; stop watching frames.+ setTimeout(() => this.#frames?.stop(), STALL_THRESHOLD_MS);}/**@@ -131,13 +188,23 @@ export class SessionLoadTrace {* The cached log was looked up: `hit` is what it held, or nothing. A miss* is a stage too; its duration is what a cold open paid for looking.*/- cacheRead(startedAt: number, hit: { rows: number } | undefined): void {+ cacheRead(+ startedAt: number,+ hit: { rows: number } | undefined,+ miss: CacheMiss | undefined+ ): void {this.stage('cache_read', startedAt, {cache_hit: hit !== undefined,...(hit && { cache_rows: hit.rows }),+ ...(!hit && miss && { cache_miss: miss }),});}+ /** Why the load is about to fail; read by `end('failed')`. */+ failing(failure: LoadFailure): void {+ this.#failure ??= failure;+ }+/** The log arrived; `rows` is what the fold is about to be given. */fetched(startedAt: number, rows: number): void {this.stage('log_fetch', startedAt, { log_rows: rows });@@ -152,36 +219,87 @@ export class SessionLoadTrace {}/**- * A surface put a transcript on screen. `cached` is the warm paint from- * the cached log; the other is the fetched one. Measured from the start- * of the load, which is what a person waited. Its own span rather than a- * stage: the load span has usually ended by the time a surface paints,- * and an ended span drops what is set on it.+ * A surface handed a transcript to the DOM. `cached` is the warm paint+ * from the cached log; the other is the fetched one. Measured from the+ * start of the load, which is what a person waited. Its own span rather+ * than a stage: the load span has usually ended by the time a surface+ * paints, and an ended span drops what is set on it.+ *+ * The DOM is not the screen: a long transcript lays out and paints in one+ * long frame after this. The span stays open until a frame has painted,+ * and records that as `painted_at_ms`, with the long animation frames+ * since the load started. It ends a little after the paint, once those+ * frames are reported, so `paint_ms` is the paint, not the span length.*/- rendered(kind: 'cached' | 'fetched'): void {+ rendered(kind: 'cached' | 'fetched', size: TranscriptSize): void {const stage: LoadStage = kind === 'cached' ? 'warm_render' : 'render';const now = performance.now();const ms = now - this.#startedAt;this.#recorded[`agent.session.load.${stage}_ms`] = Math.round(ms);const span = start('agent.session.render', this.#sessionId);+ const record: Record<string, string | number | boolean> = {+ 'agent.session.render.kind': kind,+ 'agent.session.render.since_load_ms': Math.round(ms),+ 'agent.session.render.hidden':+ typeof document !== 'undefined' && document.hidden,+ 'agent.session.load.transcript_messages': size.messages,+ 'agent.session.load.transcript_turns': size.turns,+ 'agent.session.load.transcript_chars': size.chars,+ };try {- span?.setAttr('agent.session.render.kind', kind);- span?.setAttr('agent.session.render.since_load_ms', ms);- span?.end();+ for (const [name, value] of Object.entries(record)) {+ span?.setAttr(name, value);+ }performance.measure(`agent.session.load.${stage}`, {start: this.#startedAt,end: now,detail: { sessionId: this.#sessionId, kind },});- if (loadDebugEnabled()) {- console.debug('[agent-session] render', this.#sessionId, {- kind,- since_load_ms: Math.round(ms),- });- }} catch {// See the module comment.}+ afterPaint(PAINT_TIMEOUT_MS, (paintedAt) => {+ afterLongFrames(this.#frames, () => {+ try {+ if (paintedAt === undefined) {+ record['agent.session.load.paint_timed_out'] = true;+ } else {+ const painted: LoadStage =+ kind === 'cached' ? 'warm_painted' : 'painted';+ record['agent.session.load.painted_at_ms'] = Math.round(+ paintedAt - this.#startedAt+ );+ record['agent.session.load.paint_ms'] = Math.round(paintedAt - now);+ Object.assign(+ record,+ summarizeFrames(+ this.#frames,+ 'agent.session.load',+ this.#startedAt,+ paintedAt+ )+ );+ this.#recorded[`agent.session.load.${painted}_ms`] = Math.round(+ paintedAt - this.#startedAt+ );+ performance.measure(`agent.session.load.${painted}`, {+ start: this.#startedAt,+ end: paintedAt,+ detail: { sessionId: this.#sessionId, kind },+ });+ }+ for (const [name, value] of Object.entries(record)) {+ span?.setAttr(name, value);+ }+ span?.end();+ if (loadDebugEnabled()) {+ console.debug('[agent-session] render', this.#sessionId, record);+ }+ } catch {+ // See the module comment.+ }+ });+ });}/**@@ -196,6 +314,11 @@ export class SessionLoadTrace {this.#stallTimer = undefined;this.#set('agent.session.load.outcome', outcome);this.#set('agent.session.load.total_ms', this.#since());+ if (outcome === 'failed') {+ this.#set('agent.session.load.failure', this.#failure ?? 'unknown');+ }+ // Nothing will paint for a load that did not land.+ if (outcome !== 'loaded') this.#frames?.stop();try {performance.measure('agent.session.load', {start: this.#startedAt,@@ -260,6 +383,32 @@ export function traceCacheWrite(}}+/**+ * Calls `then` with the time of the frame after the next one: by then the+ * DOM written before this call has laid out and painted. With `undefined`+ * after `timeoutMs` when no frame runs, as in a hidden tab.+ */+function afterPaint(+ timeoutMs: number,+ then: (paintedAt: number | undefined) => void+): void {+ if (typeof requestAnimationFrame === 'undefined') {+ then(undefined);+ return;+ }+ let done = false;+ const finish = (paintedAt: number | undefined) => {+ if (done) return;+ done = true;+ clearTimeout(timer);+ then(paintedAt);+ };+ const timer = setTimeout(() => finish(undefined), timeoutMs);+ requestAnimationFrame(() => {+ requestAnimationFrame(() => finish(performance.now()));+ });+}+function loadDebugEnabled(): boolean {return (typeof window !== 'undefined' &&diff --git a/apps/web/src/lib/core/agent-session/long-frames.ts b/apps/web/src/lib/core/agent-session/long-frames.tsnew file mode 100644index 0000000000..ea1d4ca6f1--- /dev/null+++ b/apps/web/src/lib/core/agent-session/long-frames.ts@@ -0,0 +1,139 @@+/**+ * Long animation frames (the Long Animation Frames API) seen while a span+ * waited for something to paint: the main-thread work a person sat through+ * that no request timing shows. Feature-detected; absent where unsupported.+ */++/**+ * The fields of a `long-animation-frame` entry read here. Typed locally:+ * the DOM lib does not carry them yet.+ */+type LongFrame = PerformanceEntry & {+ blockingDuration: number;+ scripts: readonly {+ duration: number;+ invoker: string;+ sourceURL: string;+ }[];+};++export type LongFrameWatch = {+ /** Every frame seen so far, including those not yet delivered. */+ take: () => readonly LongFrame[];+ stop: () => void;+};++/**+ * Collect long animation frames (50ms+) from now until `stop`. Undefined+ * where the browser has no Long Animation Frames API.+ */+export function observeLongFrames(): LongFrameWatch | undefined {+ try {+ if (+ typeof PerformanceObserver === 'undefined' ||+ !PerformanceObserver.supportedEntryTypes?.includes('long-animation-frame')+ ) {+ return undefined;+ }+ const frames: LongFrame[] = [];+ const collect = (entries: PerformanceEntryList) => {+ // Entries of the observed type carry the fields `LongFrame` reads.+ for (const entry of entries) frames.push(entry as LongFrame);+ };+ const observer = new PerformanceObserver((list) =>+ collect(list.getEntries())+ );+ observer.observe({ type: 'long-animation-frame' });+ let stopped = false;+ return {+ take: () => {+ if (!stopped) collect(observer.takeRecords());+ return frames;+ },+ stop: () => {+ if (stopped) return;+ collect(observer.takeRecords());+ stopped = true;+ observer.disconnect();+ },+ };+ } catch {+ return undefined;+ }+}++/**+ * How long a frame's entry may take to reach the observer after the frame:+ * the browser queues it once the frame has painted, so a paint observed+ * by animation frame comes before the entry describing it.+ */+const ENTRY_DELAY_MS = 100;++/**+ * Calls `then` once the frames up to now have reached `watch`: at once+ * without one, otherwise after {@link ENTRY_DELAY_MS}.+ */+export function afterLongFrames(+ watch: LongFrameWatch | undefined,+ then: () => void+): void {+ if (!watch) {+ then();+ return;+ }+ setTimeout(then, ENTRY_DELAY_MS);+}++/** Longest a script attribution may be on a span. */+const SCRIPT_ATTRIBUTION_MAX = 200;++/**+ * The long frames that overlap `[from, to]`, as `<prefix>.loaf_*`+ * attributes: how many, how long, how much of that blocked input, and the+ * script that held the longest one.+ */+export function summarizeFrames(+ watch: LongFrameWatch | undefined,+ prefix: string,+ from: number,+ to: number+): Record<string, number | string> {+ if (!watch) return {};+ const frames = watch+ .take()+ .filter(+ (frame) => frame.startTime < to && frame.startTime + frame.duration > from+ );+ let total = 0;+ let blocking = 0;+ let longest: LongFrame | undefined;+ for (const frame of frames) {+ total += frame.duration;+ blocking += frame.blockingDuration;+ if (!longest || frame.duration > longest.duration) longest = frame;+ }+ const summary: Record<string, number | string> = {+ [`${prefix}.loaf_count`]: frames.length,+ [`${prefix}.loaf_total_ms`]: Math.round(total),+ [`${prefix}.loaf_max_ms`]: Math.round(longest?.duration ?? 0),+ [`${prefix}.loaf_blocking_ms`]: Math.round(blocking),+ };+ const script = longest?.scripts.reduce<+ LongFrame['scripts'][number] | undefined+ >(+ (top, script) => (!top || script.duration > top.duration ? script : top),+ undefined+ );+ if (script) {+ summary[`${prefix}.loaf_top_script_source`] = script.sourceURL.slice(+ 0,+ SCRIPT_ATTRIBUTION_MAX+ );+ summary[`${prefix}.loaf_top_script_invoker`] = script.invoker.slice(+ 0,+ SCRIPT_ATTRIBUTION_MAX+ );+ summary[`${prefix}.loaf_top_script_ms`] = Math.round(script.duration);+ }+ return summary;+}diff --git a/apps/web/src/lib/core/agent-session/prompt-telemetry.ts b/apps/web/src/lib/core/agent-session/prompt-telemetry.tsindex d23c7036bb..426f782d66 100644--- a/apps/web/src/lib/core/agent-session/prompt-telemetry.ts+++ b/apps/web/src/lib/core/agent-session/prompt-telemetry.ts@@ -20,6 +20,11 @@ import type {FoldedStreamEvent,} from '@service-agent-fold/generated/types';import { match } from 'ts-pattern';+import {+ afterLongFrames,+ observeLongFrames,+ summarizeFrames,+} from './long-frames';import { observeRenderedAnswer } from './render-telemetry';/** How a sent prompt ended, as far as this span is concerned. */@@ -161,6 +166,7 @@ export class PromptTrace {readonly #renderers = new Set<() => void>();#responses: ResponseWatch[] = [];#resourceObserver: PerformanceObserver | undefined;+ readonly #frames = observeLongFrames();constructor(sessionId: string, options: PromptTraceOptions) {this.#sessionId = sessionId;@@ -301,22 +307,33 @@ export class PromptTrace {this.#stopWatchingResponses();if (outcome === 'hidden') this.#set('agent.prompt.hidden', true);this.#set('agent.prompt.outcome', outcome);- this.#set(- 'agent.prompt.total_ms',- Math.round(performance.now() - this.#startedAt)- );+ const now = performance.now();+ this.#set('agent.prompt.total_ms', Math.round(now - this.#startedAt));this.#ended = true;try {if (outcome === 'failed' && error !== undefined) this.#span?.error(error);- this.#span?.end();performance.measure('agent.prompt', {start: this.#startedAt,- end: performance.now(),+ end: now,detail: { sessionId: this.#sessionId, outcome },});} catch {// See the module comment.}+ // The long frames up to the end, which includes the frame that painted.+ afterLongFrames(this.#frames, () => {+ try {+ for (const [name, value] of Object.entries(+ summarizeFrames(this.#frames, 'agent.prompt', this.#startedAt, now)+ )) {+ this.#span?.setAttr(name, value);+ }+ this.#frames?.stop();+ this.#span?.end();+ } catch {+ // See the module comment.+ }+ });}#observeMessage(message: FoldedMessage, delivery?: FrameDelivery): void {diff --git a/apps/web/src/lib/queries/agent-session/log.test.ts b/apps/web/src/lib/queries/agent-session/log.test.tsindex 2e517ce12a..f12aa93431 100644--- a/apps/web/src/lib/queries/agent-session/log.test.ts+++ b/apps/web/src/lib/queries/agent-session/log.test.ts@@ -121,6 +121,35 @@ afterEach(() => {const settle = () => vi.advanceTimersByTimeAsync(0);describe('watchAgentSessionLog', () => {+ it('says why the cache missed', async () => {+ const hit = watchAgentSessionLog(SESSION);+ query.next(fromCache(logData([1])));+ query.next(fromNetwork(logData([1])));+ await hit.cached;+ expect(hit.cacheMiss()).toBeUndefined();++ const networkFirst = watchAgentSessionLog(SESSION);+ query.next(fromNetwork(logData([1])));+ expect(await networkFirst.cached).toBeUndefined();+ expect(networkFirst.cacheMiss()).toBe('network_first');++ const empty = watchAgentSessionLog(SESSION);+ query.next(fromCache({ user: { id: 'user-1', agentSession: null } }));+ query.next(fromNetwork(logData([1])));+ await empty.cached;+ expect(empty.cacheMiss()).toBe('empty');++ const networkOnly = watchAgentSessionLog(SESSION, 'network-only');+ query.next(fromNetwork(logData([1])));+ await networkOnly.cached;+ expect(networkOnly.cacheMiss()).toBe('network_only');++ const stopped = watchAgentSessionLog(SESSION);+ stopped.stop();+ await stopped.cached;+ expect(stopped.cacheMiss()).toBe('stopped');+ });+it('answers from the cache, then the network', async () => {const watch = watchAgentSessionLog(SESSION);query.next(fromCache(logData([1])));diff --git a/apps/web/src/lib/queries/agent-session/log.ts b/apps/web/src/lib/queries/agent-session/log.tsindex ddcf1212b5..7872d60bcb 100644--- a/apps/web/src/lib/queries/agent-session/log.ts+++ b/apps/web/src/lib/queries/agent-session/log.ts@@ -57,12 +57,29 @@ export class AgentSessionLogUnavailable extends Error {}}+/** Why {@link SessionLogWatch.cached} came back empty. */+export type CacheMiss =+ /** No persistent cache in this client, or it was disabled. */+ | 'no_cache'+ /** The watch asked for the network only. */+ | 'network_only'+ /** The network answered before the cache read hit, or the read missed. */+ | 'network_first'+ /** The cache answered without this session's log, or with an error. */+ | 'empty'+ /** The cached log could not be read into rows. */+ | 'unreadable'+ /** The watch stopped first. */+ | 'stopped';+export type SessionLogWatch = {/*** The log as the cache last held it, or nothing when the cache has no* copy or the network answered first. Never rejects.*/cached: Promise<SessionLogSnapshot | undefined>;+ /** Why `cached` resolved to nothing, once it has; for telemetry. */+ cacheMiss: () => CacheMiss | undefined;/** The log as the server holds it now. Rejects with {@link AgentSessionLogUnavailable}. */fetched: Promise<SessionLogSnapshot>;/**@@ -149,6 +166,20 @@ export function watchAgentSessionLog(follow?: SessionLogFollow): SessionLogWatch {let resolveCached!: (snapshot: SessionLogSnapshot | undefined) => void;+ let cacheSettled = false;+ let cacheMiss: CacheMiss | undefined;+ let cacheAnsweredEmpty = false;+ const hitCache = (snapshot: SessionLogSnapshot) => {+ if (cacheSettled) return;+ cacheSettled = true;+ resolveCached(snapshot);+ };+ const missCache = (reason: CacheMiss) => {+ if (cacheSettled) return;+ cacheSettled = true;+ cacheMiss = reason;+ resolveCached(undefined);+ };let resolveFetched!: (snapshot: SessionLogSnapshot) => void;let rejectFetched!: (error: AgentSessionLogUnavailable) => void;const cached = new Promise<SessionLogSnapshot | undefined>((resolve) => {@@ -167,7 +198,15 @@ export function watchAgentSessionLog(: () => undefined;const settleFetched = (result: OperationResult<AgentSessionLogQuery>) => {- resolveCached(undefined);+ missCache(+ policy === 'network-only'+ ? 'network_only'+ : !getGraphqlCacheHost()+ ? 'no_cache'+ : cacheAnsweredEmpty+ ? 'empty'+ : 'network_first'+ );if (result.error) {rejectFetched(new AgentSessionLogUnavailable(sessionId, 'failed', {@@ -213,10 +252,12 @@ export function watchAgentSessionLog(// network answer settles both.if (log && !result.error) {try {- resolveCached(toSnapshot(sessionId, log));+ hitCache(toSnapshot(sessionId, log));} catch {- resolveCached(undefined);+ missCache('unreadable');}+ } else {+ cacheAnsweredEmpty = true;}return;}@@ -227,12 +268,13 @@ export function watchAgentSessionLog(return {cached,+ cacheMiss: () => cacheMiss,fetched,stop: () => {unsubscribe();unsubscribeFollow();appender?.flush();- resolveCached(undefined);+ missCache('stopped');rejectFetched(new AgentSessionLogUnavailable(sessionId, 'failed'));},};