Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(observability): measure agent session switches until the transcript paints (#7747)

Wolf Mermelstein5e2c992870ad
diff --git a/apps/web/src/features/block-agent/context/create-agent-session.test.tsx b/apps/web/src/features/block-agent/context/create-agent-session.test.tsx
index 3a49392b23..fdd9059d0f 100644
--- a/apps/web/src/features/block-agent/context/create-agent-session.test.tsx
+++ b/apps/web/src/features/block-agent/context/create-agent-session.test.tsx
@@ -35,6 +35,7 @@ vi.mock('@queries/agent-session/log', () => ({
AgentSessionLogUnavailable: class extends Error {},
watchAgentSessionLog: () => ({
cached: logSource.cached(),
+ cacheMiss: () => undefined,
fetched: logSource.fetched(),
stop: () => {},
}),
diff --git a/apps/web/src/features/block-agent/context/create-agent-session.ts b/apps/web/src/features/block-agent/context/create-agent-session.ts
index 0b3d94d751..ca00d025dc 100644
--- a/apps/web/src/features/block-agent/context/create-agent-session.ts
+++ b/apps/web/src/features/block-agent/context/create-agent-session.ts
@@ -241,7 +241,7 @@ export function createAgentSession(
setMetadata(snapshot.metadata);
replace(snapshot.messages);
});
- session.rendered(kind);
+ session.rendered(kind, snapshot.messages);
};
/** The fetched session, shown, with a rename that landed meanwhile kept. */
diff --git a/apps/web/src/features/block-agent/context/pending-session.test.tsx b/apps/web/src/features/block-agent/context/pending-session.test.tsx
index 8c810790e6..1778bdd75c 100644
--- a/apps/web/src/features/block-agent/context/pending-session.test.tsx
+++ b/apps/web/src/features/block-agent/context/pending-session.test.tsx
@@ -35,6 +35,7 @@ vi.mock('@queries/agent-session/log', () => ({
AgentSessionLogUnavailable: class extends Error {},
watchAgentSessionLog: () => ({
cached: Promise.resolve(undefined),
+ cacheMiss: () => undefined,
fetched: Promise.resolve({ bot: { id: 'bot' }, rows: [] }),
stop: () => {},
}),
diff --git a/apps/web/src/lib/core/agent-session/AgentSession.test.ts b/apps/web/src/lib/core/agent-session/AgentSession.test.ts
index 6eca1f18d2..3eb99b8f76 100644
--- a/apps/web/src/lib/core/agent-session/AgentSession.test.ts
+++ b/apps/web/src/lib/core/agent-session/AgentSession.test.ts
@@ -36,6 +36,7 @@ const logSource = vi.hoisted(() => {
const source = {
Unavailable,
cached: vi.fn(),
+ cacheMiss: vi.fn(),
fetched: vi.fn(),
forget: vi.fn(),
watch: vi.fn(),
@@ -50,6 +51,7 @@ const logSource = vi.hoisted(() => {
source.follow = follow;
return {
cached: source.cached(),
+ cacheMiss: source.cacheMiss,
fetched: source.fetched(),
stop: source.stop,
};
@@ -113,6 +115,10 @@ vi.mock('@macro-inc/observability', () => ({
}));
const promptSpans = () =>
telemetry.spans.filter((span) => span.name === 'agent.prompt');
+const loadSpans = () =>
+ telemetry.spans.filter((span) => span.name === 'agent.session.load');
+const renderSpans = () =>
+ telemetry.spans.filter((span) => span.name === 'agent.session.render');
const updates = vi.hoisted(() => ({
listeners: new Set<(event: { agentSessionId: string }) => void>(),
@@ -994,6 +1000,168 @@ describe('AgentSession', () => {
live.release();
});
+ describe('load telemetry', () => {
+ const nextPaint = () =>
+ new Promise((resolve) =>
+ requestAnimationFrame(() => requestAnimationFrame(resolve))
+ );
+
+ it('says why the cache missed', async () => {
+ logSource.cacheMiss.mockReturnValue('network_first');
+ const live = AgentSession.acquire(SESSION);
+ await live.load();
+
+ expect(loadSpans()[0].attributes).toMatchObject({
+ 'agent.session.load.cache_hit': false,
+ 'agent.session.load.cache_miss': 'network_first',
+ });
+ live.release();
+ });
+
+ it('says why a load failed', async () => {
+ harness.get.mockResolvedValueOnce(err([{ code: 'HTTP_ERROR' }]));
+ const live = AgentSession.acquire(SESSION);
+ await expect(live.load()).rejects.toThrow();
+
+ logSource.fetched.mockRejectedValueOnce(
+ new logSource.Unavailable(SESSION, 'failed')
+ );
+ await expect(live.load()).rejects.toThrow();
+
+ harness.get.mockResolvedValueOnce(err([{ code: 'UNAUTHORIZED' }]));
+ await expect(live.load()).rejects.toThrow();
+
+ fold.readSession.mockRejectedValueOnce(new Error('worker died'));
+ await expect(live.load()).rejects.toThrow('worker died');
+
+ expect(
+ loadSpans().map((span) => [
+ span.attributes['agent.session.load.outcome'],
+ span.attributes['agent.session.load.failure'],
+ ])
+ ).toEqual([
+ ['failed', 'session_fetch'],
+ ['failed', 'log_fetch'],
+ ['failed', 'access_denied'],
+ ['failed', 'fold'],
+ ]);
+ live.release();
+ });
+
+ it('holds the render span open until the transcript has painted', async () => {
+ const live = AgentSession.acquire(SESSION);
+ await live.load();
+
+ live.rendered('fetched', [
+ {
+ agentSessionId: SESSION,
+ turn: 0,
+ author: { kind: 'user', userId: null },
+ requestId: null,
+ parts: [{ kind: 'text', text: 'Hi' }],
+ stop: null,
+ pending: false,
+ },
+ {
+ agentSessionId: SESSION,
+ turn: 0,
+ author: { kind: 'agent' },
+ requestId: null,
+ parts: [
+ { kind: 'thought', text: 'hmm' },
+ { kind: 'text', text: 'Hello' },
+ ],
+ stop: null,
+ pending: false,
+ },
+ ]);
+ const [span] = renderSpans();
+ expect(span.ends).toBe(0);
+ expect(span.attributes).toMatchObject({
+ 'agent.session.render.kind': 'fetched',
+ 'agent.session.load.transcript_messages': 2,
+ 'agent.session.load.transcript_turns': 1,
+ 'agent.session.load.transcript_chars': 10,
+ });
+
+ await nextPaint();
+
+ expect(span.ends).toBe(1);
+ expect(
+ span.attributes['agent.session.load.painted_at_ms']
+ ).toBeGreaterThanOrEqual(
+ span.attributes['agent.session.render.since_load_ms'] as number
+ );
+ expect(span.attributes['agent.session.load.paint_ms']).toEqual(
+ expect.any(Number)
+ );
+ live.release();
+ });
+
+ it('attributes the long animation frames before the paint', async () => {
+ const observed: { deliver?: (entries: unknown[]) => void } = {};
+ class FakeObserver {
+ static supportedEntryTypes = ['long-animation-frame'];
+ constructor(callback: (list: { getEntries: () => unknown[] }) => void) {
+ observed.deliver = (entries) =>
+ callback({ getEntries: () => entries });
+ }
+ observe() {}
+ takeRecords() {
+ return [];
+ }
+ disconnect() {}
+ }
+ vi.stubGlobal('PerformanceObserver', FakeObserver);
+ onTestFinished(() => {
+ vi.unstubAllGlobals();
+ });
+ const live = AgentSession.acquire(SESSION);
+ await live.load();
+ const now = performance.now();
+ const frame = (duration: number, scripts: unknown[]) => ({
+ entryType: 'long-animation-frame',
+ startTime: now - duration,
+ duration,
+ blockingDuration: duration - 50,
+ scripts,
+ });
+ observed.deliver?.([
+ frame(80, []),
+ frame(500, [
+ { duration: 100, invoker: 'small', sourceURL: 'a.js' },
+ {
+ duration: 300,
+ invoker: 'ThreadList.positionInitial',
+ sourceURL: `https://app/${'x'.repeat(300)}.js`,
+ },
+ ]),
+ ]);
+
+ live.rendered('fetched', []);
+ await nextPaint();
+ const [span] = renderSpans();
+ // Ended once the frames before the paint have been reported.
+ expect(span.ends).toBe(0);
+ await new Promise((resolve) => setTimeout(resolve, 150));
+
+ expect(span.ends).toBe(1);
+ expect(span.attributes).toMatchObject({
+ 'agent.session.load.loaf_count': 2,
+ 'agent.session.load.loaf_total_ms': 580,
+ 'agent.session.load.loaf_max_ms': 500,
+ 'agent.session.load.loaf_blocking_ms': 480,
+ 'agent.session.load.loaf_top_script_invoker':
+ 'ThreadList.positionInitial',
+ 'agent.session.load.loaf_top_script_ms': 300,
+ });
+ expect(
+ span.attributes['agent.session.load.loaf_top_script_source']
+ ).toHaveLength(200);
+ live.release();
+ });
+ });
+
describe('prompt telemetry', () => {
it('posts a prompt inside its span and ends it at the first painted agent text', async () => {
let postedUnder: RecordedSpan | undefined;
diff --git a/apps/web/src/lib/core/agent-session/AgentSession.ts b/apps/web/src/lib/core/agent-session/AgentSession.ts
index 0b33e661ca..66dc4328bf 100644
--- a/apps/web/src/lib/core/agent-session/AgentSession.ts
+++ b/apps/web/src/lib/core/agent-session/AgentSession.ts
@@ -48,8 +48,14 @@ import type {
SessionBot,
} from '@service-agent-harness/generated/schemas';
import { subscribeGraphqlSoupReconnected } from '@service-storage/graphql-soup';
+import { CombinedError } from '@urql/core';
import { v7 as uuidv7 } from 'uuid';
-import { SessionLoadTrace, traceAcquire } from './load-telemetry';
+import {
+ type LoadFailure,
+ SessionLoadTrace,
+ traceAcquire,
+ transcriptSize,
+} from './load-telemetry';
import { frameDelivery, PromptTrace } from './prompt-telemetry';
import { publishSessionTurn } from './session-turn';
@@ -97,6 +103,15 @@ const inaccessible = (error: unknown) =>
error instanceof AgentSessionLogUnavailable &&
error.reason === 'inaccessible';
+/** Why the log query failed, for the load span. */
+function logFailure(error: unknown): LoadFailure {
+ const cause = error instanceof Error ? error.cause : undefined;
+ if (cause instanceof CombinedError) {
+ return cause.networkError ? 'log_network' : 'log_graphql';
+ }
+ return cause === undefined ? 'log_fetch' : 'log_unreadable';
+}
+
const accessDenied = (errors: { code: string }[]) =>
errors.some(
(error) => error.code === 'UNAUTHORIZED' || error.code === 'FORBIDDEN'
@@ -524,7 +539,8 @@ export class AgentSession {
this.watch.cached.then((cached) => {
this.trace.cacheRead(
readStartedAt,
- cached && { rows: cached.rows.length }
+ cached && { rows: cached.rows.length },
+ this.watch.cacheMiss()
);
return cached;
}),
@@ -546,8 +562,11 @@ export class AgentSession {
}
/** A surface put this session's transcript on screen. */
- rendered(kind: 'cached' | 'fetched'): void {
- this.trace.rendered(kind);
+ rendered(
+ kind: 'cached' | 'fetched',
+ messages: SessionFoldSnapshot['messages']
+ ): void {
+ this.trace.rendered(kind, transcriptSize(messages));
}
private startLoad(): Promise<AgentSessionRecord> {
@@ -585,15 +604,19 @@ export class AgentSession {
if (session.isErr()) {
if (accessDenied(session.error)) {
this.forgetCached();
+ this.trace.failing('access_denied');
throw new AgentSessionAccessDenied(this.id);
}
+ this.trace.failing('session_fetch');
throw new Error(`agent session could not be fetched: ${this.id}`);
}
if (!log.ok) {
if (inaccessible(log.error)) {
this.forgetCached();
+ this.trace.failing('access_denied');
throw new AgentSessionAccessDenied(this.id);
}
+ this.trace.failing(logFailure(log.error));
throw new Error(`agent session log could not be fetched: ${this.id}`, {
cause: log.error,
});
@@ -602,12 +625,17 @@ export class AgentSession {
const foldStartedAt = performance.now();
this.settled = true;
- await this.apply([{ kind: 'snapshot', rows: log.log.rows }]);
- // Inputs can keep arriving while each push is in flight; drain until a
- // check finds nothing, then flip ready so the next one goes straight in.
- await this.drainBuffered();
- this.trace.folded(foldStartedAt);
- this.setTurn((await readSession(this.id)).metadata.turn);
+ try {
+ await this.apply([{ kind: 'snapshot', rows: log.log.rows }]);
+ // Inputs can keep arriving while each push is in flight; drain until a
+ // check finds nothing, then flip ready so the next one goes straight in.
+ await this.drainBuffered();
+ this.trace.folded(foldStartedAt);
+ this.setTurn((await readSession(this.id)).metadata.turn);
+ } catch (error) {
+ this.trace.failing('fold');
+ throw error;
+ }
if (this.resyncRequested) void this.resync();
return { session: session.value, bot: log.log.bot };
}
diff --git a/apps/web/src/lib/core/agent-session/load-telemetry.ts b/apps/web/src/lib/core/agent-session/load-telemetry.ts
index 4b718797db..300db8666b 100644
--- a/apps/web/src/lib/core/agent-session/load-telemetry.ts
+++ b/apps/web/src/lib/core/agent-session/load-telemetry.ts
@@ -15,6 +15,13 @@
import type { Span } from '@macro-inc/observability';
import { Telemetry } from '@macro-inc/observability';
+import type { CacheMiss } from '@queries/agent-session/log';
+import type { FoldedMessage } from '@service-agent-fold/generated/types';
+import {
+ afterLongFrames,
+ observeLongFrames,
+ summarizeFrames,
+} from './long-frames';
/** How an attempt to open a session ended. */
export type LoadOutcome =
@@ -27,6 +34,25 @@ export type LoadOutcome =
/** Still unsettled at {@link STALL_THRESHOLD_MS} — reported, not concluded. */
| 'stalled';
+/** Why a load ended `failed`, bounded so failures group. */
+export type LoadFailure =
+ /** The viewer cannot see the session. */
+ | 'access_denied'
+ /** The session row could not be fetched. */
+ | 'session_fetch'
+ /** The log query failed in transport. */
+ | 'log_network'
+ /** The log query answered with GraphQL errors. */
+ | 'log_graphql'
+ /** The log arrived but its rows could not be read. */
+ | 'log_unreadable'
+ /** The log query failed for no reason it gave. */
+ | 'log_fetch'
+ /** The fold worker failed on the snapshot. */
+ | 'fold'
+ /** Nothing above; a bug in the classification. */
+ | 'unknown';
+
/**
* How long a load may go unsettled before it is reported as stalled.
*
@@ -41,13 +67,6 @@ export type LoadOutcome =
*/
const STALL_THRESHOLD_MS = 15_000;
-/**
- * One attempt to open a session, from the first acquisition to whatever
- * ended it.
- *
- * Held rather than run around an operation because the operation it measures
- * can be abandoned: `release` ends the span from outside the load.
- */
/** A stage of the load, named as its measure and attribute suffix. */
export type LoadStage =
/** The cached log was looked up. */
@@ -63,7 +82,41 @@ export type LoadStage =
/** A surface painted the cached transcript. */
| 'warm_render'
/** A surface painted the fetched transcript. */
- | 'render';
+ | 'render'
+ /** The cached transcript laid out and a frame painted it. */
+ | 'warm_painted'
+ /** The fetched transcript laid out and a frame painted it. */
+ | 'painted';
+
+/** What a surface rendered, so slow paints can be grouped by size. */
+export type TranscriptSize = {
+ messages: number;
+ turns: number;
+ /** Text and thought characters: roughly the markdown laid out. */
+ chars: number;
+};
+
+export function transcriptSize(
+ messages: readonly FoldedMessage[]
+): TranscriptSize {
+ const turns = new Set<number>();
+ let chars = 0;
+ for (const message of messages) {
+ turns.add(message.turn);
+ for (const part of message.parts) {
+ if (part.kind === 'text' || part.kind === 'thought') {
+ chars += part.text.length;
+ }
+ }
+ }
+ return { messages: messages.length, turns: turns.size, chars };
+}
+
+/**
+ * How long a render waits for its paint before the span is ended without
+ * one: a hidden tab runs no animation frames.
+ */
+const PAINT_TIMEOUT_MS = 5_000;
/**
* One attempt to open a session, from the first acquisition to whatever
@@ -85,7 +138,9 @@ export class SessionLoadTrace {
readonly #sessionId: string;
readonly #startedAt = performance.now();
readonly #recorded: Record<string, string | number | boolean> = {};
+ readonly #frames = observeLongFrames();
#ended = false;
+ #failure: LoadFailure | undefined;
#stallTimer: ReturnType<typeof setTimeout> | undefined;
constructor(sessionId: string) {
@@ -95,6 +150,8 @@ export class SessionLoadTrace {
() => this.end('stalled'),
STALL_THRESHOLD_MS
);
+ // A paint later than this is not a load any more; stop watching frames.
+ setTimeout(() => this.#frames?.stop(), STALL_THRESHOLD_MS);
}
/**
@@ -131,13 +188,23 @@ export class SessionLoadTrace {
* The cached log was looked up: `hit` is what it held, or nothing. A miss
* is a stage too; its duration is what a cold open paid for looking.
*/
- cacheRead(startedAt: number, hit: { rows: number } | undefined): void {
+ cacheRead(
+ startedAt: number,
+ hit: { rows: number } | undefined,
+ miss: CacheMiss | undefined
+ ): void {
this.stage('cache_read', startedAt, {
cache_hit: hit !== undefined,
...(hit && { cache_rows: hit.rows }),
+ ...(!hit && miss && { cache_miss: miss }),
});
}
+ /** Why the load is about to fail; read by `end('failed')`. */
+ failing(failure: LoadFailure): void {
+ this.#failure ??= failure;
+ }
+
/** The log arrived; `rows` is what the fold is about to be given. */
fetched(startedAt: number, rows: number): void {
this.stage('log_fetch', startedAt, { log_rows: rows });
@@ -152,36 +219,87 @@ export class SessionLoadTrace {
}
/**
- * A surface put a transcript on screen. `cached` is the warm paint from
- * the cached log; the other is the fetched one. Measured from the start
- * of the load, which is what a person waited. Its own span rather than a
- * stage: the load span has usually ended by the time a surface paints,
- * and an ended span drops what is set on it.
+ * A surface handed a transcript to the DOM. `cached` is the warm paint
+ * from the cached log; the other is the fetched one. Measured from the
+ * start of the load, which is what a person waited. Its own span rather
+ * than a stage: the load span has usually ended by the time a surface
+ * paints, and an ended span drops what is set on it.
+ *
+ * The DOM is not the screen: a long transcript lays out and paints in one
+ * long frame after this. The span stays open until a frame has painted,
+ * and records that as `painted_at_ms`, with the long animation frames
+ * since the load started. It ends a little after the paint, once those
+ * frames are reported, so `paint_ms` is the paint, not the span length.
*/
- rendered(kind: 'cached' | 'fetched'): void {
+ rendered(kind: 'cached' | 'fetched', size: TranscriptSize): void {
const stage: LoadStage = kind === 'cached' ? 'warm_render' : 'render';
const now = performance.now();
const ms = now - this.#startedAt;
this.#recorded[`agent.session.load.${stage}_ms`] = Math.round(ms);
const span = start('agent.session.render', this.#sessionId);
+ const record: Record<string, string | number | boolean> = {
+ 'agent.session.render.kind': kind,
+ 'agent.session.render.since_load_ms': Math.round(ms),
+ 'agent.session.render.hidden':
+ typeof document !== 'undefined' && document.hidden,
+ 'agent.session.load.transcript_messages': size.messages,
+ 'agent.session.load.transcript_turns': size.turns,
+ 'agent.session.load.transcript_chars': size.chars,
+ };
try {
- span?.setAttr('agent.session.render.kind', kind);
- span?.setAttr('agent.session.render.since_load_ms', ms);
- span?.end();
+ for (const [name, value] of Object.entries(record)) {
+ span?.setAttr(name, value);
+ }
performance.measure(`agent.session.load.${stage}`, {
start: this.#startedAt,
end: now,
detail: { sessionId: this.#sessionId, kind },
});
- if (loadDebugEnabled()) {
- console.debug('[agent-session] render', this.#sessionId, {
- kind,
- since_load_ms: Math.round(ms),
- });
- }
} catch {
// See the module comment.
}
+ afterPaint(PAINT_TIMEOUT_MS, (paintedAt) => {
+ afterLongFrames(this.#frames, () => {
+ try {
+ if (paintedAt === undefined) {
+ record['agent.session.load.paint_timed_out'] = true;
+ } else {
+ const painted: LoadStage =
+ kind === 'cached' ? 'warm_painted' : 'painted';
+ record['agent.session.load.painted_at_ms'] = Math.round(
+ paintedAt - this.#startedAt
+ );
+ record['agent.session.load.paint_ms'] = Math.round(paintedAt - now);
+ Object.assign(
+ record,
+ summarizeFrames(
+ this.#frames,
+ 'agent.session.load',
+ this.#startedAt,
+ paintedAt
+ )
+ );
+ this.#recorded[`agent.session.load.${painted}_ms`] = Math.round(
+ paintedAt - this.#startedAt
+ );
+ performance.measure(`agent.session.load.${painted}`, {
+ start: this.#startedAt,
+ end: paintedAt,
+ detail: { sessionId: this.#sessionId, kind },
+ });
+ }
+ for (const [name, value] of Object.entries(record)) {
+ span?.setAttr(name, value);
+ }
+ span?.end();
+ if (loadDebugEnabled()) {
+ console.debug('[agent-session] render', this.#sessionId, record);
+ }
+ } catch {
+ // See the module comment.
+ }
+ });
+ });
}
/**
@@ -196,6 +314,11 @@ export class SessionLoadTrace {
this.#stallTimer = undefined;
this.#set('agent.session.load.outcome', outcome);
this.#set('agent.session.load.total_ms', this.#since());
+ if (outcome === 'failed') {
+ this.#set('agent.session.load.failure', this.#failure ?? 'unknown');
+ }
+ // Nothing will paint for a load that did not land.
+ if (outcome !== 'loaded') this.#frames?.stop();
try {
performance.measure('agent.session.load', {
start: this.#startedAt,
@@ -260,6 +383,32 @@ export function traceCacheWrite(
}
}
+/**
+ * Calls `then` with the time of the frame after the next one: by then the
+ * DOM written before this call has laid out and painted. With `undefined`
+ * after `timeoutMs` when no frame runs, as in a hidden tab.
+ */
+function afterPaint(
+ timeoutMs: number,
+ then: (paintedAt: number | undefined) => void
+): void {
+ if (typeof requestAnimationFrame === 'undefined') {
+ then(undefined);
+ return;
+ }
+ let done = false;
+ const finish = (paintedAt: number | undefined) => {
+ if (done) return;
+ done = true;
+ clearTimeout(timer);
+ then(paintedAt);
+ };
+ const timer = setTimeout(() => finish(undefined), timeoutMs);
+ requestAnimationFrame(() => {
+ requestAnimationFrame(() => finish(performance.now()));
+ });
+}
+
function loadDebugEnabled(): boolean {
return (
typeof window !== 'undefined' &&
diff --git a/apps/web/src/lib/core/agent-session/long-frames.ts b/apps/web/src/lib/core/agent-session/long-frames.ts
new file mode 100644
index 0000000000..ea1d4ca6f1
--- /dev/null
+++ b/apps/web/src/lib/core/agent-session/long-frames.ts
@@ -0,0 +1,139 @@
+/**
+ * Long animation frames (the Long Animation Frames API) seen while a span
+ * waited for something to paint: the main-thread work a person sat through
+ * that no request timing shows. Feature-detected; absent where unsupported.
+ */
+
+/**
+ * The fields of a `long-animation-frame` entry read here. Typed locally:
+ * the DOM lib does not carry them yet.
+ */
+type LongFrame = PerformanceEntry & {
+ blockingDuration: number;
+ scripts: readonly {
+ duration: number;
+ invoker: string;
+ sourceURL: string;
+ }[];
+};
+
+export type LongFrameWatch = {
+ /** Every frame seen so far, including those not yet delivered. */
+ take: () => readonly LongFrame[];
+ stop: () => void;
+};
+
+/**
+ * Collect long animation frames (50ms+) from now until `stop`. Undefined
+ * where the browser has no Long Animation Frames API.
+ */
+export function observeLongFrames(): LongFrameWatch | undefined {
+ try {
+ if (
+ typeof PerformanceObserver === 'undefined' ||
+ !PerformanceObserver.supportedEntryTypes?.includes('long-animation-frame')
+ ) {
+ return undefined;
+ }
+ const frames: LongFrame[] = [];
+ const collect = (entries: PerformanceEntryList) => {
+ // Entries of the observed type carry the fields `LongFrame` reads.
+ for (const entry of entries) frames.push(entry as LongFrame);
+ };
+ const observer = new PerformanceObserver((list) =>
+ collect(list.getEntries())
+ );
+ observer.observe({ type: 'long-animation-frame' });
+ let stopped = false;
+ return {
+ take: () => {
+ if (!stopped) collect(observer.takeRecords());
+ return frames;
+ },
+ stop: () => {
+ if (stopped) return;
+ collect(observer.takeRecords());
+ stopped = true;
+ observer.disconnect();
+ },
+ };
+ } catch {
+ return undefined;
+ }
+}
+
+/**
+ * How long a frame's entry may take to reach the observer after the frame:
+ * the browser queues it once the frame has painted, so a paint observed
+ * by animation frame comes before the entry describing it.
+ */
+const ENTRY_DELAY_MS = 100;
+
+/**
+ * Calls `then` once the frames up to now have reached `watch`: at once
+ * without one, otherwise after {@link ENTRY_DELAY_MS}.
+ */
+export function afterLongFrames(
+ watch: LongFrameWatch | undefined,
+ then: () => void
+): void {
+ if (!watch) {
+ then();
+ return;
+ }
+ setTimeout(then, ENTRY_DELAY_MS);
+}
+
+/** Longest a script attribution may be on a span. */
+const SCRIPT_ATTRIBUTION_MAX = 200;
+
+/**
+ * The long frames that overlap `[from, to]`, as `<prefix>.loaf_*`
+ * attributes: how many, how long, how much of that blocked input, and the
+ * script that held the longest one.
+ */
+export function summarizeFrames(
+ watch: LongFrameWatch | undefined,
+ prefix: string,
+ from: number,
+ to: number
+): Record<string, number | string> {
+ if (!watch) return {};
+ const frames = watch
+ .take()
+ .filter(
+ (frame) => frame.startTime < to && frame.startTime + frame.duration > from
+ );
+ let total = 0;
+ let blocking = 0;
+ let longest: LongFrame | undefined;
+ for (const frame of frames) {
+ total += frame.duration;
+ blocking += frame.blockingDuration;
+ if (!longest || frame.duration > longest.duration) longest = frame;
+ }
+ const summary: Record<string, number | string> = {
+ [`${prefix}.loaf_count`]: frames.length,
+ [`${prefix}.loaf_total_ms`]: Math.round(total),
+ [`${prefix}.loaf_max_ms`]: Math.round(longest?.duration ?? 0),
+ [`${prefix}.loaf_blocking_ms`]: Math.round(blocking),
+ };
+ const script = longest?.scripts.reduce<
+ LongFrame['scripts'][number] | undefined
+ >(
+ (top, script) => (!top || script.duration > top.duration ? script : top),
+ undefined
+ );
+ if (script) {
+ summary[`${prefix}.loaf_top_script_source`] = script.sourceURL.slice(
+ 0,
+ SCRIPT_ATTRIBUTION_MAX
+ );
+ summary[`${prefix}.loaf_top_script_invoker`] = script.invoker.slice(
+ 0,
+ SCRIPT_ATTRIBUTION_MAX
+ );
+ summary[`${prefix}.loaf_top_script_ms`] = Math.round(script.duration);
+ }
+ return summary;
+}
diff --git a/apps/web/src/lib/core/agent-session/prompt-telemetry.ts b/apps/web/src/lib/core/agent-session/prompt-telemetry.ts
index d23c7036bb..426f782d66 100644
--- a/apps/web/src/lib/core/agent-session/prompt-telemetry.ts
+++ b/apps/web/src/lib/core/agent-session/prompt-telemetry.ts
@@ -20,6 +20,11 @@ import type {
FoldedStreamEvent,
} from '@service-agent-fold/generated/types';
import { match } from 'ts-pattern';
+import {
+ afterLongFrames,
+ observeLongFrames,
+ summarizeFrames,
+} from './long-frames';
import { observeRenderedAnswer } from './render-telemetry';
/** How a sent prompt ended, as far as this span is concerned. */
@@ -161,6 +166,7 @@ export class PromptTrace {
readonly #renderers = new Set<() => void>();
#responses: ResponseWatch[] = [];
#resourceObserver: PerformanceObserver | undefined;
+ readonly #frames = observeLongFrames();
constructor(sessionId: string, options: PromptTraceOptions) {
this.#sessionId = sessionId;
@@ -301,22 +307,33 @@ export class PromptTrace {
this.#stopWatchingResponses();
if (outcome === 'hidden') this.#set('agent.prompt.hidden', true);
this.#set('agent.prompt.outcome', outcome);
- this.#set(
- 'agent.prompt.total_ms',
- Math.round(performance.now() - this.#startedAt)
- );
+ const now = performance.now();
+ this.#set('agent.prompt.total_ms', Math.round(now - this.#startedAt));
this.#ended = true;
try {
if (outcome === 'failed' && error !== undefined) this.#span?.error(error);
- this.#span?.end();
performance.measure('agent.prompt', {
start: this.#startedAt,
- end: performance.now(),
+ end: now,
detail: { sessionId: this.#sessionId, outcome },
});
} catch {
// See the module comment.
}
+ // The long frames up to the end, which includes the frame that painted.
+ afterLongFrames(this.#frames, () => {
+ try {
+ for (const [name, value] of Object.entries(
+ summarizeFrames(this.#frames, 'agent.prompt', this.#startedAt, now)
+ )) {
+ this.#span?.setAttr(name, value);
+ }
+ this.#frames?.stop();
+ this.#span?.end();
+ } catch {
+ // See the module comment.
+ }
+ });
}
#observeMessage(message: FoldedMessage, delivery?: FrameDelivery): void {
diff --git a/apps/web/src/lib/queries/agent-session/log.test.ts b/apps/web/src/lib/queries/agent-session/log.test.ts
index 2e517ce12a..f12aa93431 100644
--- a/apps/web/src/lib/queries/agent-session/log.test.ts
+++ b/apps/web/src/lib/queries/agent-session/log.test.ts
@@ -121,6 +121,35 @@ afterEach(() => {
const settle = () => vi.advanceTimersByTimeAsync(0);
describe('watchAgentSessionLog', () => {
+ it('says why the cache missed', async () => {
+ const hit = watchAgentSessionLog(SESSION);
+ query.next(fromCache(logData([1])));
+ query.next(fromNetwork(logData([1])));
+ await hit.cached;
+ expect(hit.cacheMiss()).toBeUndefined();
+
+ const networkFirst = watchAgentSessionLog(SESSION);
+ query.next(fromNetwork(logData([1])));
+ expect(await networkFirst.cached).toBeUndefined();
+ expect(networkFirst.cacheMiss()).toBe('network_first');
+
+ const empty = watchAgentSessionLog(SESSION);
+ query.next(fromCache({ user: { id: 'user-1', agentSession: null } }));
+ query.next(fromNetwork(logData([1])));
+ await empty.cached;
+ expect(empty.cacheMiss()).toBe('empty');
+
+ const networkOnly = watchAgentSessionLog(SESSION, 'network-only');
+ query.next(fromNetwork(logData([1])));
+ await networkOnly.cached;
+ expect(networkOnly.cacheMiss()).toBe('network_only');
+
+ const stopped = watchAgentSessionLog(SESSION);
+ stopped.stop();
+ await stopped.cached;
+ expect(stopped.cacheMiss()).toBe('stopped');
+ });
+
it('answers from the cache, then the network', async () => {
const watch = watchAgentSessionLog(SESSION);
query.next(fromCache(logData([1])));
diff --git a/apps/web/src/lib/queries/agent-session/log.ts b/apps/web/src/lib/queries/agent-session/log.ts
index ddcf1212b5..7872d60bcb 100644
--- a/apps/web/src/lib/queries/agent-session/log.ts
+++ b/apps/web/src/lib/queries/agent-session/log.ts
@@ -57,12 +57,29 @@ export class AgentSessionLogUnavailable extends Error {
}
}
+/** Why {@link SessionLogWatch.cached} came back empty. */
+export type CacheMiss =
+ /** No persistent cache in this client, or it was disabled. */
+ | 'no_cache'
+ /** The watch asked for the network only. */
+ | 'network_only'
+ /** The network answered before the cache read hit, or the read missed. */
+ | 'network_first'
+ /** The cache answered without this session's log, or with an error. */
+ | 'empty'
+ /** The cached log could not be read into rows. */
+ | 'unreadable'
+ /** The watch stopped first. */
+ | 'stopped';
+
export type SessionLogWatch = {
/**
* The log as the cache last held it, or nothing when the cache has no
* copy or the network answered first. Never rejects.
*/
cached: Promise<SessionLogSnapshot | undefined>;
+ /** Why `cached` resolved to nothing, once it has; for telemetry. */
+ cacheMiss: () => CacheMiss | undefined;
/** The log as the server holds it now. Rejects with {@link AgentSessionLogUnavailable}. */
fetched: Promise<SessionLogSnapshot>;
/**
@@ -149,6 +166,20 @@ export function watchAgentSessionLog(
follow?: SessionLogFollow
): SessionLogWatch {
let resolveCached!: (snapshot: SessionLogSnapshot | undefined) => void;
+ let cacheSettled = false;
+ let cacheMiss: CacheMiss | undefined;
+ let cacheAnsweredEmpty = false;
+ const hitCache = (snapshot: SessionLogSnapshot) => {
+ if (cacheSettled) return;
+ cacheSettled = true;
+ resolveCached(snapshot);
+ };
+ const missCache = (reason: CacheMiss) => {
+ if (cacheSettled) return;
+ cacheSettled = true;
+ cacheMiss = reason;
+ resolveCached(undefined);
+ };
let resolveFetched!: (snapshot: SessionLogSnapshot) => void;
let rejectFetched!: (error: AgentSessionLogUnavailable) => void;
const cached = new Promise<SessionLogSnapshot | undefined>((resolve) => {
@@ -167,7 +198,15 @@ export function watchAgentSessionLog(
: () => undefined;
const settleFetched = (result: OperationResult<AgentSessionLogQuery>) => {
- resolveCached(undefined);
+ missCache(
+ policy === 'network-only'
+ ? 'network_only'
+ : !getGraphqlCacheHost()
+ ? 'no_cache'
+ : cacheAnsweredEmpty
+ ? 'empty'
+ : 'network_first'
+ );
if (result.error) {
rejectFetched(
new AgentSessionLogUnavailable(sessionId, 'failed', {
@@ -213,10 +252,12 @@ export function watchAgentSessionLog(
// network answer settles both.
if (log && !result.error) {
try {
- resolveCached(toSnapshot(sessionId, log));
+ hitCache(toSnapshot(sessionId, log));
} catch {
- resolveCached(undefined);
+ missCache('unreadable');
}
+ } else {
+ cacheAnsweredEmpty = true;
}
return;
}
@@ -227,12 +268,13 @@ export function watchAgentSessionLog(
return {
cached,
+ cacheMiss: () => cacheMiss,
fetched,
stop: () => {
unsubscribe();
unsubscribeFollow();
appender?.flush();
- resolveCached(undefined);
+ missCache('stopped');
rejectFetched(new AgentSessionLogUnavailable(sessionId, 'failed'));
},
};