Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
/**
* Telemetry for opening a session: what a surface waited for before it could
* render, and why the wait ended.
*
* The backend traces a session thoroughly once a request reaches it. What it
* cannot see is the half that a person actually experiences — a load that
* never resolved, a surface that gave up and released mid-fetch, or the same
* log fetched over and over because rows kept acquiring and dropping the
* session. That gap is what this fills: one span per attempt to open a
* session, always ended, always carrying the outcome.
*
* Failures here are swallowed. Telemetry that can break a session load is
* worse than no telemetry.
*/
import type { Span } from '@macro-inc/observability';
import { Telemetry } from '@macro-inc/observability';
import type { CacheMiss } from '@queries/agent-session/log';
import type { FoldedMessage } from '@service-agent-fold/generated/types';
import {
afterLongFrames,
observeLongFrames,
summarizeFrames,
} from './long-frames';
/** How an attempt to open a session ended. */
export type LoadOutcome =
/** The log was fetched and folded; the surface can render. */
| 'loaded'
/** Every holder released before the load finished, so nobody wanted it. */
| 'released'
/** The fetch or the fold failed. */
| 'failed'
/** Still unsettled at {@link STALL_THRESHOLD_MS} — reported, not concluded. */
| 'stalled'
/** Already loaded when acquired, so nothing was fetched or folded. */
| 'reused';
/** Why a load ended `failed`, bounded so failures group. */
export type LoadFailure =
/** The viewer cannot see the session. */
| 'access_denied'
/**
* The session was refused, but this tab had just created it: a create that
* had not finished becoming readable, waited out and still refused. Its own
* value so a race that outlasts the retries does not land among the real
* refusals.
*/
| 'not_ready'
/** The session row could not be fetched. */
| 'session_fetch'
/** The log query failed in transport. */
| 'log_network'
/** The log query answered with GraphQL errors. */
| 'log_graphql'
/** The log arrived but its rows could not be read. */
| 'log_unreadable'
/** The log query failed for no reason it gave. */
| 'log_fetch'
/** The fold worker failed on the snapshot. */
| 'fold'
/**
* The fold worker's script never loaded, so no snapshot could be folded at
* all. Almost always this tab's build being replaced while it was open, so
* it is counted apart from `fold`: nothing is wrong with the fold, and the
* fix is a deploy-retention one.
*/
| 'worker_unavailable'
/** Nothing above; a bug in the classification. */
| 'unknown';
/**
* How long a load may go unsettled before it is reported as stalled.
*
* A span reaches the exporter only when it ends, so a load that never
* settles would otherwise be the one case that produces no telemetry at all —
* precisely the case worth seeing. Past this the span is closed and reported;
* if the load resolves afterwards it is the {@link SessionLoadTrace.end}
* no-op, and the surface is none the wiser.
*
* Well past a cold load of the largest session observed (a few hundred
* milliseconds), so a slow session is never reported as a stuck one.
*/
const STALL_THRESHOLD_MS = 15_000;
/** A stage of the load, named as its measure and attribute suffix. */
export type LoadStage =
/** The cached log was looked up. */
| 'cache_read'
/** The cached log was folded; the surface can show it. */
| 'cache_fold'
/** The session row was fetched. */
| 'session_fetch'
/** The log was fetched. */
| 'log_fetch'
/** The fetched log was folded; the surface can show the truth. */
| 'fold'
/** Snapshot events were projected into the surface's stores. */
| 'projection'
/** A surface painted the cached transcript. */
| 'warm_render'
/** A surface painted the fetched transcript. */
| 'render'
/** The cached transcript laid out and a frame painted it. */
| 'warm_painted'
/** The fetched transcript laid out and a frame painted it. */
| 'painted';
/** What a surface rendered, so slow paints can be grouped by size. */
export type TranscriptSize = {
messages: number;
turns: number;
/** Text and thought characters: roughly the markdown laid out. */
chars: number;
};
export function transcriptSize(
messages: readonly FoldedMessage[]
): TranscriptSize {
const turns = new Set<number>();
let chars = 0;
for (const message of messages) {
turns.add(message.turn);
for (const part of message.parts) {
if (part.kind === 'text' || part.kind === 'thought') {
chars += part.text.length;
}
}
}
return { messages: messages.length, turns: turns.size, chars };
}
/**
* How long a render waits for its paint before the span is ended without
* one: a hidden tab runs no animation frames.
*/
const PAINT_TIMEOUT_MS = 5_000;
/**
* One attempt to open a session, from the first acquisition to whatever
* ended it.
*
* Every stage lands in three places: an attribute and an event on the
* `agent.session.load` span, so a slow open can be read in Datadog; a
* `performance.measure` named `agent.session.load.<stage>`, so the same
* stages show in the DevTools Performance panel and can be read back with
* `performance.getEntriesByType('measure')`; and, when
* `window.debugAgentSessionLoad` is set, one `console.debug` line at the end
* with everything the attempt recorded.
*
* Held rather than run around an operation because the operation it measures
* can be abandoned: `release` ends the span from outside the load.
*/
export class SessionLoadTrace {
readonly #span: Span | undefined;
readonly #sessionId: string;
readonly #startedAt = performance.now();
readonly #recorded: Record<string, string | number | boolean> = {};
readonly #frames = observeLongFrames();
#ended = false;
#failure: LoadFailure | undefined;
#stallTimer: ReturnType<typeof setTimeout> | undefined;
constructor(sessionId: string) {
this.#sessionId = sessionId;
this.#span = start('agent.session.load', sessionId);
this.#stallTimer = setTimeout(
() => this.end('stalled'),
STALL_THRESHOLD_MS
);
// A paint later than this is not a load any more; stop watching frames.
setTimeout(() => this.#frames?.stop(), STALL_THRESHOLD_MS);
}
/**
* A stage finished: it began at `startedAt` (a `performance.now()` value)
* and ends now. Records its duration, and its distance from the start of
* the load as `<stage>_at_ms`, so stages that overlap (the two fetches, a
* cache read beside them) can still be laid out on one line.
*/
stage(
stage: LoadStage,
startedAt: number,
attributes: Record<string, string | number | boolean> = {}
): void {
const now = performance.now();
const ms = now - startedAt;
this.#set(`agent.session.load.${stage}_ms`, ms);
this.#set(`agent.session.load.${stage}_at_ms`, now - this.#startedAt);
for (const [name, value] of Object.entries(attributes)) {
this.#set(`agent.session.load.${name}`, value);
}
try {
this.#span?.event(`agent.session.load.${stage}`, { ms, ...attributes });
performance.measure(`agent.session.load.${stage}`, {
start: startedAt,
end: now,
detail: { sessionId: this.#sessionId, ...attributes },
});
} catch {
// See the module comment.
}
}
/**
* The cached log was looked up: `hit` is what it held, or nothing. A miss
* is a stage too; its duration is what a cold open paid for looking.
*/
cacheRead(
startedAt: number,
hit: { rows: number } | undefined,
miss: CacheMiss | undefined
): void {
this.stage('cache_read', startedAt, {
cache_hit: hit !== undefined,
...(hit && { cache_rows: hit.rows }),
...(!hit && miss && { cache_miss: miss }),
});
}
/** Why the load is about to fail; read by `end('failed')`. */
failing(failure: LoadFailure): void {
this.#failure ??= failure;
}
/** The log arrived; `rows` is what the fold is about to be given. */
fetched(startedAt: number, rows: number): void {
this.stage('log_fetch', startedAt, { log_rows: rows });
// Kept under its historical name: dashboards read it.
this.#set('agent.session.log.rows', rows);
this.#set('agent.session.load.fetch_ms', this.#since());
}
/** The fetched snapshot is folded and the surface can read the session. */
folded(foldStartedAt: number): void {
this.stage('fold', foldStartedAt);
}
/** Listener projection is synchronous work inside the historical fold stage. */
projected(startedAt: number, messages: number): void {
if (this.#ended) return;
this.stage('projection', startedAt, { projection_messages: messages });
}
/**
* The read was refused and tried again, because this tab had created the
* session moments earlier and the refusal reads as a create that has not
* landed. Recorded so the race stays countable once the retry has stopped
* anyone noticing it.
*/
retriedBeforeReady(attempts: number): void {
this.#set('agent.session.load.not_ready_retries', attempts);
}
/**
* A surface handed a transcript to the DOM. `cached` is the warm paint
* from the cached log; the other is the fetched one. Measured from the
* start of the load, which is what a person waited. Its own span rather
* than a stage: the load span has usually ended by the time a surface
* paints, and an ended span drops what is set on it.
*
* The DOM is not the screen: a long transcript lays out and paints in one
* long frame after this. The span stays open until a frame has painted,
* and records that as `painted_at_ms`, with the long animation frames
* since the load started. It ends a little after the paint, once those
* frames are reported, so `paint_ms` is the paint, not the span length.
*/
rendered(kind: 'cached' | 'fetched', size: TranscriptSize): void {
const stage: LoadStage = kind === 'cached' ? 'warm_render' : 'render';
const now = performance.now();
const ms = now - this.#startedAt;
this.#recorded[`agent.session.load.${stage}_ms`] = Math.round(ms);
const span = start('agent.session.render', this.#sessionId);
const record: Record<string, string | number | boolean> = {
'agent.session.render.kind': kind,
'agent.session.render.since_load_ms': Math.round(ms),
'agent.session.render.hidden':
typeof document !== 'undefined' && document.hidden,
'agent.session.load.transcript_messages': size.messages,
'agent.session.load.transcript_turns': size.turns,
'agent.session.load.transcript_chars': size.chars,
};
try {
for (const [name, value] of Object.entries(record)) {
span?.setAttr(name, value);
}
performance.measure(`agent.session.load.${stage}`, {
start: this.#startedAt,
end: now,
detail: { sessionId: this.#sessionId, kind },
});
} catch {
// See the module comment.
}
afterPaint(PAINT_TIMEOUT_MS, (paintedAt) => {
afterLongFrames(this.#frames, () => {
try {
if (paintedAt === undefined) {
record['agent.session.load.paint_timed_out'] = true;
} else {
const painted: LoadStage =
kind === 'cached' ? 'warm_painted' : 'painted';
record['agent.session.load.painted_at_ms'] = Math.round(
paintedAt - this.#startedAt
);
record['agent.session.load.paint_ms'] = Math.round(paintedAt - now);
Object.assign(
record,
summarizeFrames(
this.#frames,
'agent.session.load',
this.#startedAt,
paintedAt
)
);
this.#recorded[`agent.session.load.${painted}_ms`] = Math.round(
paintedAt - this.#startedAt
);
performance.measure(`agent.session.load.${painted}`, {
start: this.#startedAt,
end: paintedAt,
detail: { sessionId: this.#sessionId, kind },
});
}
for (const [name, value] of Object.entries(record)) {
span?.setAttr(name, value);
}
span?.end();
if (loadDebugEnabled()) {
console.debug('[agent-session] render', this.#sessionId, record);
}
} catch {
// See the module comment.
}
});
});
}
/**
* End the span with how the attempt turned out. Safe to call more than
* once; the first outcome is the one recorded, so a release racing a
* failure does not rewrite it.
*/
end(outcome: LoadOutcome, error?: unknown): void {
if (this.#ended) return;
this.#ended = true;
clearTimeout(this.#stallTimer);
this.#stallTimer = undefined;
this.#set('agent.session.load.outcome', outcome);
this.#set('agent.session.load.total_ms', this.#since());
if (outcome === 'failed') {
this.#set('agent.session.load.failure', this.#failure ?? 'unknown');
}
// Nothing will paint for a load that did not land.
if (outcome !== 'loaded' && outcome !== 'reused') this.#frames?.stop();
try {
performance.measure('agent.session.load', {
start: this.#startedAt,
end: performance.now(),
detail: { sessionId: this.#sessionId, outcome },
});
// A release is the expected end of an abandoned load, not a fault:
// recording it as an error would bury the ones worth looking at.
if (outcome === 'failed' && error !== undefined) this.#span?.error(error);
this.#span?.end();
if (loadDebugEnabled()) {
console.debug('[agent-session] load', this.#sessionId, this.#recorded);
}
} catch {
// See the module comment.
}
}
#since(): number {
return performance.now() - this.#startedAt;
}
#set(name: string, value: string | number | boolean): void {
this.#recorded[name] =
typeof value === 'number' ? Math.round(value) : value;
try {
this.#span?.setAttr(name, value);
} catch {
// See the module comment.
}
}
}
/**
* The log subscription ended and is being reopened: the server closed it
* (a subscriber that fell behind, a refused session) or the transport gave
* up. Every frame published until the new one is up reaches the session only
* through the refetch that follows, so a burst of these is a live feed that
* keeps dying.
*/
export function traceFollowResumed(
sessionId: string,
reason: 'error' | 'ended',
attempt: number,
delayMs: number
): void {
const span = start('agent.session.log.follow_resumed', sessionId);
if (!span) return;
try {
span.setAttr('agent.session.log.follow.reason', reason);
span.setAttr('agent.session.log.follow.attempt', attempt);
span.setAttr('agent.session.log.follow.delay_ms', delayMs);
span.end();
} catch {
// See the module comment.
}
}
/**
* A write of the cached log, after the load: its own short span and
* measure, since the load span has usually ended by the time rows stream in.
*/
export function traceCacheWrite(
sessionId: string,
startedAt: number,
rows: number
): void {
const now = performance.now();
const span = start('agent.session.cache.write', sessionId);
try {
span?.setAttr('agent.session.cache.write_ms', now - startedAt);
span?.setAttr('agent.session.cache.rows', rows);
span?.end();
performance.measure('agent.session.cache.write', {
start: startedAt,
end: now,
detail: { sessionId, rows },
});
if (loadDebugEnabled()) {
console.debug('[agent-session] cache write', sessionId, {
rows,
ms: Math.round(now - startedAt),
});
}
} catch {
// See the module comment.
}
}
/**
* Calls `then` with the time of the frame after the next one: by then the
* DOM written before this call has laid out and painted. With `undefined`
* after `timeoutMs` when no frame runs, as in a hidden tab.
*/
function afterPaint(
timeoutMs: number,
then: (paintedAt: number | undefined) => void
): void {
if (typeof requestAnimationFrame === 'undefined') {
then(undefined);
return;
}
let done = false;
const finish = (paintedAt: number | undefined) => {
if (done) return;
done = true;
clearTimeout(timer);
then(paintedAt);
};
const timer = setTimeout(() => finish(undefined), timeoutMs);
requestAnimationFrame(() => {
requestAnimationFrame(() => finish(performance.now()));
});
}
function loadDebugEnabled(): boolean {
return (
typeof window !== 'undefined' &&
Boolean(
(window as { debugAgentSessionLoad?: boolean }).debugAgentSessionLoad
)
);
}
/**
* Record that a surface took a reference to a session.
*
* `created` separates a shared instance being reused - free - from a new one,
* which refetches the whole log. A session id that keeps creating instances
* is a surface acquiring and releasing in a loop, which is invisible from the
* backend: it sees only an unexplained run of identical log fetches.
*/
export function traceAcquire(
sessionId: string,
created: boolean,
references: number
): void {
const span = start('agent.session.acquire', sessionId);
if (!span) return;
try {
span.setAttr('agent.session.acquire.created', created);
span.setAttr('agent.session.acquire.references', references);
span.end();
} catch {
// See the module comment.
}
}
function start(name: string, sessionId: string): Span | undefined {
try {
const span = Telemetry.span(name);
span.setAttr('agent.session.id', sessionId);
return span;
} catch {
return undefined;
}
}