Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(agents): instrument opening a session, and the Cursor poll's 429 (#6668)
Wolf Mermelstein442927564160
diff --git a/apps/web/src/lib/core/agent-session/AgentSession.test.ts b/apps/web/src/lib/core/agent-session/AgentSession.test.tsindex d3a9f99e58..28e7b2527c 100644--- a/apps/web/src/lib/core/agent-session/AgentSession.test.ts+++ b/apps/web/src/lib/core/agent-session/AgentSession.test.ts@@ -40,7 +40,7 @@ vi.mock('@queries/agent-session/queue-sync', () => ({subscribeSocketSessionStarted: socket.subscribeSocketSessionStarted,}));-import { AgentSession } from './AgentSession';+import { AgentSession, AgentSessionReleased } from './AgentSession';import { resetSessionTurns, sessionTurn } from './session-turn';const SESSION = '01a0abed-279f-724c-9f49-60dbedc79b6e';@@ -119,6 +119,21 @@ describe('AgentSession', () => {live.release();});+ it('names a load abandoned by its last release, rather than failing it', async () => {+ const log = deferred<LogResult>();+ harness.getLog.mockReturnValue(log.promise);++ const live = AgentSession.acquire(SESSION);+ const loading = live.load();+ // The surface goes away while the log is still on the wire - a list row+ // scrolling out, or a route change - and nothing is left to render it.+ live.release();+ log.resolve(logOf([row(1)]));++ await expect(loading).rejects.toBeInstanceOf(AgentSessionReleased);+ expect(fold.pushSession).not.toHaveBeenCalled();+ });+it('ignores rows for sessions nobody has open', () => {AgentSession.ingest({ agentSessionId: 'other', entries: [row(1)] });expect(fold.pushSession).not.toHaveBeenCalled();diff --git a/apps/web/src/lib/core/agent-session/AgentSession.ts b/apps/web/src/lib/core/agent-session/AgentSession.tsindex 289da98647..d55c5e9460 100644--- a/apps/web/src/lib/core/agent-session/AgentSession.ts+++ b/apps/web/src/lib/core/agent-session/AgentSession.ts@@ -34,6 +34,7 @@ import type {SessionBot,} from '@service-agent-harness/generated/schemas';import { v7 as uuidv7 } from 'uuid';+import { SessionLoadTrace, traceAcquire } from './load-telemetry';import { publishSessionTurn } from './session-turn';export type AgentSessionListener = (events: FoldedStreamEvent[]) => void;@@ -48,6 +49,21 @@ export type IssueResult = Awaited<ReturnType<typeof agentHarnessServiceClient.control>>;+/**+ * The load was abandoned because every surface holding the session let go+ * before it finished.+ *+ * Its own type because it is not a fault: a row that scrolls out of the list+ * mid-fetch is ordinary, and a caller that cannot tell it apart from a failed+ * fetch either reports a phantom error or hides a real one.+ */+export class AgentSessionReleased extends Error {+ constructor(readonly sessionId: string) {+ super(`agent session released: ${sessionId}`);+ this.name = 'AgentSessionReleased';+ }+}+/*** Whether this action takes the turn, rather than riding alongside one. ACP* runs a prompt at a time, so these are the ones the harness queues; mirrors@@ -66,9 +82,11 @@ export class AgentSession {* machine and drops the subscriptions.*/static acquire(id: string): AgentSession {- const session = AgentSession.open.get(id) ?? new AgentSession(id);+ const open = AgentSession.open.get(id);+ const session = open ?? new AgentSession(id);AgentSession.open.set(id, session);session.references += 1;+ traceAcquire(id, open === undefined, session.references);return session;}@@ -106,6 +124,8 @@ export class AgentSession {private chain: Promise<void> = Promise.resolve();private loading: Promise<AgentSessionRecord>;private loadFailed = false;+ /** The span for the attempt in flight, ended by whatever settles it. */+ private trace: SessionLoadTrace;/*** The fold's turn state, tracked off the same events listeners see. What* {@link issue} reads to know whether an action will reach the runtime or@@ -126,6 +146,7 @@ export class AgentSession {this.unsubscribeSocket = subscribeSocketSessionStarted(() => {void this.resync();});+ this.trace = new SessionLoadTrace(id);this.loading = this.startLoad();}@@ -137,6 +158,7 @@ export class AgentSession {load(): Promise<AgentSessionRecord> {if (this.loadFailed) {this.loadFailed = false;+ this.trace = new SessionLoadTrace(this.id);this.loading = this.startLoad();}return this.loading;@@ -287,16 +309,29 @@ export class AgentSession {if (AgentSession.open.get(this.id) === this)AgentSession.open.delete(this.id);this.closed = true;+ // Ended here rather than where the load notices: a fetch that never+ // answers never reaches that check, and an unended span never reports.+ this.trace.end('released');this.listeners.clear();this.unsubscribeSocket();closeSession(this.id);}private startLoad(): Promise<AgentSessionRecord> {- return this.fetchAndFold().catch((error: unknown) => {- this.loadFailed = true;- throw error;- });+ return this.fetchAndFold().then(+ (record) => {+ this.trace.end('loaded');+ return record;+ },+ (error: unknown) => {+ this.loadFailed = true;+ this.trace.end(+ error instanceof AgentSessionReleased ? 'released' : 'failed',+ error+ );+ throw error;+ }+ );}private async fetchAndFold(): Promise<AgentSessionRecord> {@@ -310,8 +345,10 @@ export class AgentSession {if (log.isErr()) {throw new Error(`agent session log could not be fetched: ${this.id}`);}- if (this.closed) throw new Error(`agent session released: ${this.id}`);+ if (this.closed) throw new AgentSessionReleased(this.id);+ this.trace.fetched(log.value.entries.length);+ const foldStartedAt = performance.now();await this.apply([{ kind: 'snapshot', rows: log.value.entries }]);// Inputs can keep arriving while each push is in flight; drain until a// check finds nothing, then flip ready so the next one goes straight in.@@ -321,6 +358,7 @@ export class AgentSession {await this.apply(inputs);}this.ready = true;+ this.trace.folded(foldStartedAt);this.setTurn((await readSession(this.id)).metadata.turn);return { session: session.value, bot: log.value.bot };}diff --git a/apps/web/src/lib/core/agent-session/load-telemetry.ts b/apps/web/src/lib/core/agent-session/load-telemetry.tsnew file mode 100644index 0000000000..e3d72f9d45--- /dev/null+++ b/apps/web/src/lib/core/agent-session/load-telemetry.ts@@ -0,0 +1,143 @@+/**+ * Telemetry for opening a session: what a surface waited for before it could+ * render, and why the wait ended.+ *+ * The backend traces a session thoroughly once a request reaches it. What it+ * cannot see is the half that a person actually experiences — a load that+ * never resolved, a surface that gave up and released mid-fetch, or the same+ * log fetched over and over because rows kept acquiring and dropping the+ * session. That gap is what this fills: one span per attempt to open a+ * session, always ended, always carrying the outcome.+ *+ * Failures here are swallowed. Telemetry that can break a session load is+ * worse than no telemetry.+ */++import type { Span } from '@macro-inc/observability';+import { Telemetry } from '@macro-inc/observability';++/** How an attempt to open a session ended. */+export type LoadOutcome =+ /** The log was fetched and folded; the surface can render. */+ | 'loaded'+ /** Every holder released before the load finished, so nobody wanted it. */+ | 'released'+ /** The fetch or the fold failed. */+ | 'failed'+ /** Still unsettled at {@link STALL_THRESHOLD_MS} — reported, not concluded. */+ | 'stalled';++/**+ * How long a load may go unsettled before it is reported as stalled.+ *+ * A span reaches the exporter only when it ends, so a load that never+ * settles would otherwise be the one case that produces no telemetry at all —+ * precisely the case worth seeing. Past this the span is closed and reported;+ * if the load resolves afterwards it is the {@link SessionLoadTrace.end}+ * no-op, and the surface is none the wiser.+ *+ * Well past a cold load of the largest session observed (a few hundred+ * milliseconds), so a slow session is never reported as a stuck one.+ */+const STALL_THRESHOLD_MS = 15_000;++/**+ * One attempt to open a session, from the first acquisition to whatever+ * ended it.+ *+ * Held rather than run around an operation because the operation it measures+ * can be abandoned: `release` ends the span from outside the load.+ */+export class SessionLoadTrace {+ readonly #span: Span | undefined;+ readonly #startedAt = performance.now();+ #ended = false;+ #stallTimer: ReturnType<typeof setTimeout> | undefined;++ constructor(sessionId: string) {+ this.#span = start('agent.session.load', sessionId);+ this.#stallTimer = setTimeout(+ () => this.end('stalled'),+ STALL_THRESHOLD_MS+ );+ }++ /** The log arrived; `rows` is what the fold is about to be given. */+ fetched(rows: number): void {+ this.#set('agent.session.log.rows', rows);+ this.#set('agent.session.load.fetch_ms', this.#since());+ }++ /** The snapshot is folded and the surface can read the session. */+ folded(foldStartedAt: number): void {+ this.#set('agent.session.load.fold_ms', performance.now() - foldStartedAt);+ }++ /**+ * End the span with how the attempt turned out. Safe to call more than+ * once; the first outcome is the one recorded, so a release racing a+ * failure does not rewrite it.+ */+ end(outcome: LoadOutcome, error?: unknown): void {+ if (this.#ended) return;+ this.#ended = true;+ clearTimeout(this.#stallTimer);+ this.#stallTimer = undefined;+ this.#set('agent.session.load.outcome', outcome);+ this.#set('agent.session.load.total_ms', this.#since());+ try {+ // A release is the expected end of an abandoned load, not a fault:+ // recording it as an error would bury the ones worth looking at.+ if (outcome === 'failed' && error !== undefined) this.#span?.error(error);+ this.#span?.end();+ } catch {+ // See the module comment.+ }+ }++ #since(): number {+ return performance.now() - this.#startedAt;+ }++ #set(name: string, value: string | number): void {+ try {+ this.#span?.setAttr(name, value);+ } catch {+ // See the module comment.+ }+ }+}++/**+ * Record that a surface took a reference to a session.+ *+ * `created` separates a shared instance being reused - free - from a new one,+ * which refetches the whole log. A session id that keeps creating instances+ * is a surface acquiring and releasing in a loop, which is invisible from the+ * backend: it sees only an unexplained run of identical log fetches.+ */+export function traceAcquire(+ sessionId: string,+ created: boolean,+ references: number+): void {+ const span = start('agent.session.acquire', sessionId);+ if (!span) return;+ try {+ span.setAttr('agent.session.acquire.created', created);+ span.setAttr('agent.session.acquire.references', references);+ span.end();+ } catch {+ // See the module comment.+ }+}++function start(name: string, sessionId: string): Span | undefined {+ try {+ const span = Telemetry.span(name);+ span.setAttr('agent.session.id', sessionId);+ return span;+ } catch {+ return undefined;+ }+}diff --git a/apps/web/src/lib/core/agent-session/use-session-turn.test.tsx b/apps/web/src/lib/core/agent-session/use-session-turn.test.tsxindex 9d1925824a..da4705cdaf 100644--- a/apps/web/src/lib/core/agent-session/use-session-turn.test.tsx+++ b/apps/web/src/lib/core/agent-session/use-session-turn.test.tsx@@ -39,7 +39,9 @@ describe('useSessionTurn', () => {});it('acquires a session that just produced a live frame', () => {- const load = vi.fn();+ // Resolves like the real one: the hook drops the load's rejection, so a+ // mock that returns nothing would not exercise that path.+ const load = vi.fn().mockResolvedValue(undefined);const release = vi.fn();session.get.mockReturnValue(undefined);session.acquire.mockReturnValue({ load, release });diff --git a/apps/web/src/lib/core/agent-session/use-session-turn.ts b/apps/web/src/lib/core/agent-session/use-session-turn.tsindex 328e846bd5..1c59c21a12 100644--- a/apps/web/src/lib/core/agent-session/use-session-turn.ts+++ b/apps/web/src/lib/core/agent-session/use-session-turn.ts@@ -37,7 +37,10 @@ export function useSessionTurn(const id = sessionId();if (!follow()) return;const session = AgentSession.acquire(id);- void session.load();+ // A row follows a session to show its turn and renders nothing from the+ // load, so a load it abandons by unmounting is not this caller's problem.+ // Left floating it surfaced as an unhandled rejection on every scroll.+ void session.load().catch(() => undefined);onCleanup(() => session.release());});diff --git a/crates/agent_session/src/inbound/axum_router.rs b/crates/agent_session/src/inbound/axum_router.rsindex 6427567e7a..bd00d472e1 100644--- a/crates/agent_session/src/inbound/axum_router.rs+++ b/crates/agent_session/src/inbound/axum_router.rs@@ -1302,7 +1302,14 @@ pub struct AgentSessionLogResponse {////// An unknown session is an error: the response has to name the session's/// agent, and a session that never existed has none to name.-#[tracing::instrument(skip_all, fields(session_id = %session_id), err(Debug))]+#[tracing::instrument(+ skip_all,+ fields(+ session_id = %session_id,+ agent.session.log.rows = tracing::field::Empty,+ ),+ err(Debug)+)]pub async fn get_agent_session_log_handler<T: AgentSessionService,Access: EntityAccessService,@@ -1317,6 +1324,12 @@ pub async fn get_agent_session_log_handler<.session_log(AgentSessionId::new_from_uuid(session_id)).await?;+ // How much history this response carries. Without it the endpoint's+ // latency cannot be read against the size of the session that produced+ // it, and "is this slow because the session is huge" costs a trip to the+ // database to answer.+ tracing::Span::current().record("agent.session.log.rows", log.entries.len());+Ok(Json(AgentSessionLogResponse {bot: log.bot,entries: log.entries.into_iter().map(Into::into).collect(),diff --git a/crates/cursor_cloud_agents/src/api.rs b/crates/cursor_cloud_agents/src/api.rsindex 523ad4732b..bbd74fd791 100644--- a/crates/cursor_cloud_agents/src/api.rs+++ b/crates/cursor_cloud_agents/src/api.rs@@ -494,7 +494,14 @@ impl CursorClient {}impl CursorAgents for CursorClient {- #[tracing::instrument(skip(self), err)]+ #[tracing::instrument(+ skip(self),+ err,+ fields(+ cursor.poll.status = tracing::field::Empty,+ cursor.poll.rate_limited = tracing::field::Empty,+ )+ )]async fn raw_result(&self,agent: &CursorAgentId,@@ -508,8 +515,33 @@ impl CursorAgents for CursorClient {.await.map_err(|e| rootcause::report!(e))?;let status = response.status();+ let retry_after = response+ .headers()+ .get(reqwest::header::RETRY_AFTER)+ .and_then(|value| value.to_str().ok())+ .map(str::to_owned);let text = response.text().await.map_err(|e| rootcause::report!(e))?;+ let span = tracing::Span::current();+ span.record("cursor.poll.status", status.as_u16());+ span.record(+ "cursor.poll.rate_limited",+ status == reqwest::StatusCode::TOO_MANY_REQUESTS,+ );if !status.is_success() {+ // The rate limit is per API key and shared by every session this+ // deployment drives, so one session's 429 is a fact about the+ // fleet, not about this run. Logged in its own right: until now+ // it existed only inside an error chain that reached the reader+ // and nothing else, so "are we over the limit" was unanswerable+ // from telemetry.+ if status == reqwest::StatusCode::TOO_MANY_REQUESTS {+ tracing::warn!(+ %agent,+ %run,+ retry_after = retry_after.as_deref().unwrap_or("unset"),+ "Cursor run poll was rate limited"+ );+ }return Err(rootcause::report!("Cursor run poll failed: {status}: {text}"));