Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(agents): instrument opening a session, and the Cursor poll's 429 (#6668)

Wolf Mermelstein442927564160
diff --git a/apps/web/src/lib/core/agent-session/AgentSession.test.ts b/apps/web/src/lib/core/agent-session/AgentSession.test.ts
index d3a9f99e58..28e7b2527c 100644
--- a/apps/web/src/lib/core/agent-session/AgentSession.test.ts
+++ b/apps/web/src/lib/core/agent-session/AgentSession.test.ts
@@ -40,7 +40,7 @@ vi.mock('@queries/agent-session/queue-sync', () => ({
subscribeSocketSessionStarted: socket.subscribeSocketSessionStarted,
}));
-import { AgentSession } from './AgentSession';
+import { AgentSession, AgentSessionReleased } from './AgentSession';
import { resetSessionTurns, sessionTurn } from './session-turn';
const SESSION = '01a0abed-279f-724c-9f49-60dbedc79b6e';
@@ -119,6 +119,21 @@ describe('AgentSession', () => {
live.release();
});
+ it('names a load abandoned by its last release, rather than failing it', async () => {
+ const log = deferred<LogResult>();
+ harness.getLog.mockReturnValue(log.promise);
+
+ const live = AgentSession.acquire(SESSION);
+ const loading = live.load();
+ // The surface goes away while the log is still on the wire - a list row
+ // scrolling out, or a route change - and nothing is left to render it.
+ live.release();
+ log.resolve(logOf([row(1)]));
+
+ await expect(loading).rejects.toBeInstanceOf(AgentSessionReleased);
+ expect(fold.pushSession).not.toHaveBeenCalled();
+ });
+
it('ignores rows for sessions nobody has open', () => {
AgentSession.ingest({ agentSessionId: 'other', entries: [row(1)] });
expect(fold.pushSession).not.toHaveBeenCalled();
diff --git a/apps/web/src/lib/core/agent-session/AgentSession.ts b/apps/web/src/lib/core/agent-session/AgentSession.ts
index 289da98647..d55c5e9460 100644
--- a/apps/web/src/lib/core/agent-session/AgentSession.ts
+++ b/apps/web/src/lib/core/agent-session/AgentSession.ts
@@ -34,6 +34,7 @@ import type {
SessionBot,
} from '@service-agent-harness/generated/schemas';
import { v7 as uuidv7 } from 'uuid';
+import { SessionLoadTrace, traceAcquire } from './load-telemetry';
import { publishSessionTurn } from './session-turn';
export type AgentSessionListener = (events: FoldedStreamEvent[]) => void;
@@ -48,6 +49,21 @@ export type IssueResult = Awaited<
ReturnType<typeof agentHarnessServiceClient.control>
>;
+/**
+ * The load was abandoned because every surface holding the session let go
+ * before it finished.
+ *
+ * Its own type because it is not a fault: a row that scrolls out of the list
+ * mid-fetch is ordinary, and a caller that cannot tell it apart from a failed
+ * fetch either reports a phantom error or hides a real one.
+ */
+export class AgentSessionReleased extends Error {
+ constructor(readonly sessionId: string) {
+ super(`agent session released: ${sessionId}`);
+ this.name = 'AgentSessionReleased';
+ }
+}
+
/**
* Whether this action takes the turn, rather than riding alongside one. ACP
* runs a prompt at a time, so these are the ones the harness queues; mirrors
@@ -66,9 +82,11 @@ export class AgentSession {
* machine and drops the subscriptions.
*/
static acquire(id: string): AgentSession {
- const session = AgentSession.open.get(id) ?? new AgentSession(id);
+ const open = AgentSession.open.get(id);
+ const session = open ?? new AgentSession(id);
AgentSession.open.set(id, session);
session.references += 1;
+ traceAcquire(id, open === undefined, session.references);
return session;
}
@@ -106,6 +124,8 @@ export class AgentSession {
private chain: Promise<void> = Promise.resolve();
private loading: Promise<AgentSessionRecord>;
private loadFailed = false;
+ /** The span for the attempt in flight, ended by whatever settles it. */
+ private trace: SessionLoadTrace;
/**
* The fold's turn state, tracked off the same events listeners see. What
* {@link issue} reads to know whether an action will reach the runtime or
@@ -126,6 +146,7 @@ export class AgentSession {
this.unsubscribeSocket = subscribeSocketSessionStarted(() => {
void this.resync();
});
+ this.trace = new SessionLoadTrace(id);
this.loading = this.startLoad();
}
@@ -137,6 +158,7 @@ export class AgentSession {
load(): Promise<AgentSessionRecord> {
if (this.loadFailed) {
this.loadFailed = false;
+ this.trace = new SessionLoadTrace(this.id);
this.loading = this.startLoad();
}
return this.loading;
@@ -287,16 +309,29 @@ export class AgentSession {
if (AgentSession.open.get(this.id) === this)
AgentSession.open.delete(this.id);
this.closed = true;
+ // Ended here rather than where the load notices: a fetch that never
+ // answers never reaches that check, and an unended span never reports.
+ this.trace.end('released');
this.listeners.clear();
this.unsubscribeSocket();
closeSession(this.id);
}
private startLoad(): Promise<AgentSessionRecord> {
- return this.fetchAndFold().catch((error: unknown) => {
- this.loadFailed = true;
- throw error;
- });
+ return this.fetchAndFold().then(
+ (record) => {
+ this.trace.end('loaded');
+ return record;
+ },
+ (error: unknown) => {
+ this.loadFailed = true;
+ this.trace.end(
+ error instanceof AgentSessionReleased ? 'released' : 'failed',
+ error
+ );
+ throw error;
+ }
+ );
}
private async fetchAndFold(): Promise<AgentSessionRecord> {
@@ -310,8 +345,10 @@ export class AgentSession {
if (log.isErr()) {
throw new Error(`agent session log could not be fetched: ${this.id}`);
}
- if (this.closed) throw new Error(`agent session released: ${this.id}`);
+ if (this.closed) throw new AgentSessionReleased(this.id);
+ this.trace.fetched(log.value.entries.length);
+ const foldStartedAt = performance.now();
await this.apply([{ kind: 'snapshot', rows: log.value.entries }]);
// Inputs can keep arriving while each push is in flight; drain until a
// check finds nothing, then flip ready so the next one goes straight in.
@@ -321,6 +358,7 @@ export class AgentSession {
await this.apply(inputs);
}
this.ready = true;
+ this.trace.folded(foldStartedAt);
this.setTurn((await readSession(this.id)).metadata.turn);
return { session: session.value, bot: log.value.bot };
}
diff --git a/apps/web/src/lib/core/agent-session/load-telemetry.ts b/apps/web/src/lib/core/agent-session/load-telemetry.ts
new file mode 100644
index 0000000000..e3d72f9d45
--- /dev/null
+++ b/apps/web/src/lib/core/agent-session/load-telemetry.ts
@@ -0,0 +1,143 @@
+/**
+ * Telemetry for opening a session: what a surface waited for before it could
+ * render, and why the wait ended.
+ *
+ * The backend traces a session thoroughly once a request reaches it. What it
+ * cannot see is the half that a person actually experiences — a load that
+ * never resolved, a surface that gave up and released mid-fetch, or the same
+ * log fetched over and over because rows kept acquiring and dropping the
+ * session. That gap is what this fills: one span per attempt to open a
+ * session, always ended, always carrying the outcome.
+ *
+ * Failures here are swallowed. Telemetry that can break a session load is
+ * worse than no telemetry.
+ */
+
+import type { Span } from '@macro-inc/observability';
+import { Telemetry } from '@macro-inc/observability';
+
+/** How an attempt to open a session ended. */
+export type LoadOutcome =
+ /** The log was fetched and folded; the surface can render. */
+ | 'loaded'
+ /** Every holder released before the load finished, so nobody wanted it. */
+ | 'released'
+ /** The fetch or the fold failed. */
+ | 'failed'
+ /** Still unsettled at {@link STALL_THRESHOLD_MS} — reported, not concluded. */
+ | 'stalled';
+
+/**
+ * How long a load may go unsettled before it is reported as stalled.
+ *
+ * A span reaches the exporter only when it ends, so a load that never
+ * settles would otherwise be the one case that produces no telemetry at all —
+ * precisely the case worth seeing. Past this the span is closed and reported;
+ * if the load resolves afterwards it is the {@link SessionLoadTrace.end}
+ * no-op, and the surface is none the wiser.
+ *
+ * Well past a cold load of the largest session observed (a few hundred
+ * milliseconds), so a slow session is never reported as a stuck one.
+ */
+const STALL_THRESHOLD_MS = 15_000;
+
+/**
+ * One attempt to open a session, from the first acquisition to whatever
+ * ended it.
+ *
+ * Held rather than run around an operation because the operation it measures
+ * can be abandoned: `release` ends the span from outside the load.
+ */
+export class SessionLoadTrace {
+ readonly #span: Span | undefined;
+ readonly #startedAt = performance.now();
+ #ended = false;
+ #stallTimer: ReturnType<typeof setTimeout> | undefined;
+
+ constructor(sessionId: string) {
+ this.#span = start('agent.session.load', sessionId);
+ this.#stallTimer = setTimeout(
+ () => this.end('stalled'),
+ STALL_THRESHOLD_MS
+ );
+ }
+
+ /** The log arrived; `rows` is what the fold is about to be given. */
+ fetched(rows: number): void {
+ this.#set('agent.session.log.rows', rows);
+ this.#set('agent.session.load.fetch_ms', this.#since());
+ }
+
+ /** The snapshot is folded and the surface can read the session. */
+ folded(foldStartedAt: number): void {
+ this.#set('agent.session.load.fold_ms', performance.now() - foldStartedAt);
+ }
+
+ /**
+ * End the span with how the attempt turned out. Safe to call more than
+ * once; the first outcome is the one recorded, so a release racing a
+ * failure does not rewrite it.
+ */
+ end(outcome: LoadOutcome, error?: unknown): void {
+ if (this.#ended) return;
+ this.#ended = true;
+ clearTimeout(this.#stallTimer);
+ this.#stallTimer = undefined;
+ this.#set('agent.session.load.outcome', outcome);
+ this.#set('agent.session.load.total_ms', this.#since());
+ try {
+ // A release is the expected end of an abandoned load, not a fault:
+ // recording it as an error would bury the ones worth looking at.
+ if (outcome === 'failed' && error !== undefined) this.#span?.error(error);
+ this.#span?.end();
+ } catch {
+ // See the module comment.
+ }
+ }
+
+ #since(): number {
+ return performance.now() - this.#startedAt;
+ }
+
+ #set(name: string, value: string | number): void {
+ try {
+ this.#span?.setAttr(name, value);
+ } catch {
+ // See the module comment.
+ }
+ }
+}
+
+/**
+ * Record that a surface took a reference to a session.
+ *
+ * `created` separates a shared instance being reused - free - from a new one,
+ * which refetches the whole log. A session id that keeps creating instances
+ * is a surface acquiring and releasing in a loop, which is invisible from the
+ * backend: it sees only an unexplained run of identical log fetches.
+ */
+export function traceAcquire(
+ sessionId: string,
+ created: boolean,
+ references: number
+): void {
+ const span = start('agent.session.acquire', sessionId);
+ if (!span) return;
+ try {
+ span.setAttr('agent.session.acquire.created', created);
+ span.setAttr('agent.session.acquire.references', references);
+ span.end();
+ } catch {
+ // See the module comment.
+ }
+}
+
+function start(name: string, sessionId: string): Span | undefined {
+ try {
+ const span = Telemetry.span(name);
+ span.setAttr('agent.session.id', sessionId);
+ return span;
+ } catch {
+ return undefined;
+ }
+}
diff --git a/apps/web/src/lib/core/agent-session/use-session-turn.test.tsx b/apps/web/src/lib/core/agent-session/use-session-turn.test.tsx
index 9d1925824a..da4705cdaf 100644
--- a/apps/web/src/lib/core/agent-session/use-session-turn.test.tsx
+++ b/apps/web/src/lib/core/agent-session/use-session-turn.test.tsx
@@ -39,7 +39,9 @@ describe('useSessionTurn', () => {
});
it('acquires a session that just produced a live frame', () => {
- const load = vi.fn();
+ // Resolves like the real one: the hook drops the load's rejection, so a
+ // mock that returns nothing would not exercise that path.
+ const load = vi.fn().mockResolvedValue(undefined);
const release = vi.fn();
session.get.mockReturnValue(undefined);
session.acquire.mockReturnValue({ load, release });
diff --git a/apps/web/src/lib/core/agent-session/use-session-turn.ts b/apps/web/src/lib/core/agent-session/use-session-turn.ts
index 328e846bd5..1c59c21a12 100644
--- a/apps/web/src/lib/core/agent-session/use-session-turn.ts
+++ b/apps/web/src/lib/core/agent-session/use-session-turn.ts
@@ -37,7 +37,10 @@ export function useSessionTurn(
const id = sessionId();
if (!follow()) return;
const session = AgentSession.acquire(id);
- void session.load();
+ // A row follows a session to show its turn and renders nothing from the
+ // load, so a load it abandons by unmounting is not this caller's problem.
+ // Left floating it surfaced as an unhandled rejection on every scroll.
+ void session.load().catch(() => undefined);
onCleanup(() => session.release());
});
diff --git a/crates/agent_session/src/inbound/axum_router.rs b/crates/agent_session/src/inbound/axum_router.rs
index 6427567e7a..bd00d472e1 100644
--- a/crates/agent_session/src/inbound/axum_router.rs
+++ b/crates/agent_session/src/inbound/axum_router.rs
@@ -1302,7 +1302,14 @@ pub struct AgentSessionLogResponse {
///
/// An unknown session is an error: the response has to name the session's
/// agent, and a session that never existed has none to name.
-#[tracing::instrument(skip_all, fields(session_id = %session_id), err(Debug))]
+#[tracing::instrument(
+ skip_all,
+ fields(
+ session_id = %session_id,
+ agent.session.log.rows = tracing::field::Empty,
+ ),
+ err(Debug)
+)]
pub async fn get_agent_session_log_handler<
T: AgentSessionService,
Access: EntityAccessService,
@@ -1317,6 +1324,12 @@ pub async fn get_agent_session_log_handler<
.session_log(AgentSessionId::new_from_uuid(session_id))
.await?;
+ // How much history this response carries. Without it the endpoint's
+ // latency cannot be read against the size of the session that produced
+ // it, and "is this slow because the session is huge" costs a trip to the
+ // database to answer.
+ tracing::Span::current().record("agent.session.log.rows", log.entries.len());
+
Ok(Json(AgentSessionLogResponse {
bot: log.bot,
entries: log.entries.into_iter().map(Into::into).collect(),
diff --git a/crates/cursor_cloud_agents/src/api.rs b/crates/cursor_cloud_agents/src/api.rs
index 523ad4732b..bbd74fd791 100644
--- a/crates/cursor_cloud_agents/src/api.rs
+++ b/crates/cursor_cloud_agents/src/api.rs
@@ -494,7 +494,14 @@ impl CursorClient {
}
impl CursorAgents for CursorClient {
- #[tracing::instrument(skip(self), err)]
+ #[tracing::instrument(
+ skip(self),
+ err,
+ fields(
+ cursor.poll.status = tracing::field::Empty,
+ cursor.poll.rate_limited = tracing::field::Empty,
+ )
+ )]
async fn raw_result(
&self,
agent: &CursorAgentId,
@@ -508,8 +515,33 @@ impl CursorAgents for CursorClient {
.await
.map_err(|e| rootcause::report!(e))?;
let status = response.status();
+ let retry_after = response
+ .headers()
+ .get(reqwest::header::RETRY_AFTER)
+ .and_then(|value| value.to_str().ok())
+ .map(str::to_owned);
let text = response.text().await.map_err(|e| rootcause::report!(e))?;
+ let span = tracing::Span::current();
+ span.record("cursor.poll.status", status.as_u16());
+ span.record(
+ "cursor.poll.rate_limited",
+ status == reqwest::StatusCode::TOO_MANY_REQUESTS,
+ );
if !status.is_success() {
+ // The rate limit is per API key and shared by every session this
+ // deployment drives, so one session's 429 is a fact about the
+ // fleet, not about this run. Logged in its own right: until now
+ // it existed only inside an error chain that reached the reader
+ // and nothing else, so "are we over the limit" was unanswerable
+ // from telemetry.
+ if status == reqwest::StatusCode::TOO_MANY_REQUESTS {
+ tracing::warn!(
+ %agent,
+ %run,
+ retry_after = retry_after.as_deref().unwrap_or("unset"),
+ "Cursor run poll was rate limited"
+ );
+ }
return Err(rootcause::report!(
"Cursor run poll failed: {status}: {text}"
));