Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

fix(capture): page the engine history so a chat past 8 MiB keeps its transcript, turn snapshot and archive delta

ogsamrat0fa2c9a85c53
diff --git a/apps/server/src/collector-observer.ts b/apps/server/src/collector-observer.ts
index 2f51ddc7..179f4bc1 100644
--- a/apps/server/src/collector-observer.ts
+++ b/apps/server/src/collector-observer.ts
@@ -89,6 +89,50 @@ function turnModelFromMessages(messages: unknown): CollectorSessionModel | null
return null;
}
+/** Messages asked of the engine per page, as many as opencode's own message stream reads at a time. */
+const MESSAGE_PAGE_SIZE = 50;
+/** One engine response (a page of messages, a session record, a status map) is parsed only up to this size. */
+const MAX_ENGINE_READ_BYTES = 8 * 1024 * 1024;
+/** The messages of one turn kept whole for its trace, the newest first. */
+const MAX_TURN_MESSAGE_BYTES = 8 * 1024 * 1024;
+
+type EngineFetch = (path: string, query?: Record<string, string>) => Promise<Response>;
+
+/** Requests to the engine a collected request went to, with its headers and query plus `query`, each under a fresh `signal()`. */
+function engineFetch(target: EngineTarget, signal: () => AbortSignal): EngineFetch {
+ return (path, query = {}) => {
+ const url = new URL(buildOpencodeProxyUrl(target.baseUrl, path, target.search));
+ for (const [key, value] of Object.entries(query)) url.searchParams.set(key, value);
+ return loopbackFetch(url.toString(), { headers: new Headers(target.headers), signal: signal() });
+ };
+}
+
+/** An engine response over its read cap, with the bytes read before it was cut off. */
+class OversizedEngineResponse extends Error {
+ readonly chunks: Uint8Array[];
+
+ constructor(chunks: Uint8Array[]) {
+ super("trace response exceeded local limit");
+ this.chunks = chunks;
+ }
+}
+
+/** The engine answered a read with an error status. */
+class EngineReadError extends Error {
+ readonly status: number;
+
+ constructor(status: number) {
+ super(`the engine answered ${status}`);
+ this.status = status;
+ }
+}
+
+/** Null for an error status, as a missing record reads; anything else is thrown on. */
+function nullOnErrorStatus(error: unknown): null {
+ if (error instanceof EngineReadError) return null;
+ throw error;
+}
+
/**
* Walks the task-tool subagent tree below a settled root session: one
* "session.child" event per child carrying only the messages that are new
@@ -99,30 +143,33 @@ async function captureChildSessions(input: {
rootSessionId: string;
parentSessionId: string;
depth: number;
- fetchJson: (path: string, maxBytes: number) => Promise<unknown>;
+ fetchEngine: EngineFetch;
checkpoints: Record<string, string>;
known: Set<string>;
visited: Set<string>;
}): Promise<void> {
if (input.depth > MAX_COLLECTOR_CHILD_SESSION_DEPTH) return;
- const children = await input.fetchJson(`/session/${encodeURIComponent(input.parentSessionId)}/children`, 1024 * 1024);
+ const children = await readEngineJson(input.fetchEngine, `/session/${encodeURIComponent(input.parentSessionId)}/children`, 1024 * 1024)
+ .catch(nullOnErrorStatus);
if (!Array.isArray(children)) return;
for (const child of children.slice(0, 200)) {
const childId = isRecord(child) && typeof child.id === "string" && child.id ? child.id : null;
if (!childId || childId === input.rootSessionId || input.visited.has(childId)) continue;
input.visited.add(childId);
- const messages = await input.fetchJson(`/session/${encodeURIComponent(childId)}/message`, 8 * 1024 * 1024);
- const list = Array.isArray(messages) ? messages : [];
- const delta = newTraceMessages(list, input.checkpoints[childId]);
- const newMessages = Array.isArray(delta) ? delta : [];
+ const history = await readEngineHistory(engineMessages(input.fetchEngine, `/session/${encodeURIComponent(childId)}/message`, true), input.checkpoints[childId])
+ .catch(nullOnErrorStatus);
+ const newMessages = history?.delta ?? [];
+ if (history && history.omitted > 0) {
+ input.collector.recordTrace(input.rootSessionId, "session.messages_omitted", { child_session_id: childId, count: history.omitted });
+ }
if (newMessages.length > 0 || !input.known.has(childId)) {
input.collector.recordChildSession(input.rootSessionId, {
childSessionId: childId,
parentSessionId: input.parentSessionId,
title: isRecord(child) ? optionalTraceString(child.title) : null,
- agent: turnModelFromMessages(list)?.agent ?? null,
+ agent: turnModelFromMessages(history?.outline ?? [])?.agent ?? null,
messages: newMessages,
- lastMessageId: traceMessageId(list.at(-1)),
+ lastMessageId: traceMessageId(history?.outline.at(-1)),
});
}
await captureChildSessions({ ...input, parentSessionId: childId, depth: input.depth + 1 });
@@ -144,7 +191,7 @@ function collectorDelay(ms: number, signal: AbortSignal): Promise<void> {
});
}
-async function readCollectorResponse(response: Response, maxBytes = 8 * 1024 * 1024): Promise<unknown> {
+async function readCollectorResponse(response: Response, maxBytes = MAX_ENGINE_READ_BYTES): Promise<unknown> {
const reader = response.body?.getReader();
if (!reader) return null;
const chunks: Uint8Array[] = [];
@@ -153,9 +200,12 @@ async function readCollectorResponse(response: Response, maxBytes = 8 * 1024 * 1
while (true) {
const { done, value } = await reader.read();
if (done) break;
- total += value.byteLength;
- if (total > maxBytes) throw new Error("trace response exceeded local limit");
chunks.push(value);
+ total += value.byteLength;
+ if (total > maxBytes) {
+ await reader.cancel().catch(() => undefined);
+ throw new OversizedEngineResponse(chunks);
+ }
}
} finally {
reader.releaseLock();
@@ -169,14 +219,126 @@ async function readCollectorResponse(response: Response, maxBytes = 8 * 1024 * 1
return JSON.parse(new TextDecoder().decode(bytes));
}
-function traceHasTerminalAssistant(messages: unknown): boolean {
- if (!Array.isArray(messages)) return false;
- for (let index = messages.length - 1; index >= 0; index -= 1) {
- const message = messages[index];
+/** One engine read under the cap; an error status throws EngineReadError. */
+async function readEngineJson(fetchEngine: EngineFetch, path: string, maxBytes?: number): Promise<unknown> {
+ const response = await fetchEngine(path);
+ if (!response.ok) {
+ await response.body?.cancel().catch(() => undefined);
+ throw new EngineReadError(response.status);
+ }
+ return readCollectorResponse(response, maxBytes);
+}
+
+/**
+ * The info of a message too large to read whole, from the start of its
+ * one-message page (`[{"info":{...},"parts":[...]}]`): its id, role and
+ * whether it ended are all there. Null when the info does not end within
+ * the bytes read.
+ */
+function leadingMessageInfo(chunks: Uint8Array[]): Record<string, unknown> | null {
+ const text = Buffer.concat(chunks).toString("utf8");
+ const start = /^\s*\[\s*\{\s*"info"\s*:\s*(?=\{)/.exec(text)?.[0].length;
+ if (start === undefined) return null;
+ let depth = 0;
+ let inString = false;
+ for (let index = start; index < text.length; index += 1) {
+ const char = text[index];
+ if (inString) {
+ if (char === "\\") index += 1;
+ else if (char === "\"") inString = false;
+ } else if (char === "\"") {
+ inString = true;
+ } else if (char === "{" || char === "[") {
+ depth += 1;
+ } else if ((char === "}" || char === "]") && --depth === 0) {
+ try {
+ const info: unknown = JSON.parse(text.slice(start, index + 1));
+ return isRecord(info) ? info : null;
+ } catch {
+ return null;
+ }
+ }
+ }
+ return null;
+}
+
+/** A message read from the engine; `whole` is false for one known only by its info (too large to read). */
+type EngineMessage = { message: unknown; whole: boolean };
+
+/**
+ * A session's messages, newest first. A v1 engine is read a page at a
+ * time, each page parsed under the read cap: after a page over it, the
+ * newest message is asked for alone and later pages are at most half that
+ * size, and a message over the cap on its own is known by the info that
+ * leads it. The v2 daemon has no pages; its list is read whole under the
+ * cap. Throws when the engine cannot be read.
+ */
+async function* engineMessages(fetchEngine: EngineFetch, path: string, paged: boolean): AsyncGenerator<EngineMessage> {
+ if (!paged) {
+ const payload = await readEngineJson(fetchEngine, path);
+ const list = isRecord(payload) && "data" in payload ? payload.data : payload;
+ if (!Array.isArray(list)) throw new Error("the engine messages could not be read");
+ for (const message of list.reverse()) yield { message, whole: true };
+ return;
+ }
+ let ceiling = MESSAGE_PAGE_SIZE;
+ let limit = MESSAGE_PAGE_SIZE;
+ let before: string | null = null;
+ while (true) {
+ const response = await fetchEngine(path, { limit: String(limit), ...(before ? { before } : {}) });
+ if (!response.ok) {
+ await response.body?.cancel().catch(() => undefined);
+ throw new EngineReadError(response.status);
+ }
+ const cursor = response.headers.get("x-next-cursor");
+ let page: EngineMessage[];
+ try {
+ const items = await readCollectorResponse(response);
+ if (!Array.isArray(items)) throw new Error("the engine messages could not be read");
+ page = items.map((message) => ({ message, whole: true }));
+ } catch (error) {
+ if (!(error instanceof OversizedEngineResponse)) throw error;
+ if (limit > 1) {
+ ceiling = Math.max(1, limit >> 1);
+ limit = 1;
+ continue;
+ }
+ page = [{ message: { info: leadingMessageInfo(error.chunks) ?? {}, parts: [] }, whole: false }];
+ }
+ for (const entry of page.reverse()) yield entry;
+ if (!cursor || cursor === before) return;
+ before = cursor;
+ limit = Math.min(ceiling, limit * 2);
+ }
+}
+
+/**
+ * What turn counting, the model and the checkpoint read of a message: its
+ * info, less a user message's diff summary (whole-file patches), and its
+ * finish parts. Other shapes (v2 context entries) are kept as they are.
+ */
+function messageOutline(message: unknown): unknown {
+ if (!isRecord(message) || !isRecord(message.info)) return message;
+ const { summary, ...info } = message.info;
+ const parts = Array.isArray(message.parts) ? message.parts : [];
+ return {
+ info: isRecord(summary) ? info : message.info,
+ parts: parts.filter((part) => isRecord(part) && ["step-finish", "finish", "error"].includes(String(part.type))),
+ };
+}
+
+async function messageOutlines(messages: AsyncIterable<EngineMessage>): Promise<unknown[]> {
+ const outline: unknown[] = [];
+ for await (const { message } of messages) outline.push(messageOutline(message));
+ return outline.reverse();
+}
+
+/** Whether the newest assistant message ended (read from the newest page on). */
+async function newestAssistantFinished(messages: AsyncIterable<EngineMessage>): Promise<boolean> {
+ for await (const { message } of messages) {
if (!isRecord(message)) continue;
const info = isRecord(message.info) ? message.info : message;
- if ((info.role ?? info.type) !== "assistant") continue;
- return isFinishedAssistantMessage(message);
+ if ((info.role ?? info.type) === "assistant") return isFinishedAssistantMessage(message);
}
return false;
}
@@ -187,10 +349,43 @@ function traceMessageId(message: unknown): string | null {
return typeof info.id === "string" && info.id ? info.id : null;
}
-function newTraceMessages(messages: unknown, previousId: string | undefined): unknown {
- if (!Array.isArray(messages) || !previousId) return messages;
- const index = messages.findIndex((message) => traceMessageId(message) === previousId);
- return index >= 0 ? messages.slice(index + 1) : messages;
+/** A session's messages as one settled turn reads them. */
+type EngineHistory = {
+ /** Every message in outline, oldest first. */
+ outline: unknown[];
+ /** The messages after the checkpoint, oldest first and whole: the newest of them, up to MAX_TURN_MESSAGE_BYTES. */
+ delta: unknown[];
+ /** Messages after the checkpoint left out of `delta`: too large to read, or past the turn's budget. */
+ omitted: number;
+};
+
+/**
+ * Reads the whole history a page at a time: the messages after the
+ * checkpoint message (all of them without one) whole, every message in
+ * outline, so a history of any length keeps its transcript and turn count.
+ */
+async function readEngineHistory(messages: AsyncIterable<EngineMessage>, checkpoint: string | undefined): Promise<EngineHistory> {
+ const outline: unknown[] = [];
+ const delta: unknown[] = [];
+ let deltaBytes = 0;
+ let budgetSpent = false;
+ let omitted = 0;
+ let afterCheckpoint = true;
+ for await (const { message, whole } of messages) {
+ if (checkpoint && traceMessageId(message) === checkpoint) afterCheckpoint = false;
+ outline.push(messageOutline(message));
+ if (!afterCheckpoint) continue;
+ const bytes = whole && !budgetSpent ? Buffer.byteLength(JSON.stringify(message)) : 0;
+ if (whole && !budgetSpent && deltaBytes + bytes <= MAX_TURN_MESSAGE_BYTES) {
+ delta.push(message);
+ deltaBytes += bytes;
+ continue;
+ }
+ // Newest first: once the budget is spent, every older message is left out too.
+ if (whole) budgetSpent = true;
+ omitted += 1;
+ }
+ return { outline: outline.reverse(), delta: delta.reverse(), omitted };
}
/**
@@ -200,23 +395,15 @@ function newTraceMessages(messages: unknown, previousId: string | undefined): un
*/
export function projectArchiveEngineReads(target: EngineTarget, sessionId: string): ArchiveEngineReads {
const v2 = target.engine === "v2";
- const read = async (path: string, maxBytes: number): Promise<unknown> => {
- const response = await loopbackFetch(buildOpencodeProxyUrl(target.baseUrl, path, target.search), {
- headers: new Headers(target.headers),
- signal: AbortSignal.timeout(20_000),
- });
- if (!response.ok) {
- await response.body?.cancel().catch(() => undefined);
- return null;
- }
- const payload = await readCollectorResponse(response, maxBytes);
- return v2 && isRecord(payload) && "data" in payload ? payload.data : payload;
- };
+ const fetchEngine = engineFetch(target, () => AbortSignal.timeout(20_000));
const session = `${v2 ? "/api/session" : "/session"}/${encodeURIComponent(sessionId)}`;
return {
- session: () => read(session, 1024 * 1024),
- // The same message list the observer counts turns from at turn end.
- messages: () => read(v2 ? `${session}/context` : `${session}/message`, 8 * 1024 * 1024),
+ session: async () => {
+ const payload = await readEngineJson(fetchEngine, session, 1024 * 1024).catch(nullOnErrorStatus);
+ return v2 && isRecord(payload) && "data" in payload ? payload.data : payload;
+ },
+ // The same messages the observer counts turns from at turn end, in outline.
+ messages: () => messageOutlines(engineMessages(fetchEngine, v2 ? `${session}/context` : `${session}/message`, !v2)),
};
}
@@ -224,6 +411,8 @@ export function projectArchiveEngineReads(target: EngineTarget, sessionId: strin
* Follows a collected session until its turn settles, then records the
* turn's model, subagents and messages, takes the turn's change snapshot,
* flushes the trace and hands the engine's messages to the project archive.
+ * The snapshot, the trace and the archive delta never depend on reading
+ * the messages: a turn whose transcript cannot be read still gets them.
*/
export function observeCollectedSession(input: {
collector: WorkspaceCollector;
@@ -240,12 +429,14 @@ export function observeCollectedSession(input: {
const v2 = input.target.engine === "v2";
const headers = new Headers(input.target.headers);
const statusUrl = buildOpencodeProxyUrl(baseUrl, v2 ? "/api/session/active" : "/session/status", search);
- const messagesUrl = buildOpencodeProxyUrl(
- baseUrl,
- v2 ? `/api/session/${encodeURIComponent(input.sessionId)}/context` : `/session/${encodeURIComponent(input.sessionId)}/message`,
- search,
- );
+ const fetchEngine = engineFetch(input.target, () => AbortSignal.any([observer.controller.signal, AbortSignal.timeout(20_000)]));
+ const messagesPath = v2 ? `/api/session/${encodeURIComponent(input.sessionId)}/context` : `/session/${encodeURIComponent(input.sessionId)}/message`;
+ const messages = () => engineMessages(fetchEngine, messagesPath, !v2);
const enginePayload = (payload: unknown) => (v2 && isRecord(payload) && "data" in payload ? payload.data : payload);
+ const falseUnlessStopped = (error: unknown): false => {
+ if (observer.controller.signal.aborted) throw error;
+ return false;
+ };
void (async () => {
let observedBusy = false;
let consecutiveSettled = 0;
@@ -270,26 +461,31 @@ export function observeCollectedSession(input: {
consecutiveSettled = 0;
continue;
}
- const messagesResponse = await loopbackFetch(messagesUrl, {
- headers,
- signal: AbortSignal.any([observer.controller.signal, AbortSignal.timeout(20_000)]),
- });
- const messages = messagesResponse.ok
- ? enginePayload(await readCollectorResponse(messagesResponse))
- : { status: messagesResponse.status, unavailable: true };
- const terminal = traceHasTerminalAssistant(messages);
consecutiveSettled += 1;
- if ((!observedBusy && !terminal) || consecutiveSettled < 2 || attempt < 3) continue;
- const delta = newTraceMessages(messages, observer.lastMessageIds.get(input.sessionId));
- if (Array.isArray(messages)) {
- const lastId = traceMessageId(messages.at(-1));
+ if (consecutiveSettled < 2 || attempt < 3) continue;
+ // Never seen busy: the turn settled only once its answer ended (the prompt may not have started yet).
+ if (!observedBusy && !(await newestAssistantFinished(messages()).catch(falseUnlessStopped))) continue;
+ let history: EngineHistory | null = null;
+ let unavailable: Record<string, unknown> = { unavailable: true };
+ try {
+ history = await readEngineHistory(messages(), observer.lastMessageIds.get(input.sessionId));
+ } catch (error) {
+ if (observer.controller.signal.aborted) throw error;
+ if (error instanceof EngineReadError) unavailable = { status: error.status, unavailable: true };
+ input.collector.recordTrace(input.sessionId, "session.messages_failed", {
+ error: error instanceof Error ? error.message : "unknown",
+ });
+ }
+ if (history) {
+ const lastId = traceMessageId(history.outline.at(-1));
if (lastId) {
observer.lastMessageIds.set(input.sessionId, lastId);
void input.collector.setSessionCheckpoint(input.sessionId, lastId);
}
+ if (history.omitted > 0) input.collector.recordTrace(input.sessionId, "session.messages_omitted", { count: history.omitted });
+ const model = turnModelFromMessages(history.delta);
+ if (model) input.collector.recordSessionModel(input.sessionId, model);
}
- const model = turnModelFromMessages(delta);
- if (model) input.collector.recordSessionModel(input.sessionId, model);
// The v2 daemon has no subagent children route; only its root turn is captured.
if (!v2) {
try {
@@ -298,17 +494,7 @@ export function observeCollectedSession(input: {
rootSessionId: input.sessionId,
parentSessionId: input.sessionId,
depth: 1,
- fetchJson: async (path, maxBytes) => {
- const response = await loopbackFetch(buildOpencodeProxyUrl(baseUrl, path, search), {
- headers,
- signal: AbortSignal.any([observer.controller.signal, AbortSignal.timeout(20_000)]),
- });
- if (!response.ok) {
- await response.body?.cancel().catch(() => undefined);
- return null;
- }
- return readCollectorResponse(response, maxBytes);
- },
+ fetchEngine,
checkpoints: await input.collector.childCheckpoints(input.sessionId),
known: new Set(await input.collector.childSessionIds(input.sessionId)),
visited: new Set([input.sessionId]),
@@ -324,9 +510,11 @@ export function observeCollectedSession(input: {
// The turn snapshot runs first so the artifacts it discovers are part of
// the trace flushed right behind it.
input.collector.captureSnapshot(input.sessionId, "turn_completed");
- input.collector.flushTrace(input.sessionId, { messages: delta });
- // The delta's turn number is the engine's completed-turn count, which survives app restarts.
- input.archive.turnCompleted(input.sessionId, messages);
+ input.collector.flushTrace(input.sessionId, { messages: history ? history.delta : unavailable });
+ // The delta's turn number is the engine's completed-turn count, which
+ // survives app restarts; without the messages the archiver numbers it
+ // right after the last archived turn.
+ input.archive.turnCompleted(input.sessionId, history ? history.outline : null);
return;
}
input.collector.recordTrace(input.sessionId, "session.observer_timeout");
diff --git a/apps/server/src/session-archive/README.md b/apps/server/src/session-archive/README.md
index b3c632be..b3a3e34b 100644
--- a/apps/server/src/session-archive/README.md
+++ b/apps/server/src/session-archive/README.md
@@ -107,7 +107,7 @@ in `captureServicesByServer` and calls:
| --- | --- |
| `startServer` (`startCaptureService`) | `start()`: a drain when enabled; otherwise `signOut()` clears what a previous run left (a no-op, touching nothing, when there is no `omnirush-archive/`) |
| A prompt dispatch on a local workspace (v1 `prompt_async`/`prompt`/`command`, v2 `prompt`/`prompt_async`/`command`/`generate`), after the collector's `startSession` | `sessionStarted({sessionId, root: workspace.path, engine})`, the engine reads built on the worker from the request's engine target |
-| The collector observer's `turn_completed` (on the worker), with the engine messages it just read | `turnCompleted(sessionId, messages)` |
+| The collector observer's `turn_completed` (on the worker), with the engine messages it just read (null when it could not read them) | `turnCompleted(sessionId, messages)` |
| `collector.finishSession` (session deleted) | `sessionEnded(sessionId)`: forgets the session, kicks a drain |
| The broker's `invalidate` hook (revoked or expired account), before `clearSpool()` | `signOut()` |
| Server shutdown and a failed start | `stop()`, started first: the main thread aborts the part uploads it is making for the worker at once, before the task-recovery checkpoint (up to 10 s), then the worker stops the archiver and the collector; the worker is terminated after 20 s at the latest. A user sign-out reaches the server this way too: the desktop clears the account, then restarts the server |
@@ -117,7 +117,7 @@ What the lifecycle adds on top of the calls below:
- **Every call returns at once.** Engine reads and captures run in the background, one at a time (`concurrency`, default 1), and each session's steps run in call order, so a turn's delta never overtakes the session's base. A thrown error or a rejected read costs one `warn` line and never reaches the request.
- **A base waits for a quiet moment.** The server passes `baseIdleMs` 2 s and `baseMaxDeferMs` 10 s (`PROJECT_ARCHIVE_BASE_IDLE_MS`, `PROJECT_ARCHIVE_BASE_MAX_DEFER_MS`): a base is packed once no prompt has been dispatched on any session for 2 s, and at the latest 10 s after its own prompt. Chats opened and prompted one after another are packed once the burst is over, one at a time; a lone chat's base still shows the folder as its first turn began. The wait holds no concurrency slot, and `stop()` or `signOut()` ends it without packing. Each session still gets its own base (sequence 0 of its own chain, section 7 of the backend spec); a second chat on the same folder packs from the root's persisted hash cache, so pass 1 of an unchanged tree reads no file (pass 2 streams and hashes the bytes it packs, as section 5.9 requires).
- **Root sessions only.** The start step reads the engine's session record (v1 `/session/:id`, v2 `/api/session/:id`); one with a `parentID` is a child and is never archived. An unreadable record, or unreadable messages, leave the session unresolved: its next prompt or its next completed turn reads the engine again. After an app restart the engine can take a minute to answer, longer than the 20 s read timeout, while the first prompt's turn still runs; at the turn's end the observer's messages give the count, the base is captured with it (a no-op, `exists`, when an earlier run captured it) and the turn's delta follows, so that turn is not lost.
-- **Turn numbers from the engine.** `completedTurnCount(messages)` counts prompts answered by an assistant message that ended (a completion time, a finish reason, an error or a finish part; several steps of one turn count once), from v1 `/session/:id/message` or the v2 `/api/session/:id/context` list. The base gets the count read when its step runs; a delta gets the count the observer read when the turn settled. The count survives restarts. A reverted or compacted history can lower it; the archiver then skips those deltas as `stale_turn` and the changes go into the first delta whose count is higher again.
+- **Turn numbers from the engine.** `completedTurnCount(messages)` counts prompts answered by an assistant message that ended (a completion time, a finish reason, an error or a finish part; several steps of one turn count once), from v1 `/session/:id/message` or the v2 `/api/session/:id/context` list. The v1 list is read a page at a time (`?limit=&before=`, each page under the 8 MiB read cap) and kept only in outline (each message's info and finish parts), so a history of any length is counted; a message over the cap on its own is counted from the info that leads it. The base gets the count read when its step runs; a delta gets the count the observer read when the turn settled. When the observer could not read the messages at all, the delta still runs with no count (`captureDelta(sessionId, root, null)`), numbered right after the last archived turn. The count survives restarts. A reverted or compacted history can lower it; the archiver then skips those deltas as `stale_turn` and the changes go into the first delta whose count is higher again.
- **Real path.** The root is `realpath`ed before `captureBase`, so the gate sees the real folder (a symlinked workspace is archived as its target; the gate still refuses a symlinked root it is handed).
- **Consent off.** A `disabled` capture result, or a drain that reports `disabled`, turns the lifecycle off for 10 minutes (`consentRecheckMs`): no capture, key check or drain. After that, the next prompt on a session without a base checks `/archives/key` again. No log line: the archiver logs the one `info`.
- **Signed out stays out.** After `signOut()` or `stop()`, queued steps do not run and nothing new is scheduled. `SessionArchiver.stop()` also bumps the generation, so a capture still packing at shutdown never commits (the server that replaces this one after a sign-out must not find it queued).
@@ -312,10 +312,10 @@ cd apps/server && bun --conditions=development test src/session-archive
- replay, chain conflicts and an unknown kid;
- backoff;
- the broker request hook.
-- `lifecycle.test.ts` (fake archiver and engine, plus one run on the real archiver): the turn count on v1 and v2 message shapes; the feature flag; one base per root session per app run at the real path; child sessions ignored; deltas numbered from the engine and ordered after the base; restart resume; a session whose start could not read the engine resolved at its next completed turn (its delta kept, also on the real archiver across an app restart), and a child resolved that way still ignored; signed-out start clearing; sign-out stopping queued steps; 428 off until the recheck; failures contained.
+- `lifecycle.test.ts` (fake archiver and engine, plus one run on the real archiver): the turn count on v1 and v2 message shapes; the feature flag; one base per root session per app run at the real path; child sessions ignored; deltas numbered from the engine and ordered after the base, and after the last archived turn when the messages could not be read; restart resume; a session whose start could not read the engine resolved at its next completed turn (its delta kept, also on the real archiver across an app restart), and a child resolved that way still ignored; signed-out start clearing; sign-out stopping queued steps; 428 off until the recheck; failures contained.
- `../project-archive.test.ts`: archives go through the broker's `archiveRequest` into `<state dir>/omnirush-archive/`; `OMNIRUSH_ARCHIVE_ENABLED=0` and a signed-out start clear leftovers without any network; a sign-out aborts a slow part PUT within a second and aborts the upload through the broker.
- `../omnirush-gateway-broker.test.ts`: `archiveRequest` URL, bearer, 401 refresh and path allowlist.
-- `../workspace-collector.server.e2e.test.ts` ("project archive wiring"): through the real proxy and observer, one base then a delta numbered 2 after a changed turn; 428 checked once with the chat unaffected; the flag off sends nothing.
+- `../workspace-collector.server.e2e.test.ts` ("project archive wiring"): through the real proxy and observer, one base then a delta numbered 2 after a changed turn; a history past the 8 MiB read cap (one message over it alone) still gets its base, its turns' transcripts and deltas, and a turn whose messages cannot be read still gets its snapshot and delta; 428 checked once with the chat unaffected; the flag off sends nothing.
- `index.test.ts`:
- the whole folder: `.git/` and gitignored `node_modules/` and `dist/` present, `.env` and `id_rsa` absent unless `archiveIncludeCredentialFiles`; the base then a delta, each decrypted and checked;
- no delta when nothing changed, `stale_turn` and `exists`;
diff --git a/apps/server/src/session-archive/index.ts b/apps/server/src/session-archive/index.ts
index 37e31d44..88986e08 100644
--- a/apps/server/src/session-archive/index.ts
+++ b/apps/server/src/session-archive/index.ts
@@ -272,21 +272,24 @@ export class SessionArchiver {
* Delta after a completed turn. No-op without a base, when the session
* stopped archiving, or when nothing in the folder changed. A session whose
* base could not be captured yet (key unavailable) gets its base instead.
+ * A null turn (the engine's messages could not be read) follows the last
+ * archived turn.
*/
- captureDelta(sessionId: string, root: string, turn: number): Promise<CaptureResult> {
- return this.guard("delta", sessionId, turn, async () => this.withSession(sessionId, async () => {
+ captureDelta(sessionId: string, root: string, turn: number | null): Promise<CaptureResult> {
+ return this.guard("delta", sessionId, turn ?? 0, async () => this.withSession(sessionId, async () => {
const generation = this.generation;
if (this.disabled) return { status: "skipped", reason: "disabled" };
const state = await this.loadSession(sessionId);
if (!state) return { status: "skipped", reason: "no_base" };
if (state.stopped || this.stoppedSessions.has(sessionId)) return { status: "skipped", reason: "stopped" };
if (resolve(root) !== state.root) this.log("warn", "OmniRush archive delta root differs from the session root; using the session root", { sessionId });
- if (state.next_sequence === 0) return this.captureBaseLocked(sessionId, state.root, turn, generation);
- if (state.last_turn !== null && turn <= state.last_turn) return { status: "skipped", reason: "stale_turn" };
+ if (state.next_sequence === 0) return this.captureBaseLocked(sessionId, state.root, turn ?? 0, generation);
+ const next = turn ?? (state.last_turn ?? 0) + 1;
+ if (state.last_turn !== null && next <= state.last_turn) return { status: "skipped", reason: "stale_turn" };
const key = await this.currentKey();
if (key === "disabled") return { status: "skipped", reason: "disabled" };
if (key === "unavailable") return { status: "skipped", reason: "unavailable" };
- return this.captureArchive(state, "delta", turn, key, generation);
+ return this.captureArchive(state, "delta", next, key, generation);
}));
}
diff --git a/apps/server/src/session-archive/lifecycle.test.ts b/apps/server/src/session-archive/lifecycle.test.ts
index 598a3cbd..ec51e596 100644
--- a/apps/server/src/session-archive/lifecycle.test.ts
+++ b/apps/server/src/session-archive/lifecycle.test.ts
@@ -27,7 +27,7 @@ const drained: DrainResult = { uploaded: 0, pending: 0, dropped: 0, blocked: nul
class FakeArchiver implements ProjectArchiver {
readonly calls: string[] = [];
base: (sessionId: string, turn: number) => Promise<CaptureResult> = async () => queued("base");
- delta: (sessionId: string, turn: number) => Promise<CaptureResult> = async (_sessionId, turn) => queued("delta", turn);
+ delta: (sessionId: string, turn: number | null) => Promise<CaptureResult> = async (_sessionId, turn) => queued("delta", turn ?? 0);
drainResult: () => Promise<DrainResult> = async () => drained;
async captureBase(sessionId: string, root: string, turn = 0): Promise<CaptureResult> {
@@ -35,7 +35,7 @@ class FakeArchiver implements ProjectArchiver {
return this.base(sessionId, turn);
}
- async captureDelta(sessionId: string, root: string, turn: number): Promise<CaptureResult> {
+ async captureDelta(sessionId: string, root: string, turn: number | null): Promise<CaptureResult> {
this.calls.push(`delta ${sessionId} ${root} ${turn}`);
return this.delta(sessionId, turn);
}
@@ -200,8 +200,8 @@ describe("ProjectArchiveLifecycle", () => {
// The turn ends while the base is still packing: its delta waits for the base.
subject.turnCompleted("ses_turns_0001", messages(1));
subject.turnCompleted("ses_turns_0001", messages(2));
- // Unreadable messages: no delta (the next one carries the changes).
- subject.turnCompleted("ses_turns_0001", { status: 503, unavailable: true });
+ // Unreadable messages still get their delta, with no turn: the archiver numbers it after the last one.
+ subject.turnCompleted("ses_turns_0001", null);
// Unknown sessions (never started in this app run) are left alone.
subject.turnCompleted("ses_unknown_0001", messages(4));
baseGate.resolve(queued("base"));
@@ -211,6 +211,7 @@ describe("ProjectArchiveLifecycle", () => {
`base ses_turns_0001 ${root} 0`,
`delta ses_turns_0001 ${root} 1`,
`delta ses_turns_0001 ${root} 2`,
+ `delta ses_turns_0001 ${root} null`,
]);
});
@@ -481,11 +482,17 @@ describe("ProjectArchiveLifecycle", () => {
second.subject.turnCompleted("ses_restarted_01", messages(2));
await second.subject.settled();
+ // A turn whose messages the observer could not read follows the last archived turn.
+ await writeFile(join(root, "after.txt"), "unread turn\n");
+ second.subject.turnCompleted("ses_restarted_01", null);
+ await second.subject.settled();
+
const archives = server.objects();
- expect(archives.map((archive) => [archive.request.kind, archive.request.sequence, archive.request.turn])).toEqual([["base", 0, 0], ["delta", 1, 1], ["delta", 2, 2]]);
+ expect(archives.map((archive) => [archive.request.kind, archive.request.sequence, archive.request.turn])).toEqual([["base", 0, 0], ["delta", 1, 1], ["delta", 2, 2], ["delta", 3, 3]]);
const latest = await openArchive(archives[2]!.object!);
expect(latest.find((member) => member.name === "after.txt")!.content.toString()).toBe("after restart 2\n");
expect(manifestOf(latest)).toMatchObject({ kind: "delta", turn: 2 });
+ expect(manifestOf(await openArchive(archives[3]!.object!))).toMatchObject({ kind: "delta", turn: 3 });
});
});
diff --git a/apps/server/src/session-archive/lifecycle.ts b/apps/server/src/session-archive/lifecycle.ts
index bd994a96..89409a7b 100644
--- a/apps/server/src/session-archive/lifecycle.ts
+++ b/apps/server/src/session-archive/lifecycle.ts
@@ -201,15 +201,16 @@ export class ProjectArchiveLifecycle {
/**
* The session settled after a turn. `messages` are the engine's messages
* the observer read at that point; the delta carries their completed-turn
- * count, and the archiver skips it when nothing changed. A session whose
- * start could not read the engine (after an app restart the engine can
- * take a minute to answer) is resolved here first, so the turn is not lost.
+ * count, and the archiver skips it when nothing changed. Messages the
+ * observer could not read (null) still get the delta, numbered right after
+ * the last archived turn. A session whose start could not read the engine
+ * (after an app restart the engine can take a minute to answer) is
+ * resolved here first, so the turn is not lost.
*/
turnCompleted(sessionId: string, messages: unknown): void {
const record = this.sessions.get(sessionId);
if (!this.active || this.consentOff || !record) return;
const turns = completedTurnCount(messages);
- if (turns === null) return;
this.schedule(sessionId, "delta", async () => {
if (this.sessions.get(sessionId) !== record) return;
if (record.start && !(await this.resolve(sessionId, record, turns))) return;
diff --git a/apps/server/src/workspace-collector.server.e2e.test.ts b/apps/server/src/workspace-collector.server.e2e.test.ts
index 2f90779d..30e227b1 100644
--- a/apps/server/src/workspace-collector.server.e2e.test.ts
+++ b/apps/server/src/workspace-collector.server.e2e.test.ts
@@ -81,24 +81,40 @@ async function createWorkspace(): Promise<{ root: string; stateDir: string }> {
type EngineMessage = { info: Record<string, unknown>; parts: unknown[] };
+/**
+ * The engine's message paging (`?limit=&before=`): the newest `limit`
+ * messages before the cursor, oldest first, with `X-Next-Cursor` while older
+ * ones remain. Without `limit`, the whole list.
+ */
+function messagePage(url: URL, list: EngineMessage[]): Response {
+ const limit = Number(url.searchParams.get("limit") ?? 0);
+ if (!limit) return Response.json(list);
+ const end = url.searchParams.has("before") ? Number(url.searchParams.get("before")) : list.length;
+ const start = Math.max(0, end - limit);
+ return Response.json(list.slice(start, end), start > 0 ? { headers: { "X-Next-Cursor": String(start) } } : {});
+}
+
/**
* A fake engine with one root session and a two-level subagent tree. Each
* prompt is a turn: the root gains a user + assistant message, the child gains
* a message on turns 1 and 2, the grandchild on turns 1 and 3. It also serves
* what a local workflow step needs: the provider catalog, session creation and
- * the synchronous prompt route on the "ses_workflow" session it creates.
+ * the synchronous prompt route on the "ses_workflow" session it creates. The
+ * root session can start with an earlier `history`.
*/
-function startMockEngine(input: { provider: string; model: string }) {
+function startMockEngine(input: { provider: string; model: string; history?: EngineMessage[] }) {
let turn = 0;
let busy = false;
let workflowBusy = false;
- const root: EngineMessage[] = [];
+ const root: EngineMessage[] = [...(input.history ?? [])];
const child: EngineMessage[] = [];
const grandchild: EngineMessage[] = [];
const workflow: EngineMessage[] = [];
const prompts: unknown[] = [];
const workflowPrompts: unknown[] = [];
const received: string[] = [];
+ /** How long a prompt keeps the root session busy, and the error status its message route answers with (null: none). */
+ const control: { busyMs: number; rootMessagesStatus: number | null } = { busyMs: 50, rootMessagesStatus: null };
const message = (session: string, id: string, role: "user" | "assistant", text: string): EngineMessage => ({
info: {
id,
@@ -145,7 +161,7 @@ function startMockEngine(input: { provider: string; model: string }) {
return Response.json(answer);
}
if (pathname === "/session/ses_workflow") return Response.json(session("ses_workflow"));
- if (pathname === "/session/ses_workflow/message") return Response.json(workflow);
+ if (pathname === "/session/ses_workflow/message") return messagePage(url, workflow);
if (pathname === "/session/ses_workflow/children") return Response.json([]);
if (pathname === "/session/ses_workflow/abort" && request.method === "POST") return Response.json(true);
if (pathname === "/session/ses_root/prompt_async" && request.method === "POST") {
@@ -155,23 +171,25 @@ function startMockEngine(input: { provider: string; model: string }) {
root.push(message("ses_root", `user_${turn}`, "user", `prompt ${turn}`), message("ses_root", `assistant_${turn}`, "assistant", `answer ${turn}`));
if (turn === 1 || turn === 2) child.push(message("ses_child", `child_${turn}`, "assistant", `child answer ${turn} for [email protected]`));
if (turn === 1 || turn === 3) grandchild.push(message("ses_grandchild", `grand_${turn}`, "assistant", `grandchild answer ${turn}`));
- setTimeout(() => { busy = false; }, 50);
+ setTimeout(() => { busy = false; }, control.busyMs);
return new Response(null, { status: 204 });
}
if (pathname === "/session" && request.method === "GET") return Response.json([session("ses_root")]);
if (pathname === "/session/ses_root") return Response.json(session("ses_root"));
- if (pathname === "/session/ses_root/message") return Response.json(root);
+ if (pathname === "/session/ses_root/message") {
+ return control.rootMessagesStatus ? Response.json({ code: "unavailable" }, { status: control.rootMessagesStatus }) : messagePage(url, root);
+ }
if (pathname === "/session/ses_root/children") return Response.json([session("ses_child", "ses_root")]);
- if (pathname === "/session/ses_child/message") return Response.json(child);
+ if (pathname === "/session/ses_child/message") return messagePage(url, child);
if (pathname === "/session/ses_child/children") return Response.json([session("ses_grandchild", "ses_child")]);
- if (pathname === "/session/ses_grandchild/message") return Response.json(grandchild);
+ if (pathname === "/session/ses_grandchild/message") return messagePage(url, grandchild);
if (pathname === "/session/ses_grandchild/children") return Response.json([]);
if (pathname === "/session/ses_root/todo") return Response.json([]);
return Response.json({ code: "not_found", message: `Not found: ${request.method} ${pathname}` }, { status: 404 });
},
}) as Served;
cleanups.push(() => server.stop(true));
- return { server, prompts, workflowPrompts, received, baseUrl: `http://127.0.0.1:${server.port}` };
+ return { server, prompts, workflowPrompts, received, control, baseUrl: `http://127.0.0.1:${server.port}` };
}
/**
@@ -835,15 +853,15 @@ describe("collection gaps: the v2 mount and local workflow steps", () => {
describe("project archive wiring", () => {
/** A server whose gateway also serves the archive routes; `ses_root` has a two-level subagent tree. */
- async function archivedServer(setup: (archive: FakeArchiveServer) => void = () => undefined) {
+ async function archivedServer(setup: (archive: FakeArchiveServer) => void = () => undefined, history?: EngineMessage[]) {
const { root, stateDir } = await createWorkspace();
- const engine = startMockEngine({ provider: "openai", model: "gpt-5" });
+ const engine = startMockEngine({ provider: "openai", model: "gpt-5", ...(history ? { history } : {}) });
const archive = new FakeArchiveServer();
archive.token = "access-token";
setup(archive);
const gateway = startMockGateway(archive);
const omnirush = await startOmniRush(serverConfig({ root, stateDir, engineBaseUrl: engine.baseUrl, gatewayUrl: gateway.gatewayUrl }));
- return { root, stateDir, archive, gateway, omnirush };
+ return { root, stateDir, engine, archive, gateway, omnirush };
}
test("a git workspace's root session is archived through the device session: one base, then a delta for a turn that changed the folder", async () => {
@@ -874,6 +892,63 @@ describe("project archive wiring", () => {
expect(archive.calls.filter((call) => call.path === "archives/key")).toHaveLength(1);
}, 60_000);
+ test("a chat whose history is past the 8 MiB read cap keeps each turn's transcript, turn snapshot and archive delta, and gets its base; a turn whose messages cannot be read keeps its snapshot and delta", async () => {
+ const message = (id: string, role: "user" | "assistant", part: Record<string, unknown>): EngineMessage => ({
+ info: { id, sessionID: "ses_root", role, time: role === "assistant" ? { created: 1, completed: 2 } : { created: 1 } },
+ parts: [{ id: `${id}_part`, messageID: id, sessionID: "ses_root", ...part }],
+ });
+ // A first prompt carrying a PDF the engine stored whole as a data URL, over
+ // the read cap on its own, then thirty turns of 100 KB answers (over one page).
+ const history = [
+ message("old_user_0", "user", { type: "file", mime: "application/pdf", filename: "scan.pdf", url: `data:application/pdf;base64,${"A".repeat(9 * 1024 * 1024)}` }),
+ message("old_assistant_0", "assistant", { type: "text", text: "read it" }),
+ ...Array.from({ length: 30 }, (_, index) => [
+ message(`old_user_${index + 1}`, "user", { type: "text", text: `question ${index + 1}` }),
+ message(`old_assistant_${index + 1}`, "assistant", { type: "text", text: "x".repeat(100 * 1024) }),
+ ]).flat(),
+ ];
+ const { root, engine, archive, gateway, omnirush } = await archivedServer(undefined, history);
+ expect((await prompt(omnirush.base)).status).toBe(204);
+ await waitFor(() => (traces(gateway.uploads).length >= 1 ? true : undefined));
+ await writeFile(join(root, "feature.txt"), "new work\n");
+ expect((await prompt(omnirush.base)).status).toBe(204);
+ await waitFor(() => (traces(gateway.uploads).length >= 2 ? true : undefined));
+ await waitFor(() => (archive.objects().some((object) => object.request.kind === "delta") ? true : undefined));
+ // Then the engine stops answering for the messages (a turn long enough to be seen running).
+ engine.control.rootMessagesStatus = 500;
+ engine.control.busyMs = 2_500;
+ await writeFile(join(root, "later.txt"), "more work\n");
+ expect((await prompt(omnirush.base)).status).toBe(204);
+ await waitFor(() => (traces(gateway.uploads).length >= 3 ? true : undefined));
+ const objects = await waitFor(() => (archive.objects().filter((object) => object.request.kind === "delta").length >= 2 ? archive.objects() : undefined));
+ await omnirush.stop();
+
+ const [turnOne, turnTwo, turnThree] = traces(gateway.uploads);
+ const turnMessageIds = (envelope: Envelope | undefined) => (events(envelope).find((event) => event.type === "turn.completed")?.data?.messages as Array<{ info: { id: string } }>)
+ .map((entry) => entry.info.id);
+ // No checkpoint on the first turn: every message but the one too large to read, oldest first.
+ expect(turnMessageIds(turnOne)).toEqual(history.slice(1).map((entry) => String(entry.info.id)).concat(["user_1", "assistant_1"]));
+ expect(events(turnOne).find((event) => event.type === "session.messages_omitted")?.data).toEqual({ count: 1 });
+ // The next turn carries just its own messages.
+ expect(turnMessageIds(turnTwo)).toEqual(["user_2", "assistant_2"]);
+ for (const envelope of [turnOne, turnTwo]) {
+ const types = events(envelope).map((event) => event.type);
+ expect(types).not.toContain("session.observer_failed");
+ expect(types).not.toContain("session.messages_failed");
+ expect(events(envelope).filter((event) => event.type === "collector.trigger").map((event) => event.data?.trigger)).toContain("turn_completed");
+ }
+ // Unreadable messages: no transcript, but the turn snapshot and the trace still go out.
+ expect(events(turnThree).find((event) => event.type === "turn.completed")?.data).toEqual({ messages: { status: 500, unavailable: true } });
+ expect(events(turnThree).find((event) => event.type === "session.messages_failed")?.data).toEqual({ error: "the engine answered 500" });
+ expect(events(turnThree).filter((event) => event.type === "collector.trigger").map((event) => event.data?.trigger)).toContain("turn_completed");
+ // The base counts every completed turn, the PDF turn included (read from
+ // its info alone): 31 earlier ones plus the first prompt's. The changed
+ // second turn gets its delta, and so does the third, numbered right after it.
+ expect(objects.map((object) => [object.request.kind, object.request.turn])).toEqual([["base", 32], ["delta", 33], ["delta", 34]]);
+ expect((await openArchive(objects[1]!.object!)).map((member) => member.name)).toContain("feature.txt");
+ expect((await openArchive(objects[2]!.object!)).map((member) => member.name)).toContain("later.txt");
+ }, 90_000);
+
test("without archive consent (428) nothing is packed or uploaded, consent is checked once, and the chat carries on", async () => {
const { stateDir, archive, gateway, omnirush } = await archivedServer((fake) => {
fake.gate = { status: 428, detail: "archive_consent_required" };