Skip to content

A self-improving RLM agent for coding workflows and long-running autonomous tasks.

Rust

fix snapshot catch-up replacement races (#442)

kt144a84ef9364
diff --git a/packages/coding-agent/CHANGELOG.md b/packages/coding-agent/CHANGELOG.md
index 939eb03ae4..1df5115b0d 100644
--- a/packages/coding-agent/CHANGELOG.md
+++ b/packages/coding-agent/CHANGELOG.md
@@ -7,6 +7,7 @@
- Changed shell commands to use discoverable agent, schedule, package, model, session, update, doctor, and full-shutdown verbs without exposing the background daemon hierarchy ([ENG-4538](https://linear.app/primeintellect/issue/ENG-4538/standardize-bash-command-conventions-and-improve-command-discovery)).
- Fixed unsupported Node versions crashing before startup by requiring Node 22.8.0 or newer and showing upgrade guidance before loading the CLI ([ENG-4260](https://linear.app/primeintellect/issue/ENG-4260/incorrect-node-version-breaks-first-launch)).
- Added `@` file-path autocomplete to new-agent and reply prompts in the Agents View.
+- Fixed slow daemon clients becoming stuck when newer session snapshots arrived during catch-up.
- Fixed queued messages getting stranded when an agent turn ended ([ENG-4653](https://linear.app/primeintellect/issue/ENG-4653/queued-messages-can-get-stuck-with-heartbeats)).
- Changed `/traces upload-all` to pace requests within the platform rate limit, honor bounded `Retry-After` responses, and support interruption.
- Fixed resuming a daemon-resident session to attach the requesting client to its existing worker without disturbing other clients ([ENG-4656](https://linear.app/primeintellect/issue/ENG-4656/resuming-prime-agent-sessions-should-attach)).
diff --git a/packages/coding-agent/src/modes/daemon/active-session-state.ts b/packages/coding-agent/src/modes/daemon/active-session-state.ts
index 48196e155d..b0a9c7f0ee 100644
--- a/packages/coding-agent/src/modes/daemon/active-session-state.ts
+++ b/packages/coding-agent/src/modes/daemon/active-session-state.ts
@@ -13,6 +13,8 @@ export interface DaemonSocketClient {
catchupActiveSessionIds?: Set<string>;
/** A real runtime replacement takes precedence over an ordinary resync for the same queued catch-up. */
catchupPurposes?: Map<string, "replacement" | "resync">;
+ /** The single catch-up drain currently serving this client. */
+ catchupPromise?: Promise<void>;
backpressured?: boolean;
authenticated?: boolean;
transport?: "jsonl" | "private-framed";
diff --git a/packages/coding-agent/src/modes/daemon/daemon-mode.ts b/packages/coding-agent/src/modes/daemon/daemon-mode.ts
index 45abcdfe83..1d21ac732e 100644
--- a/packages/coding-agent/src/modes/daemon/daemon-mode.ts
+++ b/packages/coding-agent/src/modes/daemon/daemon-mode.ts
@@ -2284,6 +2284,7 @@ export class AgentDaemon {
socket.on("close", cleanup);
socket.on("error", cleanup);
socket.on("drain", () => {
+ client.backpressured = false;
if (!client.snapshotStreaming) {
void this.catchUpBackpressuredClient(client).catch((error) =>
this.log(`could not catch up snapshot client ${client.id}: ${String(error)}`),
@@ -4602,11 +4603,37 @@ export class AgentDaemon {
}
}
- private async catchUpBackpressuredClient(client: DaemonSocketClient): Promise<void> {
+ private catchUpBackpressuredClient(client: DaemonSocketClient): Promise<void> {
+ if (client.catchupPromise) {
+ return client.catchupPromise;
+ }
+ if (client.snapshotStreaming || client.backpressured) {
+ return Promise.resolve();
+ }
+ const catchup = this.drainBackpressuredClientCatchupQueue(client).finally(() => {
+ if (client.catchupPromise === catchup) {
+ client.catchupPromise = undefined;
+ }
+ });
+ client.catchupPromise = catchup;
+ return catchup;
+ }
+
+ private async drainBackpressuredClientCatchupQueue(client: DaemonSocketClient): Promise<void> {
+ while (
+ !client.socket.destroyed &&
+ !client.snapshotStreaming &&
+ !client.backpressured &&
+ client.catchupActiveSessionIds?.size
+ ) {
+ await this.drainBackpressuredClientCatchups(client);
+ }
+ }
+
+ private async drainBackpressuredClientCatchups(client: DaemonSocketClient): Promise<void> {
if (client.socket.destroyed) {
return;
}
- client.backpressured = false;
const pending = [...(client.catchupActiveSessionIds ?? [])].map((activeSessionId) => ({
activeSessionId,
purpose: client.catchupPurposes?.get(activeSessionId) ?? ("resync" as const),
@@ -4771,6 +4798,9 @@ export class AgentDaemon {
outboundType: message.type,
...("id" in message && typeof message.id === "string" ? { requestId: message.id } : {}),
...(hasDaemonOutboundActiveSessionId(message) ? { activeSessionId: message.activeSessionId } : {}),
+ ...("snapshotId" in message && typeof message.snapshotId === "string"
+ ? { snapshotId: message.snapshotId }
+ : {}),
...(message.type === "session_event" ? { sessionEventType: message.event.type } : {}),
payloadEncoding,
...(snapshotPurpose ? { snapshotPurpose } : {}),
diff --git a/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts b/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts
index 223f7e4259..141bbce2f7 100644
--- a/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts
+++ b/packages/coding-agent/src/modes/daemon/daemon-supervisor.ts
@@ -198,17 +198,7 @@ interface ResidentWorker {
summaries: Map<string, SessionSummary>;
snapshotCache: Map<string, DaemonAttachResult>;
transcriptCaches: Map<string, SnapshotTranscriptCache>;
- incomingTranscriptActiveSessionIds: Set<string>;
- duplicateIncomingTranscriptChunkIndexes: Map<string, number>;
- snapshotTransferFrames: Map<
- string,
- {
- begin: Buffer;
- end?: Buffer;
- duplicateResult?: DaemonAttachResult;
- validation?: SnapshotDuplicateValidation;
- }
- >;
+ snapshotGenerations: Map<string, Map<string, SnapshotTranscriptGeneration>>;
snapshotLoads: Map<string, Promise<DaemonAttachResult>>;
recovery?: Promise<void>;
deferredRecovery?: Promise<void>;
@@ -222,6 +212,18 @@ interface SnapshotDuplicateValidation {
reject: (error: Error) => void;
}
+interface SnapshotTranscriptGeneration {
+ transcript: SnapshotTranscriptCache;
+ result: DaemonAttachResult;
+ begin?: Buffer;
+ end?: Buffer;
+ incoming: boolean;
+ retired: boolean;
+ duplicateChunkIndex?: number;
+ duplicateResult?: DaemonAttachResult;
+ validation?: SnapshotDuplicateValidation;
+}
+
interface DaemonSupervisorOptions {
socketPath?: string;
defaultSessionConfig: AgentSessionRuntimeConfig;
@@ -242,6 +244,8 @@ interface WorkerMatch {
interface WorkerAttachData {
result: DaemonAttachResult;
worker: ResidentWorker;
+ transcript?: SnapshotTranscriptCache;
+ releaseTranscript?: () => void;
}
class SupervisorRecoveryCancelledError extends Error {
@@ -523,10 +527,6 @@ export class DaemonSupervisor {
private commandJournal!: CommandRecoveryJournal;
private readonly streamReconstructor = new CompactAssistantStreamReconstructor();
private readonly compactCatchupInProgress = new Set<string>();
- private readonly pendingReplacementSnapshots = new WeakMap<
- DaemonSocketClient,
- Map<string, { worker: ResidentWorker; result: DaemonAttachResult; transcript: SnapshotTranscriptCache }>
- >();
private agentPeerSyncQueue: Promise<void> = Promise.resolve();
private readonly catalog: DaemonCatalogClient;
@@ -705,9 +705,7 @@ export class DaemonSupervisor {
summaries: new Map(),
snapshotCache: new Map(),
transcriptCaches: new Map(),
- incomingTranscriptActiveSessionIds: new Set(),
- duplicateIncomingTranscriptChunkIndexes: new Map(),
- snapshotTransferFrames: new Map(),
+ snapshotGenerations: new Map(),
snapshotLoads: new Map(),
intentionalStop: descriptor.stopRequestedAt !== undefined,
stopRevision: 0,
@@ -825,7 +823,6 @@ export class DaemonSupervisor {
}
cleaned = true;
client.detachInput();
- this.dropPendingReplacementSnapshot(client);
this.clients.delete(client);
for (const activeSessionId of [...client.attachedActiveSessionIds]) {
client.attachedActiveSessionIds.delete(activeSessionId);
@@ -835,6 +832,7 @@ export class DaemonSupervisor {
socket.on("close", cleanup);
socket.on("error", cleanup);
socket.on("drain", () => {
+ client.backpressured = false;
if (!client.snapshotStreaming) {
void this.catchUpClient(client).catch((error) =>
this.log(`Failed to catch up client ${client.id}: ${String(error)}`),
@@ -954,12 +952,29 @@ export class DaemonSupervisor {
case "attach": {
const attached = await this.attachClient(client, command);
if (client.capabilities.has("chunked_snapshot")) {
- const transcript = this.getOrCreateTranscriptCache(attached.worker, attached.result);
+ const transcript = attached.transcript;
+ if (!transcript) {
+ throw new Error("Session worker did not provide a snapshot transcript");
+ }
const streamedResult = this.createStreamedAttachResult(attached.result, transcript);
- this.write(client, success(command.id, "attach", streamedResult));
- void this.streamSnapshot(client, attached.worker, streamedResult, transcript).catch((error) =>
- this.log(`Failed to stream attach snapshot for ${streamedResult.activeSessionId}: ${String(error)}`),
- );
+ try {
+ this.write(client, success(command.id, "attach", streamedResult));
+ void this.streamSnapshot(
+ client,
+ attached.worker,
+ streamedResult,
+ transcript,
+ "attach",
+ attached.releaseTranscript,
+ ).catch((error) =>
+ this.log(
+ `Failed to stream attach snapshot for ${streamedResult.activeSessionId}: ${String(error)}`,
+ ),
+ );
+ } catch (error) {
+ attached.releaseTranscript?.();
+ throw error;
+ }
return undefined;
}
return success(command.id, "attach", attached.result);
@@ -972,6 +987,7 @@ export class DaemonSupervisor {
}
const targetWasAttached = client.attachedActiveSessionIds.has(targetActiveSessionId);
const releaseSnapshotReservation = this.reserveSnapshotStream(client, targetActiveSessionId);
+ let releaseTranscript: (() => void) | undefined;
client.attachedActiveSessionIds.add(targetActiveSessionId);
try {
const attached = await this.attachClient(client, {
@@ -980,18 +996,23 @@ export class DaemonSupervisor {
activeSessionId: targetActiveSessionId,
});
if (client.capabilities.has("chunked_snapshot")) {
- const transcript = this.getOrCreateTranscriptCache(attached.worker, attached.result);
+ const transcript =
+ attached.transcript ?? this.getOrCreateTranscriptCache(attached.worker, attached.result);
+ releaseTranscript = attached.releaseTranscript;
const streamedResult = this.createStreamedAttachResult(attached.result, transcript);
this.write(client, success(command.id, command.type, streamedResult));
this.detachClient(client, command.activeSessionId);
- void this.streamSnapshot(
+ const streaming = this.streamSnapshot(
client,
attached.worker,
streamedResult,
transcript,
"replacement",
+ releaseTranscript,
releaseSnapshotReservation,
- ).catch((error) =>
+ );
+ releaseTranscript = undefined;
+ void streaming.catch((error) =>
this.log(`Failed to stream reattach snapshot for ${targetActiveSessionId}: ${String(error)}`),
);
return undefined;
@@ -1001,6 +1022,7 @@ export class DaemonSupervisor {
releaseSnapshotReservation();
return undefined;
} catch (error) {
+ releaseTranscript?.();
if (!targetWasAttached) {
this.detachClient(client, targetActiveSessionId);
}
@@ -1440,9 +1462,7 @@ export class DaemonSupervisor {
summaries: new Map(),
snapshotCache: new Map(),
transcriptCaches: new Map(),
- incomingTranscriptActiveSessionIds: new Set(),
- duplicateIncomingTranscriptChunkIndexes: new Map(),
- snapshotTransferFrames: new Map(),
+ snapshotGenerations: new Map(),
snapshotLoads: new Map(),
intentionalStop: false,
stopRevision: 0,
@@ -1673,19 +1693,33 @@ export class DaemonSupervisor {
return;
}
worker.client = undefined;
- this.dropPendingReplacementSnapshotsForWorker(worker);
- const interrupted = new Set(worker.incomingTranscriptActiveSessionIds);
+ const interrupted = new Map<string, Set<string>>();
+ for (const [activeSessionId, generations] of worker.snapshotGenerations ?? []) {
+ for (const generation of generations.values()) {
+ if (generation.incoming || !generation.transcript.complete) {
+ const snapshotIds = interrupted.get(activeSessionId) ?? new Set<string>();
+ snapshotIds.add(generation.transcript.snapshotId);
+ interrupted.set(activeSessionId, snapshotIds);
+ }
+ }
+ }
for (const [activeSessionId, transcript] of worker.transcriptCaches) {
if (!transcript.complete) {
- interrupted.add(activeSessionId);
+ const snapshotIds = interrupted.get(activeSessionId) ?? new Set<string>();
+ snapshotIds.add(transcript.snapshotId);
+ interrupted.set(activeSessionId, snapshotIds);
}
}
- for (const activeSessionId of interrupted) {
- this.failWorkerSnapshotCache(
- worker,
- activeSessionId,
- new Error("Session worker disconnected during snapshot transfer"),
- );
+ for (const [activeSessionId, snapshotIds] of interrupted) {
+ for (const snapshotId of snapshotIds) {
+ this.failWorkerSnapshotCache(
+ worker,
+ activeSessionId,
+ new Error("Session worker disconnected during snapshot transfer"),
+ false,
+ snapshotId,
+ );
+ }
}
if (this.shuttingDown || worker.intentionalStop) {
return;
@@ -1768,19 +1802,39 @@ export class DaemonSupervisor {
activeSessionId: string,
error: Error,
closeWorkerChannel = false,
+ expectedSnapshotId?: string,
): void {
- const transcript = worker.transcriptCaches.get(activeSessionId);
- if (transcript) {
- transcript.markFailed(error);
+ const generations = worker.snapshotGenerations?.get(activeSessionId);
+ if (expectedSnapshotId) {
+ const generation = generations?.get(expectedSnapshotId);
+ if (generation) {
+ this.failSnapshotGeneration(worker, activeSessionId, generation, error);
+ } else {
+ const transcript = worker.transcriptCaches.get(activeSessionId);
+ if (transcript?.snapshotId !== expectedSnapshotId) {
+ return;
+ }
+ transcript.markFailed(error);
+ transcript.dispose();
+ worker.transcriptCaches.delete(activeSessionId);
+ if (worker.snapshotCache.get(activeSessionId)?.snapshotStream?.id === expectedSnapshotId) {
+ worker.snapshotCache.delete(activeSessionId);
+ }
+ }
+ } else {
+ const failedTranscripts = new Set<SnapshotTranscriptCache>();
+ for (const generation of [...(generations?.values() ?? [])]) {
+ failedTranscripts.add(generation.transcript);
+ this.failSnapshotGeneration(worker, activeSessionId, generation, error);
+ }
+ const transcript = worker.transcriptCaches.get(activeSessionId);
+ if (transcript && !failedTranscripts.has(transcript)) {
+ transcript.markFailed(error);
+ transcript.dispose();
+ }
worker.transcriptCaches.delete(activeSessionId);
- transcript.dispose();
+ worker.snapshotCache.delete(activeSessionId);
}
- worker.snapshotCache.delete(activeSessionId);
- worker.incomingTranscriptActiveSessionIds.delete(activeSessionId);
- worker.duplicateIncomingTranscriptChunkIndexes.delete(activeSessionId);
- this.settleSnapshotDuplicateValidation(worker, activeSessionId, error);
- worker.snapshotTransferFrames.delete(activeSessionId);
- this.dropPendingReplacementSnapshotsForWorker(worker, activeSessionId);
if (closeWorkerChannel) {
const client = worker.client;
if (client) {
@@ -1790,6 +1844,116 @@ export class DaemonSupervisor {
}
}
+ private retireWorkerSnapshotCache(
+ worker: ResidentWorker,
+ activeSessionId: string,
+ expectedTranscript: SnapshotTranscriptCache,
+ ): void {
+ if (worker.transcriptCaches.get(activeSessionId) === expectedTranscript) {
+ worker.transcriptCaches.delete(activeSessionId);
+ }
+ if (worker.snapshotCache.get(activeSessionId)?.snapshotStream?.id === expectedTranscript.snapshotId) {
+ worker.snapshotCache.delete(activeSessionId);
+ }
+ const generation = this.snapshotGeneration(worker, activeSessionId, expectedTranscript.snapshotId);
+ if (!generation) {
+ expectedTranscript.dispose();
+ return;
+ }
+ generation.retired = true;
+ this.settleSnapshotDuplicateValidation(generation);
+ if (generation.incoming) {
+ return;
+ }
+ this.deleteSnapshotGeneration(worker, activeSessionId, generation);
+ expectedTranscript.dispose();
+ }
+
+ private snapshotGenerationsFor(
+ worker: ResidentWorker,
+ activeSessionId: string,
+ ): Map<string, SnapshotTranscriptGeneration> {
+ worker.snapshotGenerations ??= new Map();
+ let generations = worker.snapshotGenerations.get(activeSessionId);
+ if (!generations) {
+ generations = new Map();
+ worker.snapshotGenerations.set(activeSessionId, generations);
+ }
+ return generations;
+ }
+
+ private snapshotGeneration(
+ worker: ResidentWorker,
+ activeSessionId: string,
+ snapshotId: string,
+ ): SnapshotTranscriptGeneration | undefined {
+ return worker.snapshotGenerations?.get(activeSessionId)?.get(snapshotId);
+ }
+
+ private currentSnapshotGeneration(
+ worker: ResidentWorker,
+ activeSessionId: string,
+ ): SnapshotTranscriptGeneration | undefined {
+ worker.transcriptCaches ??= new Map();
+ worker.snapshotCache ??= new Map();
+ const transcript = worker.transcriptCaches.get(activeSessionId);
+ if (!transcript) {
+ return undefined;
+ }
+ const generations = this.snapshotGenerationsFor(worker, activeSessionId);
+ let generation = generations.get(transcript.snapshotId);
+ if (generation) {
+ return generation;
+ }
+ const result = worker.snapshotCache.get(activeSessionId);
+ if (!result) {
+ return undefined;
+ }
+ generation = {
+ transcript,
+ result,
+ incoming: false,
+ retired: false,
+ };
+ generations.set(transcript.snapshotId, generation);
+ return generation;
+ }
+
+ private deleteSnapshotGeneration(
+ worker: ResidentWorker,
+ activeSessionId: string,
+ generation: SnapshotTranscriptGeneration,
+ ): void {
+ const generations = worker.snapshotGenerations?.get(activeSessionId);
+ if (!generations) {
+ return;
+ }
+ if (generations.get(generation.transcript.snapshotId) === generation) {
+ generations.delete(generation.transcript.snapshotId);
+ }
+ if (generations.size === 0) {
+ worker.snapshotGenerations.delete(activeSessionId);
+ }
+ }
+
+ private failSnapshotGeneration(
+ worker: ResidentWorker,
+ activeSessionId: string,
+ generation: SnapshotTranscriptGeneration,
+ error: Error,
+ ): void {
+ this.settleSnapshotDuplicateValidation(generation, error);
+ generation.transcript.markFailed(error);
+ generation.transcript.dispose();
+ this.deleteSnapshotGeneration(worker, activeSessionId, generation);
+ if (worker.transcriptCaches.get(activeSessionId) === generation.transcript) {
+ worker.transcriptCaches.delete(activeSessionId);
+ }
+ if (worker.snapshotCache.get(activeSessionId)?.snapshotStream?.id === generation.transcript.snapshotId) {
+ worker.snapshotCache.delete(activeSessionId);
+ }
+ }
+
private createSnapshotDuplicateValidation(): SnapshotDuplicateValidation {
let resolve!: () => void;
let reject!: (error: Error) => void;
@@ -1801,13 +1965,12 @@ export class DaemonSupervisor {
return { promise, resolve, reject };
}
- private settleSnapshotDuplicateValidation(worker: ResidentWorker, activeSessionId: string, error?: Error): void {
- const frames = worker.snapshotTransferFrames.get(activeSessionId);
- if (!frames?.validation) {
+ private settleSnapshotDuplicateValidation(generation: SnapshotTranscriptGeneration, error?: Error): void {
+ const validation = generation.validation;
+ if (!validation) {
return;
}
- const { validation, ...settledFrames } = frames;
- worker.snapshotTransferFrames.set(activeSessionId, settledFrames);
+ generation.validation = undefined;
if (error) {
validation.reject(error);
} else {
@@ -2164,7 +2327,7 @@ export class DaemonSupervisor {
): Promise<WorkerAttachData> {
const match = await this.findWorker(command.activeSessionId);
const activeSessionId = match.summary.activeSessionId ?? match.summary.id;
- const duplicateValidation = match.worker.snapshotTransferFrames.get(activeSessionId)?.validation;
+ const duplicateValidation = this.currentSnapshotGeneration(match.worker, activeSessionId)?.validation;
if (duplicateValidation) {
await duplicateValidation.promise;
}
@@ -2186,6 +2349,9 @@ export class DaemonSupervisor {
const snapshotLoadKey = `${activeSessionId}:${client.capabilities.has("chunked_snapshot") ? "chunked" : "full"}`;
let loading = match.worker.snapshotLoads.get(snapshotLoadKey);
if (!loading) {
+ const observedSnapshotId =
+ match.worker.transcriptCaches.get(activeSessionId)?.snapshotId ??
+ match.worker.snapshotCache.get(activeSessionId)?.snapshotStream?.id;
loading = (async () => {
if (!match.worker.client) {
throw new Error("Session worker is not connected");
@@ -2200,24 +2366,7 @@ export class DaemonSupervisor {
env: command.env ?? collectDaemonClientEnv(),
});
const loaded = attachResultFromResponse(response);
- if (loaded.snapshotStream) {
- let transcript = match.worker.transcriptCaches.get(activeSessionId);
- if (transcript && transcript.snapshotId !== loaded.snapshotStream.id) {
- transcript.dispose();
- transcript = undefined;
- }
- if (!transcript) {
- transcript = new SnapshotTranscriptCache({
- activeSessionId,
- snapshotId: loaded.snapshotStream.id,
- cacheRoot: this.snapshotCacheRoot,
- targetChunkBytes: loaded.snapshotStream.targetChunkBytes,
- });
- match.worker.transcriptCaches.set(activeSessionId, transcript);
- }
- }
- match.worker.snapshotCache.set(activeSessionId, loaded);
- return loaded;
+ return this.cacheLoadedSnapshot(match.worker, activeSessionId, loaded, observedSnapshotId);
})().finally(() => {
match.worker.snapshotLoads.delete(snapshotLoadKey);
});
@@ -2226,6 +2375,20 @@ export class DaemonSupervisor {
result = await loading;
}
const wasAttached = client.attachedActiveSessionIds.has(activeSessionId);
+ let transcript: SnapshotTranscriptCache | undefined;
+ if (client.capabilities.has("chunked_snapshot")) {
+ while (true) {
+ const validation = this.currentSnapshotGeneration(match.worker, activeSessionId)?.validation;
+ if (validation) {
+ await validation.promise;
+ continue;
+ }
+ result = match.worker.snapshotCache.get(activeSessionId) ?? result;
+ transcript = this.getOrCreateTranscriptCache(match.worker, result);
+ break;
+ }
+ }
+ const releaseTranscript = transcript?.retain();
client.attachedActiveSessionIds.add(activeSessionId);
try {
const publicSummary = this.publicSummary(match.worker, result.snapshot.summary);
@@ -2257,9 +2420,10 @@ export class DaemonSupervisor {
lastEventSequence: publicResult.lastEventSequence,
});
}
- await this.syncWorkerExtensionUi(activeSessionId);
- return { result: publicResult, worker: match.worker };
+ void this.syncWorkerExtensionUi(activeSessionId);
+ return { result: publicResult, worker: match.worker, transcript, releaseTranscript };
} catch (error) {
+ releaseTranscript?.();
if (!wasAttached) {
client.attachedActiveSessionIds.delete(activeSessionId);
}
@@ -2267,14 +2431,78 @@ export class DaemonSupervisor {
}
}
+ private cacheLoadedSnapshot(
+ worker: ResidentWorker,
+ activeSessionId: string,
+ loaded: DaemonAttachResult,
+ observedSnapshotId: string | undefined,
+ ): DaemonAttachResult {
+ const currentTranscript = worker.transcriptCaches.get(activeSessionId);
+ const currentGeneration = this.currentSnapshotGeneration(worker, activeSessionId);
+ const currentResult = currentGeneration?.result ?? worker.snapshotCache.get(activeSessionId);
+ const currentSnapshotId = currentTranscript?.snapshotId ?? currentResult?.snapshotStream?.id;
+ const loadedSnapshotId = loaded.snapshotStream?.id;
+ if (!loaded.snapshotStream) {
+ if (currentSnapshotId && currentSnapshotId !== observedSnapshotId) {
+ return loaded;
+ }
+ worker.snapshotCache.set(activeSessionId, loaded);
+ return loaded;
+ }
+ if (
+ currentSnapshotId &&
+ currentSnapshotId !== loadedSnapshotId &&
+ (currentSnapshotId !== observedSnapshotId ||
+ (currentResult?.lastEventSequence ?? -1) > loaded.lastEventSequence)
+ ) {
+ return currentResult ?? loaded;
+ }
+ let transcript = currentTranscript;
+ if (transcript && transcript.snapshotId !== loaded.snapshotStream.id) {
+ this.retireWorkerSnapshotCache(worker, activeSessionId, transcript);
+ transcript = undefined;
+ }
+ const generations = this.snapshotGenerationsFor(worker, activeSessionId);
+ let generation = generations.get(loaded.snapshotStream.id);
+ if (!transcript) {
+ transcript = generation?.transcript;
+ }
+ if (!transcript) {
+ transcript = new SnapshotTranscriptCache({
+ activeSessionId,
+ snapshotId: loaded.snapshotStream.id,
+ cacheRoot: this.snapshotCacheRoot,
+ targetChunkBytes: loaded.snapshotStream.targetChunkBytes,
+ });
+ }
+ if (!generation) {
+ generation = {
+ transcript,
+ result: loaded,
+ incoming: false,
+ retired: false,
+ };
+ generations.set(loaded.snapshotStream.id, generation);
+ } else {
+ generation.result = loaded;
+ generation.retired = false;
+ }
+ worker.transcriptCaches.set(activeSessionId, transcript);
+ worker.snapshotCache.set(activeSessionId, loaded);
+ return loaded;
+ }
+
private getOrCreateTranscriptCache(worker: ResidentWorker, result: DaemonAttachResult): SnapshotTranscriptCache {
const activeSessionId = result.activeSessionId;
const existing = worker.transcriptCaches.get(activeSessionId);
- if (existing) {
+ if (existing && (!result.snapshotStream || existing.snapshotId === result.snapshotStream.id)) {
return existing;
}
if (result.snapshot.messages.length < result.snapshot.summary.messageCount) {
- throw new Error("Session worker returned snapshot metadata without a transcript cache");
+ throw new Error("Session snapshot generation changed before its transcript could be selected");
+ }
+ if (existing) {
+ this.retireWorkerSnapshotCache(worker, activeSessionId, existing);
}
const revision = createHash("sha256")
.update(
@@ -2290,10 +2518,17 @@ export class DaemonSupervisor {
targetChunkBytes: SNAPSHOT_TARGET_CHUNK_BYTES,
});
worker.transcriptCaches.set(activeSessionId, transcript);
- worker.snapshotCache.set(activeSessionId, {
+ const cachedResult = {
...result,
messages: result.messages ? [] : undefined,
snapshot: { ...result.snapshot, messages: [] },
+ };
+ worker.snapshotCache.set(activeSessionId, cachedResult);
+ this.snapshotGenerationsFor(worker, activeSessionId).set(transcript.snapshotId, {
+ transcript,
+ result: cachedResult,
+ incoming: false,
+ retired: false,
});
return transcript;
}
@@ -2320,14 +2555,16 @@ export class DaemonSupervisor {
result: DaemonAttachResult,
transcript: SnapshotTranscriptCache,
purpose: "attach" | "replacement" | "resync" = "attach",
+ retainedTranscriptRelease?: () => void,
releaseSnapshotReservation = this.reserveSnapshotStream(client, result.activeSessionId),
): Promise<void> {
const stream = result.snapshotStream;
+ const releaseTranscript = retainedTranscriptRelease ?? transcript.retain();
if (!stream || client.socket.destroyed) {
releaseSnapshotReservation();
+ releaseTranscript();
return;
}
- const releaseTranscript = transcript.retain();
const { messages: _messages, ...snapshotHeader } = result.snapshot;
try {
if (
@@ -2345,7 +2582,14 @@ export class DaemonSupervisor {
}
let chunkCount = 0;
while (true) {
- const chunk = await transcript.waitForChunk(chunkCount);
+ let chunk: Buffer | undefined;
+ try {
+ chunk = await transcript.waitForChunk(chunkCount);
+ } catch (error) {
+ const streamError = error instanceof Error ? error : new Error(String(error));
+ this.failWorkerSnapshotCache(worker, result.activeSessionId, streamError, false, stream.id);
+ throw streamError;
+ }
if (!chunk) {
break;
}
@@ -2364,7 +2608,6 @@ export class DaemonSupervisor {
});
} catch (error) {
const streamError = error instanceof Error ? error : new Error(String(error));
- this.failWorkerSnapshotCache(worker, result.activeSessionId, streamError);
if (!client.socket.destroyed) {
try {
const delivered = await this.writeSnapshotRecord(client, {
@@ -2413,7 +2656,6 @@ export class DaemonSupervisor {
if (!client.snapshotStreaming) {
client.backpressured = false;
}
- this.flushPendingReplacementSnapshot(client, activeSessionId);
if (!client.snapshotStreaming && client.catchupActiveSessionIds?.size) {
void this.catchUpClient(client).catch((error) =>
this.log(`Failed to catch up client ${client.id}: ${String(error)}`),
@@ -2463,7 +2705,6 @@ export class DaemonSupervisor {
}
client.catchupActiveSessionIds?.delete(resolvedId);
client.catchupPurposes?.delete(resolvedId);
- this.dropPendingReplacementSnapshot(client, resolvedId);
this.write(client, { type: "session_detached", activeSessionId: resolvedId });
void this.syncWorkerExtensionUi(resolvedId);
}
@@ -2479,103 +2720,18 @@ export class DaemonSupervisor {
);
}
- private streamReplacementSnapshot(
- worker: ResidentWorker,
- activeSessionId: string,
- result: DaemonAttachResult,
- transcript: SnapshotTranscriptCache,
- ): void {
- for (const client of this.clients) {
- if (!client.attachedActiveSessionIds.has(activeSessionId) || !client.capabilities.has("chunked_snapshot")) {
- continue;
- }
- if (client.snapshotActiveSessionIds?.has(activeSessionId)) {
- const pending = this.pendingReplacementSnapshots.get(client) ?? new Map();
- pending.set(activeSessionId, { worker, result, transcript });
- this.pendingReplacementSnapshots.set(client, pending);
- continue;
- }
- this.startReplacementSnapshot(client, worker, activeSessionId, result, transcript);
- }
- }
-
- private startReplacementSnapshot(
- client: DaemonSocketClient,
- worker: ResidentWorker,
- activeSessionId: string,
- result: DaemonAttachResult,
- transcript: SnapshotTranscriptCache,
- ): void {
- void this.streamSnapshot(client, worker, result, transcript, "replacement").catch((error) =>
- this.log(`Failed to stream replacement snapshot for ${activeSessionId}: ${String(error)}`),
- );
- }
-
- private flushPendingReplacementSnapshot(client: DaemonSocketClient, activeSessionId: string): void {
- const pending = this.pendingReplacementSnapshots.get(client);
- const replacement = pending?.get(activeSessionId);
- if (!pending || !replacement) {
- return;
- }
- if (
- client.socket.destroyed ||
- !client.attachedActiveSessionIds.has(activeSessionId) ||
- !client.capabilities.has("chunked_snapshot") ||
- replacement.worker.transcriptCaches.get(activeSessionId) !== replacement.transcript
- ) {
- this.dropPendingReplacementSnapshot(client, activeSessionId);
- return;
- }
- if (!replacement.transcript.complete || client.snapshotActiveSessionIds?.has(activeSessionId)) {
- return;
- }
- this.dropPendingReplacementSnapshot(client, activeSessionId);
- this.startReplacementSnapshot(
- client,
- replacement.worker,
- activeSessionId,
- replacement.result,
- replacement.transcript,
- );
- }
-
- private dropPendingReplacementSnapshot(client: DaemonSocketClient, activeSessionId?: string): void {
- const pending = this.pendingReplacementSnapshots.get(client);
- if (!pending || activeSessionId === undefined) {
- this.pendingReplacementSnapshots.delete(client);
- return;
- }
- pending.delete(activeSessionId);
- if (pending.size === 0) {
- this.pendingReplacementSnapshots.delete(client);
- }
- }
-
- private dropPendingReplacementSnapshotsForWorker(worker: ResidentWorker, targetActiveSessionId?: string): void {
- for (const client of this.clients) {
- const pending = this.pendingReplacementSnapshots.get(client);
- if (!pending) {
- continue;
- }
- for (const [activeSessionId, replacement] of pending) {
- if (
- replacement.worker === worker &&
- (targetActiveSessionId === undefined || activeSessionId === targetActiveSessionId)
- ) {
- pending.delete(activeSessionId);
- }
- }
- if (pending.size === 0) {
- this.pendingReplacementSnapshots.delete(client);
- }
- }
- }
-
private handleWorkerFrame(worker: ResidentWorker, frame: PrivateFrame<DaemonWorkerFrameHeader>): void {
if (frame.header.kind !== "outbound") {
return;
}
- const { outboundType, activeSessionId, sessionEventType, payloadEncoding, snapshotPurpose } = frame.header;
+ const {
+ outboundType,
+ activeSessionId,
+ snapshotId: frameSnapshotId,
+ sessionEventType,
+ payloadEncoding,
+ snapshotPurpose,
+ } = frame.header;
if (outboundType === "heartbeats_changed") {
worker.heartbeatSnapshotStale = true;
this.broadcastHeartbeatsChanged();
@@ -2591,119 +2747,127 @@ export class DaemonSupervisor {
begin.type !== "session_snapshot_begin" ||
begin.activeSessionId !== activeSessionId ||
typeof begin.snapshotId !== "string" ||
+ (frameSnapshotId !== undefined && frameSnapshotId !== begin.snapshotId) ||
typeof begin.targetChunkBytes !== "number" ||
!begin.snapshot ||
!isSessionSummary(begin.snapshot.summary)
) {
throw new Error("Worker returned an invalid snapshot begin frame");
}
- let existing = worker.transcriptCaches.get(activeSessionId);
- if (
- worker.incomingTranscriptActiveSessionIds.has(activeSessionId) &&
- existing?.snapshotId === begin.snapshotId
- ) {
+ const publicSummary = this.publicSummary(worker, begin.snapshot.summary);
+ const snapshot = {
+ ...begin.snapshot,
+ summary: publicSummary,
+ messages: [],
+ };
+ const result: DaemonAttachResult = {
+ protocol: DAEMON_PROTOCOL_INFO,
+ activeSessionId,
+ snapshot,
+ replay: {
+ status: "complete",
+ toSequence: snapshot.lastEventSequence,
+ ...(snapshot.lastEventCursor ? { toCursor: snapshot.lastEventCursor } : {}),
+ },
+ lastEventSequence: snapshot.lastEventSequence,
+ ...(snapshot.lastEventCursor ? { lastEventCursor: snapshot.lastEventCursor } : {}),
+ snapshotStream: {
+ id: begin.snapshotId,
+ messageCount: begin.messageCount,
+ targetChunkBytes: begin.targetChunkBytes,
+ },
+ client: { id: "supervisor", capabilities: ["chunked_snapshot"] },
+ };
+ const generations = this.snapshotGenerationsFor(worker, activeSessionId);
+ let generation = generations.get(begin.snapshotId);
+ if (generation?.incoming) {
this.failWorkerSnapshotCache(
worker,
activeSessionId,
new Error(`Snapshot ${begin.snapshotId} restarted before completion`),
true,
+ begin.snapshotId,
);
return;
}
- if (worker.incomingTranscriptActiveSessionIds.has(activeSessionId)) {
- this.settleSnapshotDuplicateValidation(worker, activeSessionId);
- this.failWorkerSnapshotCache(
- worker,
- activeSessionId,
- new Error(`Snapshot ${existing?.snapshotId ?? "unknown"} was superseded`),
- );
- existing = undefined;
- }
- const transferFrames = worker.snapshotTransferFrames.get(activeSessionId);
- const sameSnapshotId = existing?.snapshotId === begin.snapshotId;
const duplicate =
- existing?.complete === true &&
- sameSnapshotId &&
- transferFrames?.end !== undefined &&
- transferFrames.begin.equals(frame.payload);
- if (existing?.complete && sameSnapshotId && !duplicate) {
+ generation?.transcript.complete === true &&
+ generation.end !== undefined &&
+ generation.begin?.equals(frame.payload) === true;
+ if (generation?.transcript.complete && !duplicate) {
this.failWorkerSnapshotCache(
worker,
activeSessionId,
new Error(`Snapshot ${begin.snapshotId} did not match the cached transfer`),
true,
+ begin.snapshotId,
);
return;
}
- if (existing?.complete && !duplicate) {
- this.failWorkerSnapshotCache(
- worker,
- activeSessionId,
- new Error(`Snapshot ${existing.snapshotId} was replaced`),
- );
- existing = undefined;
+ const currentGeneration = this.currentSnapshotGeneration(worker, activeSessionId);
+ const currentResult = currentGeneration?.result ?? worker.snapshotCache.get(activeSessionId);
+ const isOlderThanCurrent =
+ currentGeneration !== undefined &&
+ currentGeneration.transcript.snapshotId !== begin.snapshotId &&
+ currentResult !== undefined &&
+ result.lastEventSequence < currentResult.lastEventSequence;
+ if (isOlderThanCurrent && !generation) {
+ return;
}
- let transcript = existing;
- if (!existing || existing.snapshotId !== begin.snapshotId) {
- if (existing) {
+ if (duplicate && generation) {
+ generation.incoming = true;
+ generation.duplicateChunkIndex = 0;
+ generation.duplicateResult = result;
+ generation.validation = this.createSnapshotDuplicateValidation();
+ if (currentGeneration === generation) {
+ worker.snapshotCache.delete(activeSessionId);
+ }
+ return;
+ }
+ if (
+ currentGeneration &&
+ currentGeneration.transcript.snapshotId !== begin.snapshotId &&
+ !isOlderThanCurrent
+ ) {
+ if (!currentGeneration.transcript.complete && !currentGeneration.incoming) {
this.failWorkerSnapshotCache(
worker,
activeSessionId,
- new Error(`Snapshot ${begin.snapshotId} replaced stale cache data`),
+ new Error(`Snapshot ${currentGeneration.transcript.snapshotId} was superseded`),
+ false,
+ currentGeneration.transcript.snapshotId,
);
+ } else {
+ this.retireWorkerSnapshotCache(worker, activeSessionId, currentGeneration.transcript);
}
- transcript = new SnapshotTranscriptCache({
+ }
+ if (!generation) {
+ const transcript = new SnapshotTranscriptCache({
activeSessionId,
snapshotId: begin.snapshotId,
cacheRoot: this.snapshotCacheRoot,
targetChunkBytes: begin.targetChunkBytes,
});
- worker.transcriptCaches.set(activeSessionId, transcript);
- }
- if (!transcript) {
- throw new Error("Worker snapshot cache could not be initialized");
+ generation = {
+ transcript,
+ result,
+ incoming: false,
+ retired: isOlderThanCurrent,
+ };
+ this.snapshotGenerationsFor(worker, activeSessionId).set(begin.snapshotId, generation);
}
- const publicSummary = this.publicSummary(worker, begin.snapshot.summary);
- const snapshot = {
- ...begin.snapshot,
- summary: publicSummary,
- messages: [],
- };
- const result: DaemonAttachResult = {
- protocol: DAEMON_PROTOCOL_INFO,
- activeSessionId,
- snapshot,
- replay: {
- status: "complete",
- toSequence: snapshot.lastEventSequence,
- ...(snapshot.lastEventCursor ? { toCursor: snapshot.lastEventCursor } : {}),
- },
- lastEventSequence: snapshot.lastEventSequence,
- ...(snapshot.lastEventCursor ? { lastEventCursor: snapshot.lastEventCursor } : {}),
- snapshotStream: {
- id: begin.snapshotId,
- messageCount: begin.messageCount,
- targetChunkBytes: begin.targetChunkBytes,
- },
- client: { id: "supervisor", capabilities: ["chunked_snapshot"] },
- };
- worker.incomingTranscriptActiveSessionIds.add(activeSessionId);
- if (duplicate) {
- worker.duplicateIncomingTranscriptChunkIndexes.set(activeSessionId, 0);
- worker.snapshotTransferFrames.set(activeSessionId, {
- ...transferFrames,
- duplicateResult: result,
- validation: this.createSnapshotDuplicateValidation(),
- });
- worker.snapshotCache.delete(activeSessionId);
- } else {
- worker.duplicateIncomingTranscriptChunkIndexes.delete(activeSessionId);
- worker.snapshotTransferFrames.set(activeSessionId, { begin: Buffer.from(frame.payload) });
+ generation.result = result;
+ generation.begin = Buffer.from(frame.payload);
+ generation.end = undefined;
+ generation.incoming = true;
+ generation.duplicateChunkIndex = undefined;
+ generation.duplicateResult = undefined;
+ generation.validation = undefined;
+ if (!isOlderThanCurrent) {
+ generation.retired = false;
+ worker.transcriptCaches.set(activeSessionId, generation.transcript);
worker.snapshotCache.set(activeSessionId, result);
}
- if (snapshotPurpose === "replacement" && !duplicate) {
- this.streamReplacementSnapshot(worker, activeSessionId, result, transcript);
- }
} catch (error) {
this.log(`Invalid worker snapshot begin frame: ${String(error)}`);
this.failWorkerSnapshotCache(
@@ -2716,12 +2880,16 @@ export class DaemonSupervisor {
return;
}
if (outboundType === "session_snapshot_chunk" && activeSessionId) {
- const transcript = worker.transcriptCaches.get(activeSessionId);
- if (transcript && worker.incomingTranscriptActiveSessionIds.has(activeSessionId)) {
+ const snapshotId = frameSnapshotId ?? worker.transcriptCaches.get(activeSessionId)?.snapshotId;
+ if (!snapshotId) {
+ return;
+ }
+ const generation = this.snapshotGeneration(worker, activeSessionId, snapshotId);
+ if (generation?.incoming) {
try {
- const duplicateIndex = worker.duplicateIncomingTranscriptChunkIndexes.get(activeSessionId);
+ const duplicateIndex = generation.duplicateChunkIndex;
if (duplicateIndex === undefined) {
- transcript.appendEncodedChunk(Buffer.from(frame.payload));
+ generation.transcript.appendEncodedChunk(Buffer.from(frame.payload));
} else {
const chunk = JSON.parse(frame.payload.toString("utf8")) as Extract<
DaemonOutbound,
@@ -2730,13 +2898,15 @@ export class DaemonSupervisor {
if (
chunk.type !== "session_snapshot_chunk" ||
chunk.activeSessionId !== activeSessionId ||
- chunk.snapshotId !== transcript.snapshotId ||
+ chunk.snapshotId !== generation.transcript.snapshotId ||
chunk.index !== duplicateIndex ||
- !transcript.readChunk(duplicateIndex).equals(Buffer.from(frame.payload))
+ !generation.transcript.readChunk(duplicateIndex).equals(Buffer.from(frame.payload))
) {
- throw new Error(`Duplicate snapshot ${transcript.snapshotId} did not match cached bytes`);
+ throw new Error(
+ `Duplicate snapshot ${generation.transcript.snapshotId} did not match cached bytes`,
+ );
}
- worker.duplicateIncomingTranscriptChunkIndexes.set(activeSessionId, duplicateIndex + 1);
+ generation.duplicateChunkIndex = duplicateIndex + 1;
}
} catch (error) {
this.failWorkerSnapshotCache(
@@ -2744,31 +2914,31 @@ export class DaemonSupervisor {
activeSessionId,
error instanceof Error ? error : new Error(String(error)),
true,
+ generation.transcript.snapshotId,
);
}
}
return;
}
if (outboundType === "session_snapshot_end" && activeSessionId) {
- const transcript = worker.transcriptCaches.get(activeSessionId);
- if (!transcript || !worker.incomingTranscriptActiveSessionIds.has(activeSessionId)) {
+ const snapshotId = frameSnapshotId ?? worker.transcriptCaches.get(activeSessionId)?.snapshotId;
+ if (!snapshotId) {
+ return;
+ }
+ const generation = this.snapshotGeneration(worker, activeSessionId, snapshotId);
+ if (!generation?.incoming) {
return;
}
- let completedDuplicate = false;
+ const transcript = generation.transcript;
try {
- const duplicateChunkCount = worker.duplicateIncomingTranscriptChunkIndexes.get(activeSessionId);
+ const duplicateChunkCount = generation.duplicateChunkIndex;
if (duplicateChunkCount === undefined) {
transcript.markComplete();
- const begin = worker.snapshotTransferFrames.get(activeSessionId)?.begin;
- if (!begin) {
+ if (!generation.begin) {
throw new Error(`Snapshot ${transcript.snapshotId} has no begin frame`);
}
- worker.snapshotTransferFrames.set(activeSessionId, {
- begin,
- end: Buffer.from(frame.payload),
- });
+ generation.end = Buffer.from(frame.payload);
} else {
- const transferFrames = worker.snapshotTransferFrames.get(activeSessionId);
const end = JSON.parse(frame.payload.toString("utf8")) as Extract<
DaemonOutbound,
{ type: "session_snapshot_end" }
@@ -2779,47 +2949,40 @@ export class DaemonSupervisor {
end.snapshotId !== transcript.snapshotId ||
end.chunkCount !== duplicateChunkCount ||
end.chunkCount !== transcript.chunkCount ||
- !transferFrames?.end?.equals(frame.payload)
+ !generation.end?.equals(frame.payload)
) {
throw new Error(`Duplicate snapshot ${transcript.snapshotId} ended with different metadata`);
}
- if (!transferFrames.duplicateResult) {
+ if (!generation.duplicateResult) {
throw new Error(`Duplicate snapshot ${transcript.snapshotId} has no result`);
}
- worker.snapshotCache.set(activeSessionId, transferFrames.duplicateResult);
- this.settleSnapshotDuplicateValidation(worker, activeSessionId);
- worker.snapshotTransferFrames.set(activeSessionId, {
- begin: transferFrames.begin,
- end: transferFrames.end,
- });
- completedDuplicate = true;
+ generation.result = generation.duplicateResult;
+ if (worker.transcriptCaches.get(activeSessionId) === transcript) {
+ worker.snapshotCache.set(activeSessionId, generation.duplicateResult);
+ }
+ this.settleSnapshotDuplicateValidation(generation);
}
- worker.incomingTranscriptActiveSessionIds.delete(activeSessionId);
- worker.duplicateIncomingTranscriptChunkIndexes.delete(activeSessionId);
+ generation.incoming = false;
+ generation.duplicateChunkIndex = undefined;
+ generation.duplicateResult = undefined;
} catch (error) {
this.failWorkerSnapshotCache(
worker,
activeSessionId,
error instanceof Error ? error : new Error(String(error)),
true,
+ transcript.snapshotId,
);
return;
}
- if (snapshotPurpose === "replacement" && completedDuplicate) {
- const result = worker.snapshotCache.get(activeSessionId);
- if (result) {
- this.streamReplacementSnapshot(worker, activeSessionId, result, transcript);
- }
- }
- if (snapshotPurpose === "replacement") {
- for (const client of this.clients) {
- this.flushPendingReplacementSnapshot(client, activeSessionId);
- }
+ const published = worker.transcriptCaches.get(activeSessionId) === transcript;
+ if (generation.retired) {
+ this.deleteSnapshotGeneration(worker, activeSessionId, generation);
+ transcript.dispose();
}
- if (snapshotPurpose === "replacement" || snapshotPurpose === "catchup") {
+ if (published && (snapshotPurpose === "replacement" || snapshotPurpose === "catchup")) {
for (const client of this.clients) {
if (!client.attachedActiveSessionIds.has(activeSessionId)) continue;
- if (snapshotPurpose === "replacement" && client.capabilities.has("chunked_snapshot")) continue;
this.queueCatchup(client, activeSessionId, snapshotPurpose === "replacement" ? "replacement" : "resync");
void this.catchUpClient(client).catch((error) =>
this.log(`Failed to catch up client ${client.id}: ${String(error)}`),
@@ -2834,23 +2997,27 @@ export class DaemonSupervisor {
DaemonOutbound,
{ type: "session_snapshot_failed" }
>;
- const transcript = worker.transcriptCaches.get(activeSessionId);
- const cachedResult = worker.snapshotCache.get(activeSessionId);
- const expectedSnapshotId = transcript?.snapshotId ?? cachedResult?.snapshotStream?.id;
if (
failed.type !== "session_snapshot_failed" ||
failed.activeSessionId !== activeSessionId ||
typeof failed.snapshotId !== "string" ||
typeof failed.error !== "string" ||
- expectedSnapshotId !== failed.snapshotId
+ (frameSnapshotId !== undefined && frameSnapshotId !== failed.snapshotId)
) {
throw new Error("Worker returned an invalid snapshot failure frame");
}
- this.failWorkerSnapshotCache(worker, activeSessionId, new Error(failed.error));
- if (snapshotPurpose === "replacement" || snapshotPurpose === "catchup") {
+ const currentGeneration = this.currentSnapshotGeneration(worker, activeSessionId);
+ const generation =
+ this.snapshotGeneration(worker, activeSessionId, failed.snapshotId) ??
+ (currentGeneration?.transcript.snapshotId === failed.snapshotId ? currentGeneration : undefined);
+ if (!generation) {
+ return;
+ }
+ const published = worker.transcriptCaches.get(activeSessionId) === generation.transcript;
+ this.failWorkerSnapshotCache(worker, activeSessionId, new Error(failed.error), false, failed.snapshotId);
+ if (published && (snapshotPurpose === "replacement" || snapshotPurpose === "catchup")) {
for (const client of this.clients) {
if (!client.attachedActiveSessionIds.has(activeSessionId)) continue;
- if (snapshotPurpose === "replacement" && client.capabilities.has("chunked_snapshot")) continue;
this.queueCatchup(
client,
activeSessionId,
@@ -2980,17 +3147,8 @@ export class DaemonSupervisor {
}
const transcript = worker.transcriptCaches.get(activeSessionId);
if (transcript) {
- transcript.dispose();
- worker.transcriptCaches.delete(activeSessionId);
+ this.retireWorkerSnapshotCache(worker, activeSessionId, transcript);
}
- worker.incomingTranscriptActiveSessionIds.delete(activeSessionId);
- worker.duplicateIncomingTranscriptChunkIndexes.delete(activeSessionId);
- this.settleSnapshotDuplicateValidation(
- worker,
- activeSessionId,
- new Error(`Snapshot ${activeSessionId} was invalidated`),
- );
- worker.snapshotTransferFrames.delete(activeSessionId);
}
private scheduleCompactCatchup(worker: ResidentWorker, activeSessionId: string): void {
@@ -3025,11 +3183,37 @@ export class DaemonSupervisor {
}
}
- private async catchUpClient(client: DaemonSocketClient): Promise<void> {
+ private catchUpClient(client: DaemonSocketClient): Promise<void> {
+ if (client.catchupPromise) {
+ return client.catchupPromise;
+ }
+ if (client.snapshotStreaming || client.backpressured) {
+ return Promise.resolve();
+ }
+ const catchup = this.drainClientCatchupQueue(client).finally(() => {
+ if (client.catchupPromise === catchup) {
+ client.catchupPromise = undefined;
+ }
+ });
+ client.catchupPromise = catchup;
+ return catchup;
+ }
+
+ private async drainClientCatchupQueue(client: DaemonSocketClient): Promise<void> {
+ while (
+ !client.socket.destroyed &&
+ !client.snapshotStreaming &&
+ !client.backpressured &&
+ client.catchupActiveSessionIds?.size
+ ) {
+ await this.drainClientCatchups(client);
+ }
+ }
+
+ private async drainClientCatchups(client: DaemonSocketClient): Promise<void> {
if (client.socket.destroyed) {
return;
}
- client.backpressured = false;
const pending = [...(client.catchupActiveSessionIds ?? [])].map((activeSessionId) => ({
activeSessionId,
purpose: client.catchupPurposes?.get(activeSessionId) ?? ("resync" as const),
@@ -3038,6 +3222,7 @@ export class DaemonSupervisor {
client.catchupPurposes?.clear();
for (let index = 0; index < pending.length; index++) {
const { activeSessionId, purpose } = pending[index]!;
+ let releaseTranscript: (() => void) | undefined;
try {
const attached = await this.attachClient(client, {
type: "attach",
@@ -3045,8 +3230,12 @@ export class DaemonSupervisor {
capabilities: [...client.capabilities],
supportsExtensionUi: client.supportsExtensionUi,
});
+ releaseTranscript = attached.releaseTranscript;
if (client.capabilities.has("chunked_snapshot")) {
- const transcript = this.getOrCreateTranscriptCache(attached.worker, attached.result);
+ const transcript = attached.transcript;
+ if (!transcript) {
+ throw new Error("Session worker did not provide a snapshot transcript");
+ }
if (purpose === "replacement") {
this.write(client, {
type: "session_replaced",
@@ -3068,7 +3257,9 @@ export class DaemonSupervisor {
this.createStreamedAttachResult(attached.result, transcript),
transcript,
purpose,
+ releaseTranscript,
);
+ releaseTranscript = undefined;
continue;
}
const meta = createDaemonEventMeta(
@@ -3099,6 +3290,7 @@ export class DaemonSupervisor {
return;
}
} catch (error) {
+ releaseTranscript?.();
this.log(`Failed to catch up client ${client.id} for ${activeSessionId}: ${String(error)}`);
}
}
@@ -3227,32 +3419,28 @@ export class DaemonSupervisor {
}
this.reportCleanupFailure(`worker rollback state ${worker.descriptor.workerId}`, error);
}
- this.dropPendingReplacementSnapshotsForWorker(worker);
- const interruptedSnapshots = new Set([
- ...worker.incomingTranscriptActiveSessionIds,
- ...worker.duplicateIncomingTranscriptChunkIndexes.keys(),
- ...[...worker.transcriptCaches]
- .filter(([, transcript]) => !transcript.complete)
- .map(([activeSessionId]) => activeSessionId),
- ...[...worker.snapshotTransferFrames]
- .filter(([, transfer]) => transfer.validation !== undefined)
- .map(([activeSessionId]) => activeSessionId),
- ]);
- for (const activeSessionId of interruptedSnapshots) {
- this.failWorkerSnapshotCache(
- worker,
- activeSessionId,
- new Error("Session worker stopped during snapshot transfer"),
- );
+ const transferError = new Error("Session worker stopped during snapshot transfer");
+ const generationTranscripts = new Set<SnapshotTranscriptCache>();
+ for (const [activeSessionId, generations] of [...(worker.snapshotGenerations ?? new Map())]) {
+ for (const generation of [...generations.values()]) {
+ generationTranscripts.add(generation.transcript);
+ if (generation.incoming || !generation.transcript.complete || generation.validation) {
+ this.failSnapshotGeneration(worker, activeSessionId, generation, transferError);
+ } else {
+ generation.transcript.dispose();
+ this.deleteSnapshotGeneration(worker, activeSessionId, generation);
+ }
+ }
}
for (const transcript of worker.transcriptCaches.values()) {
+ if (!generationTranscripts.has(transcript) && !transcript.complete) {
+ transcript.markFailed(transferError);
+ }
transcript.dispose();
}
worker.transcriptCaches.clear();
worker.snapshotCache.clear();
- worker.incomingTranscriptActiveSessionIds.clear();
- worker.duplicateIncomingTranscriptChunkIndexes.clear();
- worker.snapshotTransferFrames.clear();
+ worker.snapshotGenerations?.clear();
if (worker.client) {
if (archiveSession) {
await worker.client
@@ -3438,11 +3626,24 @@ export class DaemonSupervisor {
for (const worker of this.workers.values()) {
await this.runCleanupStep(`worker client ${worker.descriptor.workerId}`, () => worker.client?.close());
worker.client = undefined;
- for (const transcript of worker.transcriptCaches.values()) {
+ const transcripts = new Set(worker.transcriptCaches.values());
+ for (const generations of worker.snapshotGenerations?.values() ?? []) {
+ for (const generation of generations.values()) {
+ transcripts.add(generation.transcript);
+ this.settleSnapshotDuplicateValidation(
+ generation,
+ new Error("Daemon supervisor stopped during snapshot transfer"),
+ );
+ if (!generation.transcript.complete) {
+ generation.transcript.markFailed(new Error("Daemon supervisor stopped during snapshot transfer"));
+ }
+ }
+ }
+ for (const transcript of transcripts) {
await this.runCleanupStep(`worker transcript ${worker.descriptor.workerId}`, () => transcript.dispose());
}
worker.transcriptCaches.clear();
- worker.incomingTranscriptActiveSessionIds.clear();
+ worker.snapshotGenerations?.clear();
worker.snapshotCache.clear();
worker.snapshotLoads.clear();
}
@@ -3508,7 +3709,6 @@ export class DaemonSupervisor {
}
await this.catalog.stop();
for (const client of this.clients) {
- this.dropPendingReplacementSnapshot(client);
client.detachInput();
client.socket.end();
}
diff --git a/packages/coding-agent/src/modes/daemon/daemon-worker-protocol.ts b/packages/coding-agent/src/modes/daemon/daemon-worker-protocol.ts
index 0ae2146ddf..29b55d141e 100644
--- a/packages/coding-agent/src/modes/daemon/daemon-worker-protocol.ts
+++ b/packages/coding-agent/src/modes/daemon/daemon-worker-protocol.ts
@@ -29,6 +29,7 @@ export type DaemonWorkerFrameHeader =
requestId?: string;
outboundType: DaemonOutbound["type"];
activeSessionId?: string;
+ snapshotId?: string;
sessionEventType?: string;
payloadEncoding?: "jsonl" | "assistant-delta";
snapshotPurpose?: "attach" | "replacement" | "catchup";
@@ -146,6 +147,7 @@ export function isDaemonWorkerFrameHeader(value: unknown): value is DaemonWorker
typeof candidate.outboundType === "string" &&
(candidate.requestId === undefined || typeof candidate.requestId === "string") &&
(candidate.activeSessionId === undefined || typeof candidate.activeSessionId === "string") &&
+ (candidate.snapshotId === undefined || typeof candidate.snapshotId === "string") &&
(candidate.sessionEventType === undefined || typeof candidate.sessionEventType === "string") &&
(candidate.snapshotPurpose === undefined ||
candidate.snapshotPurpose === "attach" ||
diff --git a/packages/coding-agent/src/modes/interactive/interactive-mode.ts b/packages/coding-agent/src/modes/interactive/interactive-mode.ts
index df0b74f879..02b35c0d9e 100644
--- a/packages/coding-agent/src/modes/interactive/interactive-mode.ts
+++ b/packages/coding-agent/src/modes/interactive/interactive-mode.ts
@@ -1483,7 +1483,7 @@ export class InteractiveMode {
// the text was restored to the editor by onboarding, history, or a retry.
const images = this.collectImagesFor(userInput);
try {
- await this.agentConnection.prompt(userInput, { images });
+ await this.agentConnection.prompt(userInput, { streamingBehavior: "steer", images });
} catch (error: unknown) {
const errorMessage = error instanceof Error ? error.message : "Unknown error occurred";
this.showError(errorMessage);
diff --git a/packages/coding-agent/test/interactive-mode-status.test.ts b/packages/coding-agent/test/interactive-mode-status.test.ts
index 76ce244ef9..b6a5b6e77e 100644
--- a/packages/coding-agent/test/interactive-mode-status.test.ts
+++ b/packages/coding-agent/test/interactive-mode-status.test.ts
@@ -1935,7 +1935,7 @@ describe("InteractiveMode Prime CLI onboarding", () => {
await expect(InteractiveMode.prototype.run.call(fakeThis as never)).resolves.toBe("agents_view");
expect(prompt).toHaveBeenNthCalledWith(1, "initial prompt", { images: undefined });
- expect(prompt).toHaveBeenNthCalledWith(2, "interactive prompt", { images: [] });
+ expect(prompt).toHaveBeenNthCalledWith(2, "interactive prompt", { streamingBehavior: "steer", images: [] });
expect(prompt).toHaveBeenCalledTimes(2);
});
diff --git a/packages/coding-agent/test/suite/regressions/4601-worker-snapshot-cache.test.ts b/packages/coding-agent/test/suite/regressions/4601-worker-snapshot-cache.test.ts
index e94ed1c256..038bce7142 100644
--- a/packages/coding-agent/test/suite/regressions/4601-worker-snapshot-cache.test.ts
+++ b/packages/coding-agent/test/suite/regressions/4601-worker-snapshot-cache.test.ts
@@ -12,7 +12,6 @@ import {
type DaemonAttachResult,
type DaemonOutbound,
} from "../../../src/modes/daemon/daemon-protocol.js";
-import { DaemonSupervisor } from "../../../src/modes/daemon/daemon-supervisor.js";
import {
createSnapshotTranscriptChunks,
SnapshotTranscriptCache,
@@ -444,112 +443,6 @@ describe("ENG-4601 worker snapshot cache", () => {
}
});
- it("keeps supervisor snapshot state active until overlapping same-session streams finish", async () => {
- const root = tempDirectory();
- const supervisor = new DaemonSupervisor(join(root, "supervisor.sock"), {
- defaultSessionConfig: { agentDir: root, cwd: root },
- descriptorDir: join(root, "state"),
- });
- const { client, socket } = socketClient("supervisor-overlap");
- const firstTranscript = new SnapshotTranscriptCache({
- activeSessionId,
- snapshotId: "snapshot-first",
- messages: messages("first"),
- cacheRoot: root,
- });
- const secondTranscript = new SnapshotTranscriptCache({
- activeSessionId,
- snapshotId: "snapshot-second",
- messages: messages("second"),
- cacheRoot: root,
- });
- const replacementTranscript = new SnapshotTranscriptCache({
- activeSessionId,
- snapshotId: "snapshot-replacement",
- messages: messages("replacement"),
- cacheRoot: root,
- });
- const supervisorWorker = {
- transcriptCaches: new Map([[activeSessionId, replacementTranscript]]),
- };
- const firstResult = {
- ...streamedResult(2),
- snapshotStream: { id: firstTranscript.snapshotId, messageCount: 2, targetChunkBytes: 1 },
- };
- const secondResult = {
- ...streamedResult(2),
- snapshotStream: { id: secondTranscript.snapshotId, messageCount: 2, targetChunkBytes: 1 },
- };
- const replacementResult = {
- ...streamedResult(2),
- snapshotStream: { id: replacementTranscript.snapshotId, messageCount: 2, targetChunkBytes: 1 },
- };
- const written: DaemonOutbound[] = [];
- let releaseSecondBegin!: (accepted: boolean) => void;
- const secondBegin = new Promise<boolean>((resolve) => {
- releaseSecondBegin = resolve;
- });
- const writeSnapshotBuffer = vi.fn((_client: DaemonSocketClient, buffer: Uint8Array) => {
- const message = JSON.parse(Buffer.from(buffer).toString("utf8")) as DaemonOutbound;
- written.push(message);
- if (message.type === "session_snapshot_begin" && message.snapshotId === secondTranscript.snapshotId) {
- return secondBegin;
- }
- return Promise.resolve(true);
- });
- const internals = supervisor as unknown as {
- clients: Set<DaemonSocketClient>;
- pendingReplacementSnapshots: WeakMap<DaemonSocketClient, Map<string, unknown>>;
- writeSnapshotBuffer: typeof writeSnapshotBuffer;
- streamSnapshot(
- client: DaemonSocketClient,
- worker: typeof supervisorWorker,
- result: DaemonAttachResult,
- transcript: SnapshotTranscriptCache,
- purpose: "attach" | "replacement" | "resync",
- ): Promise<void>;
- streamReplacementSnapshot(
- worker: typeof supervisorWorker,
- activeSessionId: string,
- result: DaemonAttachResult,
- transcript: SnapshotTranscriptCache,
- ): void;
- };
- internals.clients.add(client);
- internals.writeSnapshotBuffer = writeSnapshotBuffer;
-
- const firstStream = internals.streamSnapshot(client, supervisorWorker, firstResult, firstTranscript, "attach");
- const secondStream = internals.streamSnapshot(client, supervisorWorker, secondResult, secondTranscript, "attach");
- internals.streamReplacementSnapshot(supervisorWorker, activeSessionId, replacementResult, replacementTranscript);
-
- await firstStream;
-
- expect(client.snapshotActiveSessionCounts?.get(activeSessionId)).toBe(1);
- expect(client.snapshotActiveSessionIds?.has(activeSessionId)).toBe(true);
- expect(client.snapshotStreaming).toBe(true);
- expect(internals.pendingReplacementSnapshots.get(client)?.size).toBe(1);
- expect(
- written.some((message) => message.type === "session_snapshot_begin" && message.purpose === "replacement"),
- ).toBe(false);
-
- releaseSecondBegin(true);
- await secondStream;
- await new Promise<void>((resolve) => setImmediate(resolve));
-
- const secondEndIndex = written.findIndex(
- (message) => message.type === "session_snapshot_end" && message.snapshotId === secondTranscript.snapshotId,
- );
- const replacementBeginIndex = written.findIndex(
- (message) => message.type === "session_snapshot_begin" && message.purpose === "replacement",
- );
- expect(replacementBeginIndex).toBeGreaterThan(secondEndIndex);
- expect(client.snapshotActiveSessionCounts?.size).toBe(0);
- expect(client.snapshotActiveSessionIds?.size).toBe(0);
- expect(client.snapshotStreaming).toBe(false);
- expect(internals.pendingReplacementSnapshots.get(client)?.size ?? 0).toBe(0);
- socket.destroy();
- });
-
it("leaves the supervisor transcript cache file-backed and reader-owned", () => {
const cache = new SnapshotTranscriptCache({
activeSessionId,
diff --git a/packages/coding-agent/test/suite/regressions/4602-snapshot-transfer-idempotency.test.ts b/packages/coding-agent/test/suite/regressions/4602-snapshot-transfer-idempotency.test.ts
index eaec8fbc06..d706f5e1fe 100644
--- a/packages/coding-agent/test/suite/regressions/4602-snapshot-transfer-idempotency.test.ts
+++ b/packages/coding-agent/test/suite/regressions/4602-snapshot-transfer-idempotency.test.ts
@@ -28,16 +28,18 @@ interface WorkerHarness {
summaries: Map<string, SessionSummary>;
snapshotCache: Map<string, DaemonAttachResult>;
transcriptCaches: Map<string, SnapshotTranscriptCache>;
- incomingTranscriptActiveSessionIds: Set<string>;
- duplicateIncomingTranscriptChunkIndexes: Map<string, number>;
- snapshotTransferFrames: Map<
+ snapshotGenerations: Map<
string,
- {
- begin: Buffer;
- end?: Buffer;
- duplicateResult?: DaemonAttachResult;
- validation?: { promise: Promise<void>; resolve: () => void; reject: (error: Error) => void };
- }
+ Map<
+ string,
+ {
+ transcript: SnapshotTranscriptCache;
+ result: DaemonAttachResult;
+ incoming: boolean;
+ retired: boolean;
+ validation?: { promise: Promise<void>; resolve: () => void; reject: (error: Error) => void };
+ }
+ >
>;
snapshotLoads: Map<string, Promise<DaemonAttachResult>>;
intentionalStop: boolean;
@@ -87,6 +89,9 @@ function frame(
kind: "outbound",
outboundType: message.type,
...("activeSessionId" in message ? { activeSessionId: message.activeSessionId } : {}),
+ ...("snapshotId" in message && typeof message.snapshotId === "string"
+ ? { snapshotId: message.snapshotId }
+ : {}),
payloadEncoding: "jsonl",
...(snapshotPurpose ? { snapshotPurpose } : {}),
},
@@ -108,9 +113,7 @@ function workerHarness() {
summaries: new Map([[activeSessionId, summary()]]),
snapshotCache: new Map(),
transcriptCaches: new Map<string, SnapshotTranscriptCache>(),
- incomingTranscriptActiveSessionIds: new Set<string>(),
- duplicateIncomingTranscriptChunkIndexes: new Map<string, number>(),
- snapshotTransferFrames: new Map(),
+ snapshotGenerations: new Map(),
snapshotLoads: new Map(),
intentionalStop: false,
stopRevision: 0,
@@ -300,7 +303,6 @@ describe("ENG-4602 snapshot transfer containment", () => {
});
worker.snapshotCache.set(activeSessionId, streamedResult([]));
worker.transcriptCaches.set(activeSessionId, transcript);
- worker.incomingTranscriptActiveSessionIds.add(activeSessionId);
const internals = supervisor as unknown as {
handleWorkerFrame(worker: WorkerHarness, frame: PrivateFrame<DaemonWorkerFrameHeader>): void;
};
@@ -320,10 +322,10 @@ describe("ENG-4602 snapshot transfer containment", () => {
expect(worker.summaries.has("active-4602-sibling")).toBe(true);
expect(worker.snapshotCache.has(activeSessionId)).toBe(false);
expect(worker.transcriptCaches.has(activeSessionId)).toBe(false);
- expect(worker.incomingTranscriptActiveSessionIds.has(activeSessionId)).toBe(false);
+ expect(worker.snapshotGenerations.has(activeSessionId)).toBe(false);
});
- it("quarantines a completed duplicate until exact replacement validation", () => {
+ it("quarantines a completed duplicate until exact replacement validation", async () => {
const supervisor = new DaemonSupervisor("/tmp/eng-4602-supervisor.sock", {
defaultSessionConfig: { agentDir: "/tmp", cwd: "/tmp" },
descriptorDir: "/tmp/eng-4602-supervisor-state",
@@ -334,10 +336,14 @@ describe("ENG-4602 snapshot transfer containment", () => {
const streamSnapshot = vi.fn(async () => {});
const internals = supervisor as unknown as {
clients: Set<DaemonSocketClient>;
+ workers: Map<string, WorkerHarness>;
+ syncWorkerExtensionUi: ReturnType<typeof vi.fn>;
streamSnapshot: typeof streamSnapshot;
handleWorkerFrame(worker: WorkerHarness, frame: PrivateFrame<DaemonWorkerFrameHeader>): void;
};
internals.clients.add(client);
+ internals.workers.set(worker.descriptor.workerId, worker);
+ internals.syncWorkerExtensionUi = vi.fn(async () => {});
internals.streamSnapshot = streamSnapshot;
const messages: AgentMessage[] = [{ role: "user", content: "stable", timestamp: 1 }];
const frames = snapshotFrames(messages);
@@ -356,6 +362,7 @@ describe("ENG-4602 snapshot transfer containment", () => {
expect(streamSnapshot).not.toHaveBeenCalled();
internals.handleWorkerFrame(worker, frame(frames.end, "replacement"));
+ await new Promise<void>((resolve) => setImmediate(resolve));
expect(worker.snapshotCache.has(activeSessionId)).toBe(true);
expect(streamSnapshot).toHaveBeenCalledOnce();
expect(close).not.toHaveBeenCalled();
@@ -379,170 +386,6 @@ describe("ENG-4602 snapshot transfer containment", () => {
expect(close).toHaveBeenCalledOnce();
});
- it("queues a replacement until the active client snapshot finishes", async () => {
- const supervisor = new DaemonSupervisor("/tmp/eng-4602-supervisor-ordered-replacement.sock", {
- defaultSessionConfig: { agentDir: "/tmp", cwd: "/tmp" },
- descriptorDir: "/tmp/eng-4602-supervisor-ordered-replacement-state",
- });
- const { worker } = workerHarness();
- const client = socketClient("public", new PassThrough());
- const messages: AgentMessage[] = [{ role: "user", content: "stable", timestamp: 1 }];
- const result = streamedResult(messages);
- const transcript = new SnapshotTranscriptCache({
- activeSessionId,
- snapshotId,
- messages,
- cacheRoot: "/tmp",
- });
- worker.transcriptCaches.set(activeSessionId, transcript);
- const written: DaemonOutbound[] = [];
- let releaseFirstWrite!: (accepted: boolean) => void;
- const firstWrite = new Promise<boolean>((resolve) => {
- releaseFirstWrite = resolve;
- });
- const writeSnapshotBuffer = vi.fn((_client: DaemonSocketClient, buffer: Uint8Array) => {
- written.push(JSON.parse(Buffer.from(buffer).toString("utf8")) as DaemonOutbound);
- return written.length === 1 ? firstWrite : Promise.resolve(true);
- });
- const internals = supervisor as unknown as {
- clients: Set<DaemonSocketClient>;
- pendingReplacementSnapshots: WeakMap<DaemonSocketClient, Map<string, unknown>>;
- writeSnapshotBuffer: typeof writeSnapshotBuffer;
- streamSnapshot(
- client: DaemonSocketClient,
- worker: WorkerHarness,
- result: DaemonAttachResult,
- transcript: SnapshotTranscriptCache,
- purpose: "attach" | "replacement" | "resync",
- ): Promise<void>;
- streamReplacementSnapshot(
- worker: WorkerHarness,
- activeSessionId: string,
- result: DaemonAttachResult,
- transcript: SnapshotTranscriptCache,
- ): void;
- };
- internals.clients.add(client);
- internals.writeSnapshotBuffer = writeSnapshotBuffer;
-
- const activeStream = internals.streamSnapshot(client, worker, result, transcript, "resync");
- await Promise.resolve();
- internals.streamReplacementSnapshot(worker, activeSessionId, result, transcript);
- await Promise.resolve();
-
- expect(written).toHaveLength(1);
- expect(written[0]).toMatchObject({ type: "session_snapshot_begin", purpose: "resync" });
- expect(client.snapshotActiveSessionIds?.has(activeSessionId)).toBe(true);
-
- releaseFirstWrite(true);
- await activeStream;
- await new Promise<void>((resolve) => setImmediate(resolve));
-
- expect(written.map((message) => message.type)).toEqual([
- "session_snapshot_begin",
- "session_snapshot_chunk",
- "session_snapshot_end",
- "session_snapshot_begin",
- "session_snapshot_chunk",
- "session_snapshot_end",
- ]);
- expect(written.filter((message) => message.type === "session_snapshot_begin")).toMatchObject([
- { purpose: "resync" },
- { purpose: "replacement" },
- ]);
- expect(client.snapshotActiveSessionIds?.has(activeSessionId)).toBe(false);
- expect(client.snapshotStreaming).toBe(false);
- expect(internals.pendingReplacementSnapshots.get(client)?.size ?? 0).toBe(0);
- });
-
- it("retains a queued replacement until its worker transcript completes", async () => {
- const supervisor = new DaemonSupervisor("/tmp/eng-4602-supervisor-pending-replacement.sock", {
- defaultSessionConfig: { agentDir: "/tmp", cwd: "/tmp" },
- descriptorDir: "/tmp/eng-4602-supervisor-pending-replacement-state",
- });
- const { worker } = workerHarness();
- const client = socketClient("public", new PassThrough());
- const firstMessages: AgentMessage[] = [{ role: "user", content: "first", timestamp: 1 }];
- const firstResult = streamedResult(firstMessages);
- const firstTranscript = new SnapshotTranscriptCache({
- activeSessionId,
- snapshotId,
- messages: firstMessages,
- cacheRoot: "/tmp",
- });
- worker.transcriptCaches.set(activeSessionId, firstTranscript);
- const written: DaemonOutbound[] = [];
- let releaseActiveEnd!: (accepted: boolean) => void;
- const activeEnd = new Promise<boolean>((resolve) => {
- releaseActiveEnd = resolve;
- });
- let markActiveEndStarted!: () => void;
- const activeEndStarted = new Promise<void>((resolve) => {
- markActiveEndStarted = resolve;
- });
- const writeSnapshotBuffer = vi.fn((_client: DaemonSocketClient, buffer: Uint8Array) => {
- const message = JSON.parse(Buffer.from(buffer).toString("utf8")) as DaemonOutbound;
- written.push(message);
- if (message.type === "session_snapshot_end" && written.length === 3) {
- markActiveEndStarted();
- return activeEnd;
- }
- return Promise.resolve(true);
- });
- const internals = supervisor as unknown as {
- clients: Set<DaemonSocketClient>;
- pendingReplacementSnapshots: WeakMap<DaemonSocketClient, Map<string, unknown>>;
- writeSnapshotBuffer: typeof writeSnapshotBuffer;
- streamSnapshot(
- client: DaemonSocketClient,
- worker: WorkerHarness,
- result: DaemonAttachResult,
- transcript: SnapshotTranscriptCache,
- purpose: "attach" | "replacement" | "resync",
- ): Promise<void>;
- handleWorkerFrame(worker: WorkerHarness, frame: PrivateFrame<DaemonWorkerFrameHeader>): void;
- };
- internals.clients.add(client);
- internals.writeSnapshotBuffer = writeSnapshotBuffer;
-
- const activeStream = internals.streamSnapshot(client, worker, firstResult, firstTranscript, "resync");
- await activeEndStarted;
- const frames = snapshotFrames([{ role: "user", content: "replacement", timestamp: 2 }]);
- const replacementSnapshotId = "snapshot-4602-replacement";
- const replacementBegin = { ...frames.begin, snapshotId: replacementSnapshotId };
- const replacementChunk = { ...frames.chunk, snapshotId: replacementSnapshotId };
- const replacementEnd = { ...frames.end, snapshotId: replacementSnapshotId };
- internals.handleWorkerFrame(worker, frame(replacementBegin, "replacement"));
-
- expect(worker.transcriptCaches.get(activeSessionId)?.complete).toBe(false);
- expect(internals.pendingReplacementSnapshots.get(client)?.size).toBe(1);
-
- releaseActiveEnd(true);
- await activeStream;
- await new Promise<void>((resolve) => setImmediate(resolve));
-
- expect(written.filter((message) => message.type === "session_snapshot_begin")).toHaveLength(1);
- expect(internals.pendingReplacementSnapshots.get(client)?.size).toBe(1);
-
- internals.handleWorkerFrame(worker, frame(replacementChunk, "replacement"));
- internals.handleWorkerFrame(worker, frame(replacementEnd, "replacement"));
- await new Promise<void>((resolve) => setImmediate(resolve));
-
- expect(written.map((message) => message.type)).toEqual([
- "session_snapshot_begin",
- "session_snapshot_chunk",
- "session_snapshot_end",
- "session_snapshot_begin",
- "session_snapshot_chunk",
- "session_snapshot_end",
- ]);
- expect(written.filter((message) => message.type === "session_snapshot_begin")).toMatchObject([
- { purpose: "resync", snapshotId },
- { purpose: "replacement", snapshotId: replacementSnapshotId },
- ]);
- expect(internals.pendingReplacementSnapshots.get(client)?.size ?? 0).toBe(0);
- });
-
it("holds catch-up behind duplicate validation and rejects it on mismatch", async () => {
const supervisor = new DaemonSupervisor("/tmp/eng-4602-supervisor-gate.sock", {
defaultSessionConfig: { agentDir: "/tmp", cwd: "/tmp" },
@@ -635,7 +478,7 @@ describe("ENG-4602 snapshot transfer containment", () => {
for (const message of [frames.begin, frames.chunk, frames.end, frames.begin]) {
internals.handleWorkerFrame(worker, frame(message));
}
- const validation = worker.snapshotTransferFrames.get(activeSessionId)?.validation?.promise;
+ const validation = worker.snapshotGenerations.get(activeSessionId)?.get(snapshotId)?.validation?.promise;
if (!validation) {
throw new Error("duplicate validation was not created");
}
@@ -664,9 +507,7 @@ describe("ENG-4602 snapshot transfer containment", () => {
expect(close).toHaveBeenCalledOnce();
expect(worker.snapshotCache.size).toBe(0);
expect(worker.transcriptCaches.size).toBe(0);
- expect(worker.incomingTranscriptActiveSessionIds.size).toBe(0);
- expect(worker.duplicateIncomingTranscriptChunkIndexes.size).toBe(0);
- expect(worker.snapshotTransferFrames.size).toBe(0);
+ expect(worker.snapshotGenerations.size).toBe(0);
expect(unhandled).not.toHaveBeenCalled();
});
@@ -700,7 +541,7 @@ describe("ENG-4602 snapshot transfer containment", () => {
release();
expect(worker.transcriptCaches.size).toBe(0);
expect(worker.snapshotCache.size).toBe(0);
- expect(worker.snapshotTransferFrames.size).toBe(0);
+ expect(worker.snapshotGenerations.size).toBe(0);
});
it("rejects same-ID reentrant begins and mismatched duplicate metadata", async () => {
diff --git a/packages/coding-agent/test/suite/regressions/4677-snapshot-catchup-replacement.test.ts b/packages/coding-agent/test/suite/regressions/4677-snapshot-catchup-replacement.test.ts
new file mode 100644
index 0000000000..695a211694
--- /dev/null
+++ b/packages/coding-agent/test/suite/regressions/4677-snapshot-catchup-replacement.test.ts
@@ -0,0 +1,800 @@
+import { existsSync, mkdtempSync, rmSync } from "node:fs";
+import type { Socket } from "node:net";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import { PassThrough } from "node:stream";
+import type { AgentMessage } from "@earendil-works/pi-agent-core";
+import { afterEach, describe, expect, it, vi } from "vitest";
+import type { DaemonSocketClient } from "../../../src/modes/daemon/active-session-state.js";
+import { AgentDaemon } from "../../../src/modes/daemon/daemon-mode.js";
+import {
+ DAEMON_PROTOCOL_INFO,
+ type DaemonAttachResult,
+ type DaemonOutbound,
+} from "../../../src/modes/daemon/daemon-protocol.js";
+import type { SessionSummary } from "../../../src/modes/daemon/daemon-session-list.js";
+import { DaemonSupervisor } from "../../../src/modes/daemon/daemon-supervisor.js";
+import type { DaemonWorkerFrameHeader } from "../../../src/modes/daemon/daemon-worker-protocol.js";
+import { SnapshotTranscriptCache } from "../../../src/modes/daemon/snapshot-transcript-cache.js";
+import type { PrivateFrame } from "../../../src/modes/session-worker/private-framing.js";
+
+const activeSessionId = "active-4677";
+const directories: string[] = [];
+
+interface WorkerHarness {
+ descriptor: { workerId: string; rootActiveSessionId: string; lifecycle: "ready"; pid: number };
+ client?: { request: ReturnType<typeof vi.fn> };
+ summaries: Map<string, SessionSummary>;
+ snapshotCache: Map<string, DaemonAttachResult>;
+ transcriptCaches: Map<string, SnapshotTranscriptCache>;
+ snapshotGenerations: Map<
+ string,
+ Map<
+ string,
+ {
+ transcript: SnapshotTranscriptCache;
+ result: DaemonAttachResult;
+ incoming: boolean;
+ retired: boolean;
+ validation?: { promise: Promise<void>; resolve: () => void; reject: (error: Error) => void };
+ }
+ >
+ >;
+ snapshotLoads: Map<string, Promise<DaemonAttachResult>>;
+ intentionalStop: boolean;
+ stopRevision: number;
+}
+
+afterEach(() => {
+ for (const directory of directories.splice(0)) {
+ rmSync(directory, { recursive: true, force: true });
+ }
+});
+
+function tempDirectory(): string {
+ const directory = mkdtempSync(join(tmpdir(), "eng-4677-"));
+ directories.push(directory);
+ return directory;
+}
+
+function summary(messageCount: number): SessionSummary {
+ return {
+ id: activeSessionId,
+ activeSessionId,
+ lifecycle: "live",
+ activity: "idle",
+ sessionId: "session-4677",
+ cwd: "/tmp",
+ isStreaming: false,
+ isCompacting: false,
+ attachedClients: 0,
+ messageCount,
+ pendingMessageCount: 0,
+ };
+}
+
+function streamedResult(snapshotId: string, messageCount: number, lastEventSequence: number): DaemonAttachResult {
+ return {
+ protocol: DAEMON_PROTOCOL_INFO,
+ activeSessionId,
+ snapshot: {
+ activeSessionId,
+ summary: summary(messageCount),
+ state: { activeSessionId, sessionId: "session-4677" } as DaemonAttachResult["snapshot"]["state"],
+ messages: [],
+ lastEventSequence,
+ },
+ replay: { status: "complete", toSequence: lastEventSequence },
+ lastEventSequence,
+ snapshotStream: { id: snapshotId, messageCount, targetChunkBytes: 1 },
+ client: { id: "supervisor", capabilities: ["chunked_snapshot"] },
+ };
+}
+
+function snapshotFrame(
+ message: Extract<
+ DaemonOutbound,
+ { type: "session_snapshot_begin" | "session_snapshot_chunk" | "session_snapshot_end" }
+ >,
+ purpose: "attach" | "replacement" | "catchup" = "replacement",
+): PrivateFrame<DaemonWorkerFrameHeader> {
+ return {
+ header: {
+ kind: "outbound",
+ outboundType: message.type,
+ activeSessionId,
+ snapshotId: message.snapshotId,
+ payloadEncoding: "jsonl",
+ snapshotPurpose: purpose,
+ },
+ payload: Buffer.from(JSON.stringify(message)),
+ };
+}
+
+function socketClient(id: string): DaemonSocketClient {
+ const socket = new PassThrough();
+ return {
+ id,
+ socket: socket as unknown as Socket,
+ attachedActiveSessionIds: new Set([activeSessionId]),
+ catchupActiveSessionIds: new Set(),
+ detachInput: () => {},
+ supportsExtensionUi: false,
+ capabilities: new Set(["chunked_snapshot"]),
+ };
+}
+
+function workerHarness(result: DaemonAttachResult, transcript: SnapshotTranscriptCache): WorkerHarness {
+ return {
+ descriptor: {
+ workerId: "worker-4677",
+ rootActiveSessionId: activeSessionId,
+ lifecycle: "ready",
+ pid: 4677,
+ },
+ summaries: new Map([[activeSessionId, result.snapshot.summary]]),
+ snapshotCache: new Map([[activeSessionId, result]]),
+ transcriptCaches: new Map([[activeSessionId, transcript]]),
+ snapshotGenerations: new Map(),
+ snapshotLoads: new Map(),
+ intentionalStop: false,
+ stopRevision: 0,
+ };
+}
+
+describe("ENG-4677 snapshot catch-up replacement", () => {
+ it("lets an incomplete retained snapshot finish after a newer generation begins", async () => {
+ const root = tempDirectory();
+ const supervisor = new DaemonSupervisor(join(root, "supervisor.sock"), {
+ defaultSessionConfig: { agentDir: root, cwd: root },
+ descriptorDir: join(root, "state"),
+ });
+ const firstSnapshotId = "snapshot-4677-incomplete-a";
+ const replacementSnapshotId = "snapshot-4677-incomplete-b";
+ const placeholder = new SnapshotTranscriptCache({
+ activeSessionId,
+ snapshotId: firstSnapshotId,
+ cacheRoot: root,
+ targetChunkBytes: 1,
+ });
+ const worker = workerHarness(streamedResult(firstSnapshotId, 1, 1), placeholder);
+ worker.snapshotCache.clear();
+ worker.transcriptCaches.clear();
+ const client = socketClient("incomplete-reader");
+ const written: DaemonOutbound[] = [];
+ const writeSnapshotBuffer = vi.fn((_client: DaemonSocketClient, buffer: Uint8Array) => {
+ written.push(JSON.parse(Buffer.from(buffer).toString("utf8")) as DaemonOutbound);
+ return Promise.resolve(true);
+ });
+ const internals = supervisor as unknown as {
+ writeSnapshotBuffer: typeof writeSnapshotBuffer;
+ handleWorkerFrame(worker: WorkerHarness, frame: PrivateFrame<DaemonWorkerFrameHeader>): void;
+ streamSnapshot(
+ client: DaemonSocketClient,
+ worker: WorkerHarness,
+ result: DaemonAttachResult,
+ transcript: SnapshotTranscriptCache,
+ ): Promise<void>;
+ };
+ internals.writeSnapshotBuffer = writeSnapshotBuffer;
+
+ const firstResult = streamedResult(firstSnapshotId, 1, 1);
+ const { messages: _firstMessages, ...firstSnapshot } = firstResult.snapshot;
+ internals.handleWorkerFrame(
+ worker,
+ snapshotFrame({
+ type: "session_snapshot_begin",
+ activeSessionId,
+ snapshotId: firstSnapshotId,
+ snapshot: firstSnapshot,
+ messageCount: 1,
+ targetChunkBytes: 1,
+ }),
+ );
+ const firstTranscript = worker.transcriptCaches.get(activeSessionId);
+ if (!firstTranscript) {
+ throw new Error("first transcript was not initialized");
+ }
+ const firstStream = internals.streamSnapshot(client, worker, firstResult, firstTranscript);
+ void firstStream.catch(() => undefined);
+ await new Promise<void>((resolve) => setImmediate(resolve));
+
+ const replacementResult = streamedResult(replacementSnapshotId, 1, 2);
+ const { messages: _replacementMessages, ...replacementSnapshot } = replacementResult.snapshot;
+ internals.handleWorkerFrame(
+ worker,
+ snapshotFrame({
+ type: "session_snapshot_begin",
+ activeSessionId,
+ snapshotId: replacementSnapshotId,
+ snapshot: replacementSnapshot,
+ messageCount: 1,
+ targetChunkBytes: 1,
+ }),
+ );
+ internals.handleWorkerFrame(
+ worker,
+ snapshotFrame({
+ type: "session_snapshot_chunk",
+ activeSessionId,
+ snapshotId: firstSnapshotId,
+ index: 0,
+ messages: [{ role: "user", content: "first", timestamp: 1 }],
+ }),
+ );
+ internals.handleWorkerFrame(
+ worker,
+ snapshotFrame({
+ type: "session_snapshot_end",
+ activeSessionId,
+ snapshotId: firstSnapshotId,
+ chunkCount: 1,
+ lastEventSequence: 1,
+ }),
+ );
+ internals.handleWorkerFrame(
+ worker,
+ snapshotFrame({
+ type: "session_snapshot_chunk",
+ activeSessionId,
+ snapshotId: replacementSnapshotId,
+ index: 0,
+ messages: [{ role: "user", content: "replacement", timestamp: 2 }],
+ }),
+ );
+ internals.handleWorkerFrame(
+ worker,
+ snapshotFrame({
+ type: "session_snapshot_end",
+ activeSessionId,
+ snapshotId: replacementSnapshotId,
+ chunkCount: 1,
+ lastEventSequence: 2,
+ }),
+ );
+
+ await expect(firstStream).resolves.toBeUndefined();
+ expect(written.some((message) => message.type === "session_snapshot_failed")).toBe(false);
+ expect(worker.transcriptCaches.get(activeSessionId)?.snapshotId).toBe(replacementSnapshotId);
+ client.socket.destroy();
+ });
+
+ it("does not let a stale attach response replace a newer completed generation", async () => {
+ const root = tempDirectory();
+ const supervisor = new DaemonSupervisor(join(root, "supervisor.sock"), {
+ defaultSessionConfig: { agentDir: root, cwd: root },
+ descriptorDir: join(root, "state"),
+ });
+ const firstSnapshotId = "snapshot-4677-stale-a";
+ const replacementSnapshotId = "snapshot-4677-stale-b";
+ const firstResult = streamedResult(firstSnapshotId, 1, 1);
+ const placeholder = new SnapshotTranscriptCache({
+ activeSessionId,
+ snapshotId: firstSnapshotId,
+ cacheRoot: root,
+ targetChunkBytes: 1,
+ });
+ const worker = workerHarness(firstResult, placeholder);
+ worker.snapshotCache.clear();
+ worker.transcriptCaches.clear();
+ let resolveAttach!: (response: { success: true; data: DaemonAttachResult }) => void;
+ worker.client = {
+ request: vi.fn(
+ () =>
+ new Promise<{ success: true; data: DaemonAttachResult }>((resolve) => {
+ resolveAttach = resolve;
+ }),
+ ),
+ };
+ const client = socketClient("stale-attach");
+ const internals = supervisor as unknown as {
+ workers: Map<string, WorkerHarness>;
+ attachClient(
+ client: DaemonSocketClient,
+ command: { type: "attach"; activeSessionId: string; capabilities: ["chunked_snapshot"] },
+ ): Promise<{
+ result: DaemonAttachResult;
+ transcript?: SnapshotTranscriptCache;
+ releaseTranscript?: () => void;
+ }>;
+ handleWorkerFrame(worker: WorkerHarness, frame: PrivateFrame<DaemonWorkerFrameHeader>): void;
+ };
+ internals.workers.set(worker.descriptor.workerId, worker);
+ const attaching = internals.attachClient(client, {
+ type: "attach",
+ activeSessionId,
+ capabilities: ["chunked_snapshot"],
+ });
+ await Promise.resolve();
+
+ for (const result of [firstResult, streamedResult(replacementSnapshotId, 1, 2)]) {
+ const { messages: _messages, ...snapshot } = result.snapshot;
+ const message = result.lastEventSequence === 1 ? "first" : "replacement";
+ internals.handleWorkerFrame(
+ worker,
+ snapshotFrame({
+ type: "session_snapshot_begin",
+ activeSessionId,
+ snapshotId: result.snapshotStream!.id,
+ snapshot,
+ messageCount: 1,
+ targetChunkBytes: 1,
+ }),
+ );
+ internals.handleWorkerFrame(
+ worker,
+ snapshotFrame({
+ type: "session_snapshot_chunk",
+ activeSessionId,
+ snapshotId: result.snapshotStream!.id,
+ index: 0,
+ messages: [{ role: "user", content: message, timestamp: result.lastEventSequence }],
+ }),
+ );
+ internals.handleWorkerFrame(
+ worker,
+ snapshotFrame({
+ type: "session_snapshot_end",
+ activeSessionId,
+ snapshotId: result.snapshotStream!.id,
+ chunkCount: 1,
+ lastEventSequence: result.lastEventSequence,
+ }),
+ );
+ }
+ resolveAttach({ success: true, data: firstResult });
+
+ const attached = await attaching;
+ expect(attached.result.snapshotStream?.id).toBe(replacementSnapshotId);
+ expect(attached.transcript?.snapshotId).toBe(replacementSnapshotId);
+ expect(attached.transcript?.complete).toBe(true);
+ expect(worker.transcriptCaches.get(activeSessionId)?.snapshotId).toBe(replacementSnapshotId);
+ attached.releaseTranscript?.();
+ client.socket.destroy();
+ });
+
+ it("fails and disposes an orphaned incomplete transcript before replacing it", async () => {
+ const root = tempDirectory();
+ const supervisor = new DaemonSupervisor(join(root, "supervisor.sock"), {
+ defaultSessionConfig: { agentDir: root, cwd: root },
+ descriptorDir: join(root, "state"),
+ });
+ const firstSnapshotId = "snapshot-4677-orphan-a";
+ const replacementSnapshotId = "snapshot-4677-orphan-b";
+ const firstResult = streamedResult(firstSnapshotId, 1, 1);
+ const firstTranscript = new SnapshotTranscriptCache({
+ activeSessionId,
+ snapshotId: firstSnapshotId,
+ cacheRoot: root,
+ targetChunkBytes: 1,
+ memoryCacheBytes: 1,
+ });
+ firstTranscript.appendEncodedChunk(Buffer.from("orphaned transcript chunk"));
+ expect(firstTranscript.fileBacked).toBe(true);
+ const cacheDirectory = join(root, firstSnapshotId);
+ expect(existsSync(cacheDirectory)).toBe(true);
+ const waiter = firstTranscript.waitForChunk(1);
+ void waiter.catch(() => undefined);
+ const worker = workerHarness(firstResult, firstTranscript);
+ const internals = supervisor as unknown as {
+ handleWorkerFrame(worker: WorkerHarness, frame: PrivateFrame<DaemonWorkerFrameHeader>): void;
+ };
+ const replacementResult = streamedResult(replacementSnapshotId, 1, 2);
+ const { messages: _messages, ...snapshot } = replacementResult.snapshot;
+
+ internals.handleWorkerFrame(
+ worker,
+ snapshotFrame({
+ type: "session_snapshot_begin",
+ activeSessionId,
+ snapshotId: replacementSnapshotId,
+ snapshot,
+ messageCount: 1,
+ targetChunkBytes: 1,
+ }),
+ );
+
+ await expect(waiter).rejects.toThrow("superseded");
+ expect(firstTranscript.fileBacked).toBe(false);
+ expect(existsSync(cacheDirectory)).toBe(false);
+ expect(worker.transcriptCaches.get(activeSessionId)?.snapshotId).toBe(replacementSnapshotId);
+ });
+
+ it("releases duplicate validation waiters when a newer generation retires it", async () => {
+ const root = tempDirectory();
+ const supervisor = new DaemonSupervisor(join(root, "supervisor.sock"), {
+ defaultSessionConfig: { agentDir: root, cwd: root },
+ descriptorDir: join(root, "state"),
+ });
+ const firstSnapshotId = "snapshot-4677-validation-a";
+ const replacementSnapshotId = "snapshot-4677-validation-b";
+ const firstResult = streamedResult(firstSnapshotId, 1, 1);
+ const placeholder = new SnapshotTranscriptCache({
+ activeSessionId,
+ snapshotId: firstSnapshotId,
+ cacheRoot: root,
+ targetChunkBytes: 1,
+ });
+ const worker = workerHarness(firstResult, placeholder);
+ worker.snapshotCache.clear();
+ worker.transcriptCaches.clear();
+ const client = socketClient("validation-waiter");
+ const internals = supervisor as unknown as {
+ workers: Map<string, WorkerHarness>;
+ attachClient(
+ client: DaemonSocketClient,
+ command: { type: "attach"; activeSessionId: string; capabilities: ["chunked_snapshot"] },
+ ): Promise<{ result: DaemonAttachResult; releaseTranscript?: () => void }>;
+ handleWorkerFrame(worker: WorkerHarness, frame: PrivateFrame<DaemonWorkerFrameHeader>): void;
+ };
+ internals.workers.set(worker.descriptor.workerId, worker);
+ const { messages: _firstMessages, ...firstSnapshot } = firstResult.snapshot;
+ const firstBegin = {
+ type: "session_snapshot_begin",
+ activeSessionId,
+ snapshotId: firstSnapshotId,
+ snapshot: firstSnapshot,
+ messageCount: 1,
+ targetChunkBytes: 1,
+ } satisfies DaemonOutbound;
+ const firstChunk = {
+ type: "session_snapshot_chunk",
+ activeSessionId,
+ snapshotId: firstSnapshotId,
+ index: 0,
+ messages: [{ role: "user", content: "first", timestamp: 1 }],
+ } satisfies DaemonOutbound;
+ const firstEnd = {
+ type: "session_snapshot_end",
+ activeSessionId,
+ snapshotId: firstSnapshotId,
+ chunkCount: 1,
+ lastEventSequence: 1,
+ } satisfies DaemonOutbound;
+ for (const message of [firstBegin, firstChunk, firstEnd, firstBegin]) {
+ internals.handleWorkerFrame(worker, snapshotFrame(message));
+ }
+ expect(worker.snapshotGenerations.get(activeSessionId)?.get(firstSnapshotId)?.validation).toBeDefined();
+
+ let attached = false;
+ const attaching = internals
+ .attachClient(client, {
+ type: "attach",
+ activeSessionId,
+ capabilities: ["chunked_snapshot"],
+ })
+ .then((result) => {
+ attached = true;
+ return result;
+ });
+ await Promise.resolve();
+ const replacementResult = streamedResult(replacementSnapshotId, 1, 2);
+ const { messages: _replacementMessages, ...replacementSnapshot } = replacementResult.snapshot;
+ internals.handleWorkerFrame(
+ worker,
+ snapshotFrame({
+ type: "session_snapshot_begin",
+ activeSessionId,
+ snapshotId: replacementSnapshotId,
+ snapshot: replacementSnapshot,
+ messageCount: 1,
+ targetChunkBytes: 1,
+ }),
+ );
+ await new Promise<void>((resolve) => setImmediate(resolve));
+ const attachedBeforeRetiredTransferFinished = attached;
+
+ internals.handleWorkerFrame(worker, snapshotFrame(firstChunk));
+ internals.handleWorkerFrame(worker, snapshotFrame(firstEnd));
+ internals.handleWorkerFrame(
+ worker,
+ snapshotFrame({
+ type: "session_snapshot_chunk",
+ activeSessionId,
+ snapshotId: replacementSnapshotId,
+ index: 0,
+ messages: [{ role: "user", content: "replacement", timestamp: 2 }],
+ }),
+ );
+ internals.handleWorkerFrame(
+ worker,
+ snapshotFrame({
+ type: "session_snapshot_end",
+ activeSessionId,
+ snapshotId: replacementSnapshotId,
+ chunkCount: 1,
+ lastEventSequence: 2,
+ }),
+ );
+ const result = await attaching;
+ result.releaseTranscript?.();
+
+ expect(attachedBeforeRetiredTransferFinished).toBe(true);
+ expect(result.result.snapshotStream?.id).toBe(replacementSnapshotId);
+ client.socket.destroy();
+ });
+
+ it("waits for supervisor socket drain before continuing catch-up", async () => {
+ const root = tempDirectory();
+ const supervisor = new DaemonSupervisor(join(root, "supervisor.sock"), {
+ defaultSessionConfig: { agentDir: root, cwd: root },
+ descriptorDir: join(root, "state"),
+ });
+ const client = socketClient("supervisor-backpressure");
+ client.backpressured = true;
+ client.catchupActiveSessionIds?.add(activeSessionId);
+ const drainClientCatchups = vi.fn(async (target: DaemonSocketClient) => {
+ target.catchupActiveSessionIds?.clear();
+ });
+ const internals = supervisor as unknown as {
+ drainClientCatchups: typeof drainClientCatchups;
+ catchUpClient(client: DaemonSocketClient): Promise<void>;
+ };
+ internals.drainClientCatchups = drainClientCatchups;
+
+ await internals.catchUpClient(client);
+ expect(drainClientCatchups).not.toHaveBeenCalled();
+ expect(client.catchupActiveSessionIds).toContain(activeSessionId);
+
+ client.backpressured = false;
+ await internals.catchUpClient(client);
+ expect(drainClientCatchups).toHaveBeenCalledOnce();
+ client.socket.destroy();
+ });
+
+ it("waits for worker socket drain before continuing catch-up", async () => {
+ const root = tempDirectory();
+ const daemon = new AgentDaemon(join(root, "worker.sock"), {
+ defaultSessionConfig: { agentDir: root, cwd: root },
+ createRuntime: async () => {
+ throw new Error("unexpected runtime creation");
+ },
+ });
+ const client = socketClient("worker-backpressure");
+ client.backpressured = true;
+ client.catchupActiveSessionIds?.add(activeSessionId);
+ const drainBackpressuredClientCatchups = vi.fn(async (target: DaemonSocketClient) => {
+ target.catchupActiveSessionIds?.clear();
+ });
+ const internals = daemon as unknown as {
+ drainBackpressuredClientCatchups: typeof drainBackpressuredClientCatchups;
+ catchUpBackpressuredClient(client: DaemonSocketClient): Promise<void>;
+ };
+ internals.drainBackpressuredClientCatchups = drainBackpressuredClientCatchups;
+
+ await internals.catchUpBackpressuredClient(client);
+ expect(drainBackpressuredClientCatchups).not.toHaveBeenCalled();
+ expect(client.catchupActiveSessionIds).toContain(activeSessionId);
+
+ client.backpressured = false;
+ await internals.catchUpBackpressuredClient(client);
+ expect(drainBackpressuredClientCatchups).toHaveBeenCalledOnce();
+ client.socket.destroy();
+ });
+
+ it("lets a retained snapshot finish while a newer generation becomes current", async () => {
+ const root = tempDirectory();
+ const supervisor = new DaemonSupervisor(join(root, "supervisor.sock"), {
+ defaultSessionConfig: { agentDir: root, cwd: root },
+ descriptorDir: join(root, "state"),
+ });
+ const firstSnapshotId = "snapshot-4677-a";
+ const replacementSnapshotId = "snapshot-4677-b";
+ const firstMessages: AgentMessage[] = [
+ { role: "user", content: "first", timestamp: 1 },
+ { role: "user", content: "second", timestamp: 2 },
+ ];
+ const firstResult = streamedResult(firstSnapshotId, firstMessages.length, 1);
+ const firstTranscript = new SnapshotTranscriptCache({
+ activeSessionId,
+ snapshotId: firstSnapshotId,
+ messages: firstMessages,
+ cacheRoot: root,
+ targetChunkBytes: 1,
+ });
+ const worker = workerHarness(firstResult, firstTranscript);
+ const client = socketClient("slow-client");
+ const written: DaemonOutbound[] = [];
+ let releaseFirstChunk!: (accepted: boolean) => void;
+ const firstChunkBlocked = new Promise<boolean>((resolve) => {
+ releaseFirstChunk = resolve;
+ });
+ let firstChunkStarted!: () => void;
+ const firstChunkReached = new Promise<void>((resolve) => {
+ firstChunkStarted = resolve;
+ });
+ const writeSnapshotBuffer = vi.fn((_client: DaemonSocketClient, buffer: Uint8Array) => {
+ const message = JSON.parse(Buffer.from(buffer).toString("utf8")) as DaemonOutbound;
+ written.push(message);
+ if (
+ message.type === "session_snapshot_chunk" &&
+ message.snapshotId === firstSnapshotId &&
+ message.index === 0
+ ) {
+ firstChunkStarted();
+ return firstChunkBlocked;
+ }
+ return Promise.resolve(true);
+ });
+ const internals = supervisor as unknown as {
+ clients: Set<DaemonSocketClient>;
+ workers: Map<string, WorkerHarness>;
+ writeSnapshotBuffer: typeof writeSnapshotBuffer;
+ syncWorkerExtensionUi: ReturnType<typeof vi.fn>;
+ streamSnapshot(
+ client: DaemonSocketClient,
+ worker: WorkerHarness,
+ result: DaemonAttachResult,
+ transcript: SnapshotTranscriptCache,
+ purpose: "attach" | "replacement" | "resync",
+ ): Promise<void>;
+ handleWorkerFrame(worker: WorkerHarness, frame: PrivateFrame<DaemonWorkerFrameHeader>): void;
+ queueCatchup(client: DaemonSocketClient, activeSessionId: string, purpose: "replacement" | "resync"): void;
+ catchUpClient(client: DaemonSocketClient): Promise<void>;
+ };
+ internals.writeSnapshotBuffer = writeSnapshotBuffer;
+ internals.syncWorkerExtensionUi = vi.fn(async () => {});
+
+ const firstStream = internals.streamSnapshot(client, worker, firstResult, firstTranscript, "attach");
+ await firstChunkReached;
+
+ const replacementResult = streamedResult(replacementSnapshotId, 1, 2);
+ const { messages: _messages, ...replacementSnapshot } = replacementResult.snapshot;
+ const begin = {
+ type: "session_snapshot_begin",
+ activeSessionId,
+ snapshotId: replacementSnapshotId,
+ snapshot: replacementSnapshot,
+ messageCount: 1,
+ targetChunkBytes: 1,
+ } satisfies DaemonOutbound;
+ const staleChunk = {
+ type: "session_snapshot_chunk",
+ activeSessionId,
+ snapshotId: firstSnapshotId,
+ index: 2,
+ messages: [{ role: "user", content: "stale", timestamp: 3 }],
+ } satisfies DaemonOutbound;
+ const replacementChunk = {
+ type: "session_snapshot_chunk",
+ activeSessionId,
+ snapshotId: replacementSnapshotId,
+ index: 0,
+ messages: [{ role: "user", content: "replacement", timestamp: 4 }],
+ } satisfies DaemonOutbound;
+ const end = {
+ type: "session_snapshot_end",
+ activeSessionId,
+ snapshotId: replacementSnapshotId,
+ chunkCount: 1,
+ lastEventSequence: 2,
+ } satisfies DaemonOutbound;
+ internals.handleWorkerFrame(worker, snapshotFrame(begin));
+ internals.handleWorkerFrame(worker, snapshotFrame(staleChunk));
+ internals.handleWorkerFrame(worker, snapshotFrame(replacementChunk));
+ internals.handleWorkerFrame(worker, snapshotFrame(end));
+
+ expect(worker.transcriptCaches.get(activeSessionId)?.snapshotId).toBe(replacementSnapshotId);
+ expect(worker.transcriptCaches.get(activeSessionId)?.chunkCount).toBe(1);
+ expect(worker.transcriptCaches.get(activeSessionId)?.complete).toBe(true);
+
+ releaseFirstChunk(true);
+ await firstStream;
+
+ expect(written.filter((message) => message.type === "session_snapshot_failed")).toHaveLength(0);
+ expect(
+ written.filter(
+ (message) => message.type === "session_snapshot_chunk" && message.snapshotId === firstSnapshotId,
+ ),
+ ).toHaveLength(2);
+ expect(worker.transcriptCaches.get(activeSessionId)?.snapshotId).toBe(replacementSnapshotId);
+
+ internals.clients.add(client);
+ internals.workers.set(worker.descriptor.workerId, worker);
+ internals.queueCatchup(client, activeSessionId, "replacement");
+ await internals.catchUpClient(client);
+
+ expect(
+ written.some(
+ (message) =>
+ message.type === "session_snapshot_begin" &&
+ message.snapshotId === replacementSnapshotId &&
+ message.purpose === "replacement",
+ ),
+ ).toBe(true);
+ expect(worker.transcriptCaches.get(activeSessionId)?.snapshotId).toBe(replacementSnapshotId);
+ client.socket.destroy();
+ });
+
+ it("coalesces concurrent catch-up triggers into one drain", async () => {
+ const root = tempDirectory();
+ const supervisor = new DaemonSupervisor(join(root, "supervisor.sock"), {
+ defaultSessionConfig: { agentDir: root, cwd: root },
+ descriptorDir: join(root, "state"),
+ });
+ const client = socketClient("coalesced-client");
+ let releaseFirstDrain!: () => void;
+ const firstDrainBlocked = new Promise<void>((resolve) => {
+ releaseFirstDrain = resolve;
+ });
+ const drainClientCatchups = vi.fn(async (target: DaemonSocketClient) => {
+ target.catchupActiveSessionIds?.clear();
+ target.catchupPurposes?.clear();
+ if (drainClientCatchups.mock.calls.length === 1) {
+ await firstDrainBlocked;
+ }
+ });
+ const internals = supervisor as unknown as {
+ drainClientCatchups: typeof drainClientCatchups;
+ queueCatchup(client: DaemonSocketClient, activeSessionId: string, purpose?: "replacement" | "resync"): void;
+ catchUpClient(client: DaemonSocketClient): Promise<void>;
+ };
+ internals.drainClientCatchups = drainClientCatchups;
+ internals.queueCatchup(client, activeSessionId);
+
+ const first = internals.catchUpClient(client);
+ const concurrent = internals.catchUpClient(client);
+ expect(concurrent).toBe(first);
+ expect(drainClientCatchups).toHaveBeenCalledOnce();
+
+ internals.queueCatchup(client, activeSessionId, "replacement");
+ releaseFirstDrain();
+ await first;
+ for (let attempt = 0; attempt < 10 && drainClientCatchups.mock.calls.length < 2; attempt++) {
+ await new Promise<void>((resolve) => setImmediate(resolve));
+ }
+
+ expect(drainClientCatchups).toHaveBeenCalledTimes(2);
+ expect(client.catchupActiveSessionIds?.size).toBe(0);
+ client.socket.destroy();
+ });
+
+ it("coalesces worker catch-up triggers into one drain", async () => {
+ const root = tempDirectory();
+ const daemon = new AgentDaemon(join(root, "worker.sock"), {
+ defaultSessionConfig: { agentDir: root, cwd: root },
+ createRuntime: async () => {
+ throw new Error("unexpected runtime creation");
+ },
+ });
+ const client = socketClient("worker-client");
+ let releaseFirstDrain!: () => void;
+ const firstDrainBlocked = new Promise<void>((resolve) => {
+ releaseFirstDrain = resolve;
+ });
+ const drainBackpressuredClientCatchups = vi.fn(async (target: DaemonSocketClient) => {
+ target.catchupActiveSessionIds?.clear();
+ target.catchupPurposes?.clear();
+ if (drainBackpressuredClientCatchups.mock.calls.length === 1) {
+ await firstDrainBlocked;
+ }
+ });
+ const internals = daemon as unknown as {
+ drainBackpressuredClientCatchups: typeof drainBackpressuredClientCatchups;
+ queueClientCatchup(
+ client: DaemonSocketClient,
+ activeSessionId: string,
+ purpose?: "replacement" | "resync",
+ ): void;
+ catchUpBackpressuredClient(client: DaemonSocketClient): Promise<void>;
+ };
+ internals.drainBackpressuredClientCatchups = drainBackpressuredClientCatchups;
+ internals.queueClientCatchup(client, activeSessionId);
+
+ const first = internals.catchUpBackpressuredClient(client);
+ const concurrent = internals.catchUpBackpressuredClient(client);
+ expect(concurrent).toBe(first);
+ expect(drainBackpressuredClientCatchups).toHaveBeenCalledOnce();
+
+ internals.queueClientCatchup(client, activeSessionId, "replacement");
+ releaseFirstDrain();
+ await first;
+ for (let attempt = 0; attempt < 10 && drainBackpressuredClientCatchups.mock.calls.length < 2; attempt++) {
+ await new Promise<void>((resolve) => setImmediate(resolve));
+ }
+
+ expect(drainBackpressuredClientCatchups).toHaveBeenCalledTimes(2);
+ expect(client.catchupActiveSessionIds?.size).toBe(0);
+ client.socket.destroy();
+ });
+});