Skip to content
TypeScript

fix(server): a late checkpoint keeps the stopped run's baselines (#30830)

Navid Nadali66fa3fe53614
diff --git a/apps/server/src/orchestration-v2/CheckpointCaptureService.test.ts b/apps/server/src/orchestration-v2/CheckpointCaptureService.test.ts
index 17fe7224773..9d34049c3b1 100644
--- a/apps/server/src/orchestration-v2/CheckpointCaptureService.test.ts
+++ b/apps/server/src/orchestration-v2/CheckpointCaptureService.test.ts
@@ -477,6 +477,204 @@ it.layer(layerProjectionStoreTest)("CheckpointCaptureServiceV2", (it) => {
}),
);
+ // A stopped run is captured after it ends; by then the next root run, dispatched or queued, may
+ // have re-created the thread's one root scope with its own node and time.
+ it.effect.each([1, 2])(
+ "attributes the baselines a late capture materializes to the stopped run %i",
+ (ordinal) =>
+ Effect.gen(function* () {
+ const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
+ const requestedAt = yield* DateTime.now;
+ const stoppedAt = DateTime.add(requestedAt, { seconds: 1 });
+ const rescopedAt = DateTime.add(requestedAt, { seconds: 2 });
+ const lateThreadId = ThreadId.make(`thread:checkpoint-capture-late-${ordinal}`);
+ const lateRunId = RunId.make(`run:checkpoint-capture-late-${ordinal}`);
+ const lateRootNodeId = NodeId.make(`node:checkpoint-capture-late-${ordinal}-root`);
+ const lateScopeId = CheckpointScopeId.make(`scope:checkpoint-capture-late-${ordinal}`);
+ const lateProviderThreadId = ProviderThreadId.make(
+ `provider-thread:checkpoint-capture-late-${ordinal}`,
+ );
+ yield* projectionStore.apply({
+ id: EventId.make(`event:checkpoint-capture-late-${ordinal}:thread`),
+ type: "thread.created",
+ threadId: lateThreadId,
+ occurredAt: requestedAt,
+ payload: {
+ createdBy: "user",
+ creationSource: "web",
+ id: lateThreadId,
+ projectId,
+ title: "Checkpoint capture after the next run re-created the scope",
+ providerInstanceId,
+ modelSelection,
+ runtimeMode: "full-access",
+ interactionMode: "default",
+ branch: null,
+ worktreePath: null,
+ activeProviderThreadId: null,
+ lineage: {
+ parentThreadId: null,
+ relationshipToParent: null,
+ rootThreadId: lateThreadId,
+ },
+ forkedFrom: null,
+ createdAt: requestedAt,
+ updatedAt: requestedAt,
+ archivedAt: null,
+ settledOverride: null,
+ settledAt: null,
+ lastVisitedAt: null,
+ deletedAt: null,
+ },
+ });
+ yield* projectionStore.apply({
+ id: EventId.make(`event:checkpoint-capture-late-${ordinal}:run`),
+ type: "run.updated",
+ threadId: lateThreadId,
+ runId: lateRunId,
+ nodeId: lateRootNodeId,
+ providerInstanceId,
+ occurredAt: stoppedAt,
+ payload: {
+ id: lateRunId,
+ threadId: lateThreadId,
+ ordinal,
+ providerInstanceId,
+ modelSelection,
+ providerThreadId: lateProviderThreadId,
+ userMessageId: MessageId.make(`message:checkpoint-capture-late-${ordinal}`),
+ rootNodeId: lateRootNodeId,
+ activeAttemptId: null,
+ status: "interrupted",
+ requestedAt,
+ startedAt: requestedAt,
+ completedAt: stoppedAt,
+ checkpointId: null,
+ contextHandoffId: null,
+ },
+ });
+ yield* projectionStore.apply({
+ id: EventId.make(`event:checkpoint-capture-late-${ordinal}:node`),
+ type: "node.updated",
+ threadId: lateThreadId,
+ runId: lateRunId,
+ nodeId: lateRootNodeId,
+ providerInstanceId,
+ occurredAt: stoppedAt,
+ payload: {
+ id: lateRootNodeId,
+ threadId: lateThreadId,
+ runId: lateRunId,
+ parentNodeId: null,
+ rootNodeId: lateRootNodeId,
+ kind: "root_turn",
+ status: "interrupted",
+ countsForRun: true,
+ providerThreadId: lateProviderThreadId,
+ providerTurnId: null,
+ nativeItemRef: null,
+ runtimeRequestId: null,
+ checkpointScopeId: lateScopeId,
+ startedAt: requestedAt,
+ completedAt: stoppedAt,
+ },
+ });
+ yield* projectionStore.apply({
+ id: EventId.make(`event:checkpoint-capture-late-${ordinal}:provider-thread`),
+ type: "provider-thread.updated",
+ threadId: lateThreadId,
+ nodeId: lateRootNodeId,
+ driver,
+ providerInstanceId,
+ occurredAt: requestedAt,
+ payload: {
+ id: lateProviderThreadId,
+ driver,
+ providerInstanceId,
+ providerSessionId: null,
+ appThreadId: lateThreadId,
+ ownerNodeId: lateRootNodeId,
+ nativeThreadRef: null,
+ nativeConversationHeadRef: null,
+ status: "idle",
+ firstRunOrdinal: 1,
+ lastRunOrdinal: ordinal,
+ handoffIds: [],
+ forkedFrom: null,
+ createdAt: requestedAt,
+ updatedAt: requestedAt,
+ },
+ });
+ const nextRunId = RunId.make(`run:checkpoint-capture-late-${ordinal}-next`);
+ const nextRootNodeId = NodeId.make(`node:checkpoint-capture-late-${ordinal}-next-root`);
+ yield* projectionStore.apply({
+ id: EventId.make(`event:checkpoint-capture-late-${ordinal}:scope-of-next-run`),
+ type: "checkpoint-scope.created",
+ threadId: lateThreadId,
+ runId: nextRunId,
+ nodeId: nextRootNodeId,
+ occurredAt: rescopedAt,
+ payload: {
+ id: lateScopeId,
+ threadId: lateThreadId,
+ runId: nextRunId,
+ nodeId: nextRootNodeId,
+ parentScopeId: null,
+ providerThreadId: lateProviderThreadId,
+ kind: "root_run",
+ ordinalWithinParent: 0,
+ advancesAppRunCount: true,
+ cwd: "/repo",
+ createdAt: rescopedAt,
+ },
+ });
+
+ const committed = yield* Ref.make<ReadonlyArray<OrchestrationV2DomainEvent>>([]);
+ const layerCapture = CheckpointCaptureService.layer.pipe(
+ Layer.provide(
+ Layer.mergeAll(
+ IdAllocator.layer,
+ CheckpointService.layer.pipe(
+ Layer.provide(
+ Layer.mergeAll(
+ IdAllocator.layer,
+ NodeCrypto.layer,
+ Layer.mock(CheckpointStore.CheckpointStore)({
+ isGitRepository: () => Effect.succeed(true),
+ hasCheckpointRef: () => Effect.succeed(true),
+ captureCheckpoint: () => Effect.void,
+ diffCheckpoints: () => Effect.succeed(""),
+ listAuthoredPaths: () => Effect.succeed(null),
+ }),
+ ),
+ ),
+ ),
+ Layer.mock(EventSink.EventSinkV2)({
+ commitCommand: (input) =>
+ Ref.set(committed, input.events).pipe(Effect.as({ committed: true } as never)),
+ }),
+ ),
+ ),
+ );
+ yield* CheckpointCaptureService.CheckpointCaptureServiceV2.use((service) =>
+ service.execute({ threadId: lateThreadId, runId: lateRunId, scopeId: lateScopeId }),
+ ).pipe(Effect.provide(layerCapture));
+
+ const baselines = (yield* Ref.get(committed)).flatMap((event) =>
+ event.type === "checkpoint.captured" && event.payload.runId === null ? [event] : [],
+ );
+ assert.deepEqual(
+ baselines.map((event) => event.payload.ordinalWithinScope),
+ ordinal === 1 ? [0] : [0, 1],
+ );
+ for (const baseline of baselines) {
+ assert.equal(baseline.nodeId, lateRootNodeId);
+ assert.equal(baseline.payload.nodeId, lateRootNodeId);
+ assert.deepEqual(baseline.payload.capturedAt, requestedAt);
+ }
+ }),
+ );
+
it.effect("records the checkpoint of a cancelled run and keeps it cancelled", () =>
Effect.gen(function* () {
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
diff --git a/apps/server/src/orchestration-v2/CheckpointCaptureService.ts b/apps/server/src/orchestration-v2/CheckpointCaptureService.ts
index 9d7d3cfd4ae..6a2d9b2062b 100644
--- a/apps/server/src/orchestration-v2/CheckpointCaptureService.ts
+++ b/apps/server/src/orchestration-v2/CheckpointCaptureService.ts
@@ -103,21 +103,23 @@ export const layer: Layer.Layer<
const baselineOrdinalWithinScope = Math.max(0, run.ordinal - 1);
const hasReadyCheckpoint = (ordinalWithinScope: number) =>
readyCheckpointOrdinals.includes(ordinalWithinScope);
+ // The baselines this capture materializes are attributed to the run materializing them. The
+ // thread's scope may already be a later run's: one queued meanwhile, or sent after Stop.
+ const materializeBaseline = (ordinalWithinScope: number) =>
+ checkpoints.materializeBaselineCheckpoint({
+ scope,
+ ordinalWithinScope,
+ nodeId: rootNode.id,
+ capturedAt: run.requestedAt,
+ forceMissing: sideChat,
+ });
const threadStartCheckpoint =
baselineOrdinalWithinScope === 0 || hasReadyCheckpoint(0)
? null
- : yield* checkpoints.materializeBaselineCheckpoint({
- scope,
- ordinalWithinScope: 0,
- forceMissing: sideChat,
- });
+ : yield* materializeBaseline(0);
const baselineCheckpoint = hasReadyCheckpoint(baselineOrdinalWithinScope)
? null
- : yield* checkpoints.materializeBaselineCheckpoint({
- scope,
- ordinalWithinScope: baselineOrdinalWithinScope,
- forceMissing: sideChat,
- });
+ : yield* materializeBaseline(baselineOrdinalWithinScope);
const checkpoint = yield* checkpoints.capture({
scope,
runId: run.id,
diff --git a/apps/server/src/orchestration-v2/CheckpointService.test.ts b/apps/server/src/orchestration-v2/CheckpointService.test.ts
index 39643ff38cf..250a079c08b 100644
--- a/apps/server/src/orchestration-v2/CheckpointService.test.ts
+++ b/apps/server/src/orchestration-v2/CheckpointService.test.ts
@@ -87,6 +87,8 @@ it.effect(
const materialized = yield* checkpoints.materializeBaselineCheckpoint({
scope,
ordinalWithinScope: 0,
+ nodeId: scope.nodeId,
+ capturedAt: scope.createdAt,
});
assert.strictEqual(materialized.status, "missing");
}
diff --git a/apps/server/src/orchestration-v2/CheckpointService.ts b/apps/server/src/orchestration-v2/CheckpointService.ts
index fc9d84cddfb..09d6f83faef 100644
--- a/apps/server/src/orchestration-v2/CheckpointService.ts
+++ b/apps/server/src/orchestration-v2/CheckpointService.ts
@@ -118,6 +118,9 @@ export interface CheckpointServiceV2Shape {
readonly materializeBaselineCheckpoint: (input: {
readonly scope: OrchestrationV2CheckpointScope;
readonly ordinalWithinScope: number;
+ /** The run materializing the baseline: its root node, and when it was requested. */
+ readonly nodeId: NodeId;
+ readonly capturedAt: DateTime.Utc;
/** Records a missing checkpoint without touching git, for side chats sharing a parent's worktree. */
readonly forceMissing?: boolean;
}) => Effect.Effect<OrchestrationV2Checkpoint, CheckpointServiceV2Error>;
@@ -324,14 +327,14 @@ export const layer: Layer.Layer<
id: checkpointId,
scope: input.scope,
runId: null,
- nodeId: input.scope.nodeId,
+ nodeId: input.nodeId,
parentCheckpointId: null,
ordinalWithinScope: input.ordinalWithinScope,
appRunOrdinal: null,
ref: checkpointRef,
status: available ? "ready" : "missing",
files: [],
- capturedAt: input.scope.createdAt,
+ capturedAt: input.capturedAt,
});
}),
).pipe(