TypeScript
fix(server): a late checkpoint keeps the stopped run's baselines (#30830)
Navid Nadali66fa3fe53614
diff --git a/apps/server/src/orchestration-v2/CheckpointCaptureService.test.ts b/apps/server/src/orchestration-v2/CheckpointCaptureService.test.tsindex 17fe7224773..9d34049c3b1 100644--- a/apps/server/src/orchestration-v2/CheckpointCaptureService.test.ts+++ b/apps/server/src/orchestration-v2/CheckpointCaptureService.test.ts@@ -477,6 +477,204 @@ it.layer(layerProjectionStoreTest)("CheckpointCaptureServiceV2", (it) => {}),);+ // A stopped run is captured after it ends; by then the next root run, dispatched or queued, may+ // have re-created the thread's one root scope with its own node and time.+ it.effect.each([1, 2])(+ "attributes the baselines a late capture materializes to the stopped run %i",+ (ordinal) =>+ Effect.gen(function* () {+ const projectionStore = yield* ProjectionStore.ProjectionStoreV2;+ const requestedAt = yield* DateTime.now;+ const stoppedAt = DateTime.add(requestedAt, { seconds: 1 });+ const rescopedAt = DateTime.add(requestedAt, { seconds: 2 });+ const lateThreadId = ThreadId.make(`thread:checkpoint-capture-late-${ordinal}`);+ const lateRunId = RunId.make(`run:checkpoint-capture-late-${ordinal}`);+ const lateRootNodeId = NodeId.make(`node:checkpoint-capture-late-${ordinal}-root`);+ const lateScopeId = CheckpointScopeId.make(`scope:checkpoint-capture-late-${ordinal}`);+ const lateProviderThreadId = ProviderThreadId.make(+ `provider-thread:checkpoint-capture-late-${ordinal}`,+ );+ yield* projectionStore.apply({+ id: EventId.make(`event:checkpoint-capture-late-${ordinal}:thread`),+ type: "thread.created",+ threadId: lateThreadId,+ occurredAt: requestedAt,+ payload: {+ createdBy: "user",+ creationSource: "web",+ id: lateThreadId,+ projectId,+ title: "Checkpoint capture after the next run re-created the scope",+ providerInstanceId,+ modelSelection,+ runtimeMode: "full-access",+ interactionMode: "default",+ branch: null,+ worktreePath: null,+ activeProviderThreadId: null,+ lineage: {+ parentThreadId: null,+ relationshipToParent: null,+ rootThreadId: lateThreadId,+ },+ forkedFrom: null,+ createdAt: requestedAt,+ updatedAt: requestedAt,+ archivedAt: null,+ settledOverride: null,+ settledAt: null,+ lastVisitedAt: null,+ deletedAt: null,+ },+ });+ yield* projectionStore.apply({+ id: EventId.make(`event:checkpoint-capture-late-${ordinal}:run`),+ type: "run.updated",+ threadId: lateThreadId,+ runId: lateRunId,+ nodeId: lateRootNodeId,+ providerInstanceId,+ occurredAt: stoppedAt,+ payload: {+ id: lateRunId,+ threadId: lateThreadId,+ ordinal,+ providerInstanceId,+ modelSelection,+ providerThreadId: lateProviderThreadId,+ userMessageId: MessageId.make(`message:checkpoint-capture-late-${ordinal}`),+ rootNodeId: lateRootNodeId,+ activeAttemptId: null,+ status: "interrupted",+ requestedAt,+ startedAt: requestedAt,+ completedAt: stoppedAt,+ checkpointId: null,+ contextHandoffId: null,+ },+ });+ yield* projectionStore.apply({+ id: EventId.make(`event:checkpoint-capture-late-${ordinal}:node`),+ type: "node.updated",+ threadId: lateThreadId,+ runId: lateRunId,+ nodeId: lateRootNodeId,+ providerInstanceId,+ occurredAt: stoppedAt,+ payload: {+ id: lateRootNodeId,+ threadId: lateThreadId,+ runId: lateRunId,+ parentNodeId: null,+ rootNodeId: lateRootNodeId,+ kind: "root_turn",+ status: "interrupted",+ countsForRun: true,+ providerThreadId: lateProviderThreadId,+ providerTurnId: null,+ nativeItemRef: null,+ runtimeRequestId: null,+ checkpointScopeId: lateScopeId,+ startedAt: requestedAt,+ completedAt: stoppedAt,+ },+ });+ yield* projectionStore.apply({+ id: EventId.make(`event:checkpoint-capture-late-${ordinal}:provider-thread`),+ type: "provider-thread.updated",+ threadId: lateThreadId,+ nodeId: lateRootNodeId,+ driver,+ providerInstanceId,+ occurredAt: requestedAt,+ payload: {+ id: lateProviderThreadId,+ driver,+ providerInstanceId,+ providerSessionId: null,+ appThreadId: lateThreadId,+ ownerNodeId: lateRootNodeId,+ nativeThreadRef: null,+ nativeConversationHeadRef: null,+ status: "idle",+ firstRunOrdinal: 1,+ lastRunOrdinal: ordinal,+ handoffIds: [],+ forkedFrom: null,+ createdAt: requestedAt,+ updatedAt: requestedAt,+ },+ });+ const nextRunId = RunId.make(`run:checkpoint-capture-late-${ordinal}-next`);+ const nextRootNodeId = NodeId.make(`node:checkpoint-capture-late-${ordinal}-next-root`);+ yield* projectionStore.apply({+ id: EventId.make(`event:checkpoint-capture-late-${ordinal}:scope-of-next-run`),+ type: "checkpoint-scope.created",+ threadId: lateThreadId,+ runId: nextRunId,+ nodeId: nextRootNodeId,+ occurredAt: rescopedAt,+ payload: {+ id: lateScopeId,+ threadId: lateThreadId,+ runId: nextRunId,+ nodeId: nextRootNodeId,+ parentScopeId: null,+ providerThreadId: lateProviderThreadId,+ kind: "root_run",+ ordinalWithinParent: 0,+ advancesAppRunCount: true,+ cwd: "/repo",+ createdAt: rescopedAt,+ },+ });++ const committed = yield* Ref.make<ReadonlyArray<OrchestrationV2DomainEvent>>([]);+ const layerCapture = CheckpointCaptureService.layer.pipe(+ Layer.provide(+ Layer.mergeAll(+ IdAllocator.layer,+ CheckpointService.layer.pipe(+ Layer.provide(+ Layer.mergeAll(+ IdAllocator.layer,+ NodeCrypto.layer,+ Layer.mock(CheckpointStore.CheckpointStore)({+ isGitRepository: () => Effect.succeed(true),+ hasCheckpointRef: () => Effect.succeed(true),+ captureCheckpoint: () => Effect.void,+ diffCheckpoints: () => Effect.succeed(""),+ listAuthoredPaths: () => Effect.succeed(null),+ }),+ ),+ ),+ ),+ Layer.mock(EventSink.EventSinkV2)({+ commitCommand: (input) =>+ Ref.set(committed, input.events).pipe(Effect.as({ committed: true } as never)),+ }),+ ),+ ),+ );+ yield* CheckpointCaptureService.CheckpointCaptureServiceV2.use((service) =>+ service.execute({ threadId: lateThreadId, runId: lateRunId, scopeId: lateScopeId }),+ ).pipe(Effect.provide(layerCapture));++ const baselines = (yield* Ref.get(committed)).flatMap((event) =>+ event.type === "checkpoint.captured" && event.payload.runId === null ? [event] : [],+ );+ assert.deepEqual(+ baselines.map((event) => event.payload.ordinalWithinScope),+ ordinal === 1 ? [0] : [0, 1],+ );+ for (const baseline of baselines) {+ assert.equal(baseline.nodeId, lateRootNodeId);+ assert.equal(baseline.payload.nodeId, lateRootNodeId);+ assert.deepEqual(baseline.payload.capturedAt, requestedAt);+ }+ }),+ );+it.effect("records the checkpoint of a cancelled run and keeps it cancelled", () =>Effect.gen(function* () {const projectionStore = yield* ProjectionStore.ProjectionStoreV2;diff --git a/apps/server/src/orchestration-v2/CheckpointCaptureService.ts b/apps/server/src/orchestration-v2/CheckpointCaptureService.tsindex 9d7d3cfd4ae..6a2d9b2062b 100644--- a/apps/server/src/orchestration-v2/CheckpointCaptureService.ts+++ b/apps/server/src/orchestration-v2/CheckpointCaptureService.ts@@ -103,21 +103,23 @@ export const layer: Layer.Layer<const baselineOrdinalWithinScope = Math.max(0, run.ordinal - 1);const hasReadyCheckpoint = (ordinalWithinScope: number) =>readyCheckpointOrdinals.includes(ordinalWithinScope);+ // The baselines this capture materializes are attributed to the run materializing them. The+ // thread's scope may already be a later run's: one queued meanwhile, or sent after Stop.+ const materializeBaseline = (ordinalWithinScope: number) =>+ checkpoints.materializeBaselineCheckpoint({+ scope,+ ordinalWithinScope,+ nodeId: rootNode.id,+ capturedAt: run.requestedAt,+ forceMissing: sideChat,+ });const threadStartCheckpoint =baselineOrdinalWithinScope === 0 || hasReadyCheckpoint(0)? null- : yield* checkpoints.materializeBaselineCheckpoint({- scope,- ordinalWithinScope: 0,- forceMissing: sideChat,- });+ : yield* materializeBaseline(0);const baselineCheckpoint = hasReadyCheckpoint(baselineOrdinalWithinScope)? null- : yield* checkpoints.materializeBaselineCheckpoint({- scope,- ordinalWithinScope: baselineOrdinalWithinScope,- forceMissing: sideChat,- });+ : yield* materializeBaseline(baselineOrdinalWithinScope);const checkpoint = yield* checkpoints.capture({scope,runId: run.id,diff --git a/apps/server/src/orchestration-v2/CheckpointService.test.ts b/apps/server/src/orchestration-v2/CheckpointService.test.tsindex 39643ff38cf..250a079c08b 100644--- a/apps/server/src/orchestration-v2/CheckpointService.test.ts+++ b/apps/server/src/orchestration-v2/CheckpointService.test.ts@@ -87,6 +87,8 @@ it.effect(const materialized = yield* checkpoints.materializeBaselineCheckpoint({scope,ordinalWithinScope: 0,+ nodeId: scope.nodeId,+ capturedAt: scope.createdAt,});assert.strictEqual(materialized.status, "missing");}diff --git a/apps/server/src/orchestration-v2/CheckpointService.ts b/apps/server/src/orchestration-v2/CheckpointService.tsindex fc9d84cddfb..09d6f83faef 100644--- a/apps/server/src/orchestration-v2/CheckpointService.ts+++ b/apps/server/src/orchestration-v2/CheckpointService.ts@@ -118,6 +118,9 @@ export interface CheckpointServiceV2Shape {readonly materializeBaselineCheckpoint: (input: {readonly scope: OrchestrationV2CheckpointScope;readonly ordinalWithinScope: number;+ /** The run materializing the baseline: its root node, and when it was requested. */+ readonly nodeId: NodeId;+ readonly capturedAt: DateTime.Utc;/** Records a missing checkpoint without touching git, for side chats sharing a parent's worktree. */readonly forceMissing?: boolean;}) => Effect.Effect<OrchestrationV2Checkpoint, CheckpointServiceV2Error>;@@ -324,14 +327,14 @@ export const layer: Layer.Layer<id: checkpointId,scope: input.scope,runId: null,- nodeId: input.scope.nodeId,+ nodeId: input.nodeId,parentCheckpointId: null,ordinalWithinScope: input.ordinalWithinScope,appRunOrdinal: null,ref: checkpointRef,status: available ? "ready" : "missing",files: [],- capturedAt: input.scope.createdAt,+ capturedAt: input.capturedAt,});}),).pipe(