TypeScript
fix(server): a usage-limit auto-resume survives a server restart (#31292)
shivam240446e2d962
diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.tsindex bcec0a13cf2..5d3c4391b1f 100644--- a/apps/server/src/orchestration-v2/Orchestrator.ts+++ b/apps/server/src/orchestration-v2/Orchestrator.ts@@ -6279,6 +6279,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio...(command.restartContinuationOfRunId === undefined? {}: { restartContinuationOfRunId: command.restartContinuationOfRunId }),+ ...(command.usageLimitContinuationOfRunId === undefined+ ? {}+ : { usageLimitContinuationOfRunId: command.usageLimitContinuationOfRunId }),...(continuedTaskId === undefined ? {} : { delegatedTaskId: continuedTaskId }),...(freshContext ? { freshContext: true } : {}),};@@ -6659,6 +6662,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio...(command.restartContinuationOfRunId === undefined? {}: { restartContinuationOfRunId: command.restartContinuationOfRunId }),+ ...(command.usageLimitContinuationOfRunId === undefined+ ? {}+ : { usageLimitContinuationOfRunId: command.usageLimitContinuationOfRunId }),...(dispatchMode.type === "defer_start" && dispatchMode.workspaceStrategy !== undefined? { workspacePreparation: dispatchMode.workspaceStrategy }: {}),@@ -7445,6 +7451,9 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio...(command.restartContinuationOfRunId === undefined? {}: { restartContinuationOfRunId: command.restartContinuationOfRunId }),+ ...(command.usageLimitContinuationOfRunId === undefined+ ? {}+ : { usageLimitContinuationOfRunId: command.usageLimitContinuationOfRunId }),...(continuedTaskId === undefined ? {} : { delegatedTaskId: continuedTaskId }),...wakeWorkStartedAt(projection.runs, command),};diff --git a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.tsindex 222747c4139..89e43e5b1ed 100644--- a/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts+++ b/apps/server/src/orchestration-v2/ProviderRuntimeRecoveryService.ts@@ -442,6 +442,64 @@ export const make = Effect.gen(function* () {},});}+ const unsentResumeRunIds = runs.flatMap((run) =>+ run.startedAt === null && run.usageLimitContinuationOfRunId !== undefined ? [run.id] : [],+ );+ const limitRecovery = projection.thread.limitRecovery;+ // The unsent resume spent the worker's request id, so a fresh one lets it send again. An+ // untagged resume from an older server stays cancelled: its copy couldn't be hidden.+ if (limitRecovery?.autoResume === true && unsentResumeRunIds.length > 0) {+ events.push({+ id: yield* allocateEventId(),+ type: "thread.metadata-updated",+ threadId: projection.thread.id,+ occurredAt: now,+ payload: {+ ...projection.thread,+ limitRecovery: { ...limitRecovery, requestId: commandId },+ updatedAt: now,+ },+ });+ }+ // The worker sends the auto-resume again, so the unsent copy leaves the transcript the way+ // cancelled queued input does.+ const unsentResumePrompts =+ unsentResumeRunIds.length === 0+ ? []+ : (yield* projections+ .getThreadRecords(projection.thread.id, ["turnItems"], {+ turnItemTypes: ["user_message"],+ turnItemRunIds: unsentResumeRunIds,+ })+ .pipe(+ Effect.mapError(+ (cause) =>+ new ProviderRuntimeRecoveryError({+ operation: "read-projections",+ threadId: projection.thread.id,+ cause,+ }),+ ),+ )).turnItems;+ const unsentResumePromptIds = new Set(unsentResumePrompts.map((item) => item.id));+ for (const item of unsentResumePrompts) {+ if (item.type !== "user_message") continue;+ const active =+ item.status === "pending" || item.status === "running" || item.status === "waiting";+ events.push({+ id: yield* allocateEventId(),+ type: "turn-item.updated",+ threadId: projection.thread.id,+ ...(item.runId === null ? {} : { runId: item.runId }),+ occurredAt: now,+ payload: {+ ...item,+ ...(active ? { status: "cancelled", completedAt: now } : {}),+ inputIntent: "queued_turn",+ updatedAt: now,+ },+ });+ }for (const question of questions) {events.push({id: yield* allocateEventId(),@@ -552,6 +610,7 @@ export const make = Effect.gen(function* () {for (const item of projection.turnItems.filter((candidate) =>candidate.runId === run.id &&+ !unsentResumePromptIds.has(candidate.id) &&(candidate.nodeId === null || !messageRequestNodeIds.has(candidate.nodeId)) &&!isAppOwnedDelegationItem(candidate) &&// A link card waits on the user, not the run, and answers by message.diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.tsindex b90f491dfa9..170ceee73a3 100644--- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts+++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts@@ -1,6 +1,7 @@import { PullRequestProviderError } from "@t3tools/source-control-core/server/PullRequestProvider";import { SourceControlProviderKind } from "@t3tools/contracts";import { limitRecoveryCommand } from "./UsageLimitRecoveryWorker.ts";+import { createOrchestrationV2TurnItemVisibility } from "@t3tools/shared/orchestrationV2Timeline";import * as McpProviderSessions from "@t3tools/provider-core/server/McpProviderSessions";import * as SourceControlProviderRegistry from "../sourceControl/SourceControlProviderRegistry.ts";import * as NodeCrypto from "@effect/platform-node/NodeCrypto";@@ -8136,6 +8137,8 @@ it.layer(layerTest)("usage-limit recovery", (it) => {it.effect.each(["resume",+ "resume-started",+ "resume-untagged","queued-resume","rearm","snooze-race",@@ -8614,17 +8617,97 @@ it.layer(layerTest)("usage-limit recovery", (it) => {yield* orchestrator.dispatch(resume);}const after = yield* orchestrator.getThreadProjection(threadId);- if (scenario === "resume") {+ if (+ scenario === "resume" ||+ scenario === "resume-started" ||+ scenario === "resume-untagged"+ ) {assert.deepInclude(after.messages.at(-1), {text: "Continue where you left off.",createdBy: "agent",creationSource: "server",});+ const continuation = after.runs.at(-1)!;+ assert.equal(continuation.status, "starting");+ if (scenario === "resume-started") {+ const startedAt = yield* DateTime.now;+ yield* events.write({+ events: [+ {+ id: EventId.make("recovery:continuation-started:resume-started"),+ type: "run.updated",+ threadId,+ runId: continuation.id,+ occurredAt: startedAt,+ payload: { ...continuation, status: "running", startedAt },+ },+ ],+ });+ }+ if (scenario === "resume-untagged") {+ // A server older than the run tag stored the resume run without it.+ const { usageLimitContinuationOfRunId: _, ...untagged } = continuation;+ yield* events.write({+ events: [+ {+ id: EventId.make("recovery:continuation-untagged"),+ type: "run.updated",+ threadId,+ runId: continuation.id,+ occurredAt: yield* DateTime.now,+ payload: untagged,+ },+ ],+ });+ }+ yield* (yield* ProviderRuntimeRecoveryService.ProviderRuntimeRecoveryService).reconcile(+ "startup",+ );+ const restartedShell = (yield* orchestrator.getShellSnapshot()).threads.find(+ (thread) => thread.id === threadId,+ )!;+ const retry = limitRecoveryCommand(+ restartedShell,+ true,+ DateTime.toEpochMillis(yield* DateTime.now),+ );+ if (scenario === "resume-started") assert.isNull(retry);+ else assert(retry?.type === "message.dispatch");+ if (scenario === "resume") {+ assert.notEqual(retry!.commandId, resume!.commandId);+ } else {+ assert.equal(+ restartedShell.limitRecovery?.requestId,+ after.thread.limitRecovery?.requestId,+ );+ }+ if (retry !== null) {+ yield* orchestrator.dispatch(retry);+ yield* orchestrator.dispatch(retry);+ }+ const resumed = yield* orchestrator.getThreadProjection(threadId);+ const visible = createOrchestrationV2TurnItemVisibility({+ runs: resumed.runs,+ attempts: resumed.attempts,+ items: resumed.turnItems,+ });+ assert.deepEqual(+ resumed.turnItems.flatMap((item) =>+ item.type === "user_message" && visible(item) ? [item.runId] : [],+ ),+ [run.id, resumed.runs.at(-1)!.id],+ );+ assert.deepEqual(+ resumed.runs.map((row) => row.status),+ scenario === "resume" ? ["failed", "cancelled", "starting"] : ["failed", "cancelled"],+ );}assert.lengthOf(after.runs,before.runs.length +(scenario === "resume" ||+ scenario === "resume-started" ||+ scenario === "resume-untagged" ||stopBeforeArm ||scenario === "queued-resume" ||scenario === "independent-patches" ||@@ -8636,6 +8719,8 @@ it.layer(layerTest)("usage-limit recovery", (it) => {after.messages,before.messages.length +(scenario === "resume" ||+ scenario === "resume-started" ||+ scenario === "resume-untagged" ||stopBeforeArm ||scenario === "queued-resume" ||scenario === "independent-patches" ||diff --git a/packages/contracts/src/orchestrationV2.ts b/packages/contracts/src/orchestrationV2.tsindex 461ec5e4b3f..5c430ef7558 100644--- a/packages/contracts/src/orchestrationV2.ts+++ b/packages/contracts/src/orchestrationV2.ts@@ -692,6 +692,7 @@ export const OrchestrationV2Run = Schema.Struct({checkpointId: Schema.NullOr(CheckpointId),contextHandoffId: Schema.NullOr(ContextHandoffId),restartContinuationOfRunId: Schema.optional(RunId),+ usageLimitContinuationOfRunId: Schema.optional(RunId),delegatedTaskId: Schema.optional(NodeId),workStartedAt: Schema.optional(Schema.DateTimeUtc),restartCancelledBackgroundWork: Schema.optional(