TypeScript
fix(server): attribute usage-limit auto-resume to T3 Code (#29838)
shivam90f0158e60b0
diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.tsindex 956de5eca83..db3a2759e3a 100644--- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts+++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts@@ -27,6 +27,7 @@ import * as Fiber from "effect/Fiber";import * as Layer from "effect/Layer";import * as Stream from "effect/Stream";import * as Tracer from "effect/Tracer";+import * as TestClock from "effect/testing/TestClock";import * as CheckpointStore from "../checkpointing/CheckpointStore.ts";import * as ServerConfig from "../config.ts";@@ -46,6 +47,7 @@ import * as Orchestrator from "./Orchestrator.ts";import { continueRestartedRun } from "./RestartContinuation.ts";import * as RuntimeLayer from "./runtimeLayer.ts";import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";+import { limitRecoveryCommand } from "./UsageLimitRecoveryWorker.ts";import * as ProviderTurnStartServiceTestkit from "./ProviderTurnStartService.testkit.ts";import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests";@@ -2398,6 +2400,111 @@ it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))(}),);+ it.effect("tracks a child's usage-limit auto-resume as a follow-up", () =>+ Effect.gen(function* () {+ const orchestrator = yield* Orchestrator.OrchestratorV2;+ const eventSink = yield* EventSink.EventSinkV2;+ const now = yield* DateTime.now;+ const parentThreadId = ThreadId.make("thread:limit-resume-parent");+ const projectId = ProjectId.make("project:limit-resume");+ const parentRunId = RunId.make("run:limit-resume-parent");+ const rootNodeId = NodeId.make("node:limit-resume-parent");+ yield* seedParentWithTerminalTask({+ threadId: parentThreadId,+ projectId,+ runId: parentRunId,+ rootNodeId,+ taskId: NodeId.make("node:limit-resume-seed"),+ deliveryState: "acknowledged",+ now,+ });+ const child = yield* seedRestartCancelledChild({+ parentThreadId,+ projectId,+ parentRunId,+ rootNodeId,+ name: "limit-resume-child",+ completionWake: "always",+ continuationPending: false,+ runStatus: "completed",+ now,+ });+ const childRun = (yield* orchestrator.getThreadProjection(child.childThreadId)).runs[0]!;+ yield* eventSink.write({+ commandId: reconcileCommandId("limit-resume-child-limited"),+ events: [+ {+ id: EventId.make("event:limit-resume-child-failed"),+ type: "run.updated",+ threadId: child.childThreadId,+ runId: childRun.id,+ occurredAt: now,+ payload: { ...childRun, status: "failed", completedAt: now },+ },+ {+ id: EventId.make("event:limit-resume-child-error"),+ type: "turn-item.updated",+ threadId: child.childThreadId,+ occurredAt: now,+ payload: {+ id: TurnItemId.make("turn-item:limit-resume-child-error"),+ type: "error",+ threadId: child.childThreadId,+ runId: childRun.id,+ nodeId: childRun.rootNodeId,+ providerThreadId: null,+ providerTurnId: null,+ nativeItemRef: null,+ parentItemId: null,+ ordinal: 1,+ status: "failed",+ title: "Usage limit reached",+ startedAt: now,+ completedAt: now,+ updatedAt: now,+ failure: {+ class: "usage_limit",+ message: "Plan limit reached.",+ code: "usageLimitExceeded",+ retryable: null,+ resetAt: DateTime.formatIso(DateTime.add(now, { minutes: 1 })),+ },+ },+ },+ ],+ });+ yield* orchestrator.recoverDelegatedTasks;+ const childShell = Effect.map(orchestrator.getShellSnapshot(), (snapshot) =>+ snapshot.threads.find((thread) => thread.id === child.childThreadId),+ );+ const arm = limitRecoveryCommand((yield* childShell)!, true, DateTime.toEpochMillis(now));+ assert(arm?.type === "thread.metadata.update" && arm.limitRecovery != null);+ yield* orchestrator.autoArmLimitRecovery({+ commandId: arm.commandId,+ threadId: arm.threadId,+ limitRecovery: arm.limitRecovery,+ });+ yield* TestClock.adjust("1 minute");+ const resume = limitRecoveryCommand(+ (yield* childShell)!,+ true,+ DateTime.toEpochMillis(yield* DateTime.now),+ );+ assert(resume?.type === "message.dispatch");+ yield* orchestrator.dispatch(resume);++ const resumed = (yield* orchestrator.getThreadProjection(child.childThreadId)).runs.find(+ (run) => run.userMessageId === resume.messageId,+ );+ assert.isDefined(resumed?.delegatedTaskId);+ const followup = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find(+ (task) => task.id === resumed?.delegatedTaskId,+ );+ assert.equal(followup?.childThreadId, child.childThreadId);+ assert.isNull(followup?.result);+ }),+ );+it.effect("serializes a recovered child answer with its parent and tracks the follow-up", () =>Effect.gen(function* () {const orchestrator = yield* Orchestrator.OrchestratorV2;diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.tsindex b13d9290905..a6b876db1de 100644--- a/apps/server/src/orchestration-v2/Orchestrator.ts+++ b/apps/server/src/orchestration-v2/Orchestrator.ts@@ -701,7 +701,11 @@ function mayStartSubagentFollowup(if (command.scheduledTaskId !== undefined || command.restartContinuationOfRunId !== undefined)return false;if (command.notification !== undefined) return command.notification.source.kind !== "monitor";- return command.createdBy === "user" || command.creationSource === "mcp";+ return (+ command.createdBy === "user" ||+ command.creationSource === "mcp" ||+ command.usageLimitContinuationOfRunId !== undefined+ );}/** Waiting runs only drain post-terminal output; their agent turn is over and still needs a wake. */@@ -8041,6 +8045,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functioparentThreadId === null ||(command.notification === undefined &&command.createdBy !== "user" &&+ command.usageLimitContinuationOfRunId === undefined &&command.senderThreadId !== parentThreadId))return;diff --git a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.tsindex e46ac34c8be..985815bd73e 100644--- a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts+++ b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts@@ -73,7 +73,7 @@ export function limitRecoveryCommand(text: "Continue where you left off.",attachments: [],dispatchMode: { type: "start_immediately" },- createdBy: "user",+ createdBy: "agent",creationSource: "server",};}diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.tsindex 9cbb8ead594..17b103f75b8 100644--- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts+++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts@@ -8471,6 +8471,13 @@ it.layer(layerTest)("usage-limit recovery", (it) => {yield* orchestrator.dispatch(resume);}const after = yield* orchestrator.getThreadProjection(threadId);+ if (scenario === "resume") {+ assert.deepInclude(after.messages.at(-1), {+ text: "Continue where you left off.",+ createdBy: "agent",+ creationSource: "server",+ });+ }assert.lengthOf(after.runs,before.runs.length +