Skip to content
TypeScript

fix(server): attribute usage-limit auto-resume to T3 Code (#29838)

shivam90f0158e60b0
diff --git a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts
index 956de5eca83..db3a2759e3a 100644
--- a/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts
+++ b/apps/server/src/orchestration-v2/DelegatedCompletionDelivery.test.ts
@@ -27,6 +27,7 @@ import * as Fiber from "effect/Fiber";
import * as Layer from "effect/Layer";
import * as Stream from "effect/Stream";
import * as Tracer from "effect/Tracer";
+import * as TestClock from "effect/testing/TestClock";
import * as CheckpointStore from "../checkpointing/CheckpointStore.ts";
import * as ServerConfig from "../config.ts";
@@ -46,6 +47,7 @@ import * as Orchestrator from "./Orchestrator.ts";
import { continueRestartedRun } from "./RestartContinuation.ts";
import * as RuntimeLayer from "./runtimeLayer.ts";
import * as ThreadCommandExecutor from "./ThreadCommandExecutor.ts";
+import { limitRecoveryCommand } from "./UsageLimitRecoveryWorker.ts";
import * as ProviderTurnStartServiceTestkit from "./ProviderTurnStartService.testkit.ts";
import type * as ProviderAdapter from "@t3tools/provider-core/server/ProviderAdapter";
import * as ProviderContinuationRequests from "@t3tools/provider-core/server/ProviderContinuationRequests";
@@ -2398,6 +2400,111 @@ it.layer(Layer.merge(layerTest, ProviderContinuationRequests.layer))(
}),
);
+ it.effect("tracks a child's usage-limit auto-resume as a follow-up", () =>
+ Effect.gen(function* () {
+ const orchestrator = yield* Orchestrator.OrchestratorV2;
+ const eventSink = yield* EventSink.EventSinkV2;
+ const now = yield* DateTime.now;
+ const parentThreadId = ThreadId.make("thread:limit-resume-parent");
+ const projectId = ProjectId.make("project:limit-resume");
+ const parentRunId = RunId.make("run:limit-resume-parent");
+ const rootNodeId = NodeId.make("node:limit-resume-parent");
+ yield* seedParentWithTerminalTask({
+ threadId: parentThreadId,
+ projectId,
+ runId: parentRunId,
+ rootNodeId,
+ taskId: NodeId.make("node:limit-resume-seed"),
+ deliveryState: "acknowledged",
+ now,
+ });
+ const child = yield* seedRestartCancelledChild({
+ parentThreadId,
+ projectId,
+ parentRunId,
+ rootNodeId,
+ name: "limit-resume-child",
+ completionWake: "always",
+ continuationPending: false,
+ runStatus: "completed",
+ now,
+ });
+ const childRun = (yield* orchestrator.getThreadProjection(child.childThreadId)).runs[0]!;
+ yield* eventSink.write({
+ commandId: reconcileCommandId("limit-resume-child-limited"),
+ events: [
+ {
+ id: EventId.make("event:limit-resume-child-failed"),
+ type: "run.updated",
+ threadId: child.childThreadId,
+ runId: childRun.id,
+ occurredAt: now,
+ payload: { ...childRun, status: "failed", completedAt: now },
+ },
+ {
+ id: EventId.make("event:limit-resume-child-error"),
+ type: "turn-item.updated",
+ threadId: child.childThreadId,
+ occurredAt: now,
+ payload: {
+ id: TurnItemId.make("turn-item:limit-resume-child-error"),
+ type: "error",
+ threadId: child.childThreadId,
+ runId: childRun.id,
+ nodeId: childRun.rootNodeId,
+ providerThreadId: null,
+ providerTurnId: null,
+ nativeItemRef: null,
+ parentItemId: null,
+ ordinal: 1,
+ status: "failed",
+ title: "Usage limit reached",
+ startedAt: now,
+ completedAt: now,
+ updatedAt: now,
+ failure: {
+ class: "usage_limit",
+ message: "Plan limit reached.",
+ code: "usageLimitExceeded",
+ retryable: null,
+ resetAt: DateTime.formatIso(DateTime.add(now, { minutes: 1 })),
+ },
+ },
+ },
+ ],
+ });
+ yield* orchestrator.recoverDelegatedTasks;
+ const childShell = Effect.map(orchestrator.getShellSnapshot(), (snapshot) =>
+ snapshot.threads.find((thread) => thread.id === child.childThreadId),
+ );
+ const arm = limitRecoveryCommand((yield* childShell)!, true, DateTime.toEpochMillis(now));
+ assert(arm?.type === "thread.metadata.update" && arm.limitRecovery != null);
+ yield* orchestrator.autoArmLimitRecovery({
+ commandId: arm.commandId,
+ threadId: arm.threadId,
+ limitRecovery: arm.limitRecovery,
+ });
+ yield* TestClock.adjust("1 minute");
+ const resume = limitRecoveryCommand(
+ (yield* childShell)!,
+ true,
+ DateTime.toEpochMillis(yield* DateTime.now),
+ );
+ assert(resume?.type === "message.dispatch");
+ yield* orchestrator.dispatch(resume);
+
+ const resumed = (yield* orchestrator.getThreadProjection(child.childThreadId)).runs.find(
+ (run) => run.userMessageId === resume.messageId,
+ );
+ assert.isDefined(resumed?.delegatedTaskId);
+ const followup = (yield* orchestrator.getThreadProjection(parentThreadId)).subagents.find(
+ (task) => task.id === resumed?.delegatedTaskId,
+ );
+ assert.equal(followup?.childThreadId, child.childThreadId);
+ assert.isNull(followup?.result);
+ }),
+ );
+
it.effect("serializes a recovered child answer with its parent and tracks the follow-up", () =>
Effect.gen(function* () {
const orchestrator = yield* Orchestrator.OrchestratorV2;
diff --git a/apps/server/src/orchestration-v2/Orchestrator.ts b/apps/server/src/orchestration-v2/Orchestrator.ts
index b13d9290905..a6b876db1de 100644
--- a/apps/server/src/orchestration-v2/Orchestrator.ts
+++ b/apps/server/src/orchestration-v2/Orchestrator.ts
@@ -701,7 +701,11 @@ function mayStartSubagentFollowup(
if (command.scheduledTaskId !== undefined || command.restartContinuationOfRunId !== undefined)
return false;
if (command.notification !== undefined) return command.notification.source.kind !== "monitor";
- return command.createdBy === "user" || command.creationSource === "mcp";
+ return (
+ command.createdBy === "user" ||
+ command.creationSource === "mcp" ||
+ command.usageLimitContinuationOfRunId !== undefined
+ );
}
/** Waiting runs only drain post-terminal output; their agent turn is over and still needs a wake. */
@@ -8041,6 +8045,7 @@ const makeOrchestrator = Effect.fn("orchestrationV2.Orchestrator.layer")(functio
parentThreadId === null ||
(command.notification === undefined &&
command.createdBy !== "user" &&
+ command.usageLimitContinuationOfRunId === undefined &&
command.senderThreadId !== parentThreadId)
)
return;
diff --git a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts
index e46ac34c8be..985815bd73e 100644
--- a/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts
+++ b/apps/server/src/orchestration-v2/UsageLimitRecoveryWorker.ts
@@ -73,7 +73,7 @@ export function limitRecoveryCommand(
text: "Continue where you left off.",
attachments: [],
dispatchMode: { type: "start_immediately" },
- createdBy: "user",
+ createdBy: "agent",
creationSource: "server",
};
}
diff --git a/apps/server/src/orchestration-v2/runtimeLayer.test.ts b/apps/server/src/orchestration-v2/runtimeLayer.test.ts
index 9cbb8ead594..17b103f75b8 100644
--- a/apps/server/src/orchestration-v2/runtimeLayer.test.ts
+++ b/apps/server/src/orchestration-v2/runtimeLayer.test.ts
@@ -8471,6 +8471,13 @@ it.layer(layerTest)("usage-limit recovery", (it) => {
yield* orchestrator.dispatch(resume);
}
const after = yield* orchestrator.getThreadProjection(threadId);
+ if (scenario === "resume") {
+ assert.deepInclude(after.messages.at(-1), {
+ text: "Continue where you left off.",
+ createdBy: "agent",
+ creationSource: "server",
+ });
+ }
assert.lengthOf(
after.runs,
before.runs.length +