Skip to content
TypeScript

fix(server): usage-limit auto-resume no longer arms children of settled parents (#27951)

Julius Marminge95414cf1ad5a
diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.ts
index 0fd9724ae9d..4efe34dd306 100644
--- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts
+++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts
@@ -48,6 +48,7 @@ import {
THREAD_HISTORY_PAGE_POLICY,
THREAD_HISTORY_TURN_SNAPSHOT_ROW_LIMIT,
} from "./threadHistoryPaging.ts";
+import { limitRecoveryCommand } from "./UsageLimitRecoveryWorker.ts";
const layerTest = Layer.mergeAll(
ProjectionStore.layer.pipe(Layer.provideMerge(SqlitePersistence.layerMemory)),
@@ -5457,6 +5458,183 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {
}),
);
+ it.effect("does not freshly arm limit recovery for a subagent of a settled parent", () =>
+ Effect.gen(function* () {
+ const store = yield* ProjectionStore.ProjectionStoreV2;
+ const now = yield* DateTime.now;
+ const resetAt = DateTime.formatIso(DateTime.add(now, { hours: 1 }));
+ const threadPayload = (
+ threadId: ThreadId,
+ parentThreadId: ThreadId | null,
+ settledOverride: "settled" | null,
+ ) =>
+ ({
+ createdBy: "user",
+ creationSource: "web",
+ id: threadId,
+ projectId: ProjectId.make("project:settled-parent-limit"),
+ title: "Settled parent limit recovery",
+ providerInstanceId,
+ modelSelection,
+ runtimeMode: "full-access",
+ interactionMode: "default",
+ branch: null,
+ worktreePath: null,
+ activeProviderThreadId: null,
+ lineage: {
+ parentThreadId,
+ relationshipToParent: parentThreadId === null ? null : "subagent",
+ rootThreadId: parentThreadId ?? threadId,
+ },
+ forkedFrom: null,
+ createdAt: now,
+ updatedAt: now,
+ archivedAt: null,
+ settledOverride,
+ settledAt: settledOverride === null ? null : now,
+ lastVisitedAt: null,
+ deletedAt: null,
+ }) as const;
+ const createThread = (
+ threadId: ThreadId,
+ parentThreadId: ThreadId | null,
+ settledOverride: "settled" | null,
+ ) =>
+ store.apply({
+ id: EventId.make(`event:${threadId}:created`),
+ type: "thread.created",
+ threadId,
+ occurredAt: now,
+ payload: threadPayload(threadId, parentThreadId, settledOverride),
+ });
+ const addLimitedChild = Effect.fnUntraced(function* (name: string, parentSettled: boolean) {
+ const parentId = ThreadId.make(`thread:settled-parent-limit:${name}:parent`);
+ const childId = ThreadId.make(`thread:settled-parent-limit:${name}:child`);
+ const runId = RunId.make(`run:settled-parent-limit:${name}`);
+ const rootNodeId = NodeId.make(`node:settled-parent-limit:${name}`);
+ yield* createThread(parentId, null, parentSettled ? "settled" : null);
+ yield* createThread(childId, parentId, null);
+ const run = {
+ id: runId,
+ threadId: childId,
+ ordinal: 1,
+ providerInstanceId,
+ modelSelection,
+ providerThreadId: null,
+ userMessageId: MessageId.make(`message:settled-parent-limit:${name}`),
+ rootNodeId,
+ activeAttemptId: null,
+ status: "failed" as const,
+ requestedAt: now,
+ startedAt: now,
+ completedAt: now,
+ checkpointId: null,
+ contextHandoffId: null,
+ };
+ yield* store.apply({
+ id: EventId.make(`event:settled-parent-limit:${name}:run`),
+ type: "run.created",
+ threadId: childId,
+ runId,
+ nodeId: rootNodeId,
+ driver,
+ providerInstanceId,
+ occurredAt: now,
+ payload: run,
+ });
+ yield* store.apply({
+ id: EventId.make(`event:settled-parent-limit:${name}:error`),
+ type: "turn-item.updated",
+ threadId: childId,
+ occurredAt: now,
+ payload: {
+ id: TurnItemId.make(`item:settled-parent-limit:${name}:error`),
+ threadId: childId,
+ runId,
+ nodeId: rootNodeId,
+ providerThreadId: null,
+ providerTurnId: null,
+ nativeItemRef: null,
+ parentItemId: null,
+ ordinal: 1,
+ status: "failed",
+ title: "Usage limit reached",
+ startedAt: now,
+ completedAt: now,
+ updatedAt: now,
+ type: "error",
+ failure: {
+ class: "usage_limit",
+ message: "Plan limit reached.",
+ resetAt,
+ code: "usageLimitExceeded",
+ retryable: null,
+ },
+ },
+ });
+ return childId;
+ });
+ const settledParentChild = yield* addLimitedChild("settled", true);
+ const activeParentChild = yield* addLimitedChild("active", false);
+ for (const options of [
+ { now, autoResume: true, snooze: false },
+ { now, autoResume: false, snooze: true },
+ ]) {
+ const armed = (yield* store.getLimitRecoveryCandidates(options))
+ .filter((row) => row.id === settledParentChild || row.id === activeParentChild)
+ .filter(
+ (row) =>
+ limitRecoveryCommand(
+ row,
+ options.autoResume,
+ DateTime.toEpochMillis(now),
+ options.snooze,
+ )?.type === "thread.metadata.update",
+ )
+ .map((row) => row.id);
+ assert.deepEqual(
+ armed,
+ [activeParentChild],
+ `only the child of the unsettled parent is armed (${JSON.stringify(options)})`,
+ );
+ }
+ // A recovery armed before the parent settled still resumes at its reset.
+ yield* store.apply({
+ id: EventId.make("event:settled-parent-limit:settled:armed"),
+ type: "thread.metadata-updated",
+ threadId: settledParentChild,
+ occurredAt: now,
+ payload: {
+ ...threadPayload(
+ settledParentChild,
+ ThreadId.make("thread:settled-parent-limit:settled:parent"),
+ null,
+ ),
+ limitRecovery: {
+ runId: RunId.make("run:settled-parent-limit:settled"),
+ resetAt,
+ autoResume: true,
+ },
+ },
+ });
+ const afterReset = DateTime.add(now, { hours: 2 });
+ const resumed = (yield* store.getLimitRecoveryCandidates({
+ now: afterReset,
+ autoResume: true,
+ snooze: false,
+ })).find((row) => row.id === settledParentChild);
+ assert.isDefined(resumed);
+ assert.isTrue(
+ limitRecoveryCommand(
+ resumed,
+ true,
+ DateTime.toEpochMillis(afterReset),
+ false,
+ )?.commandId.startsWith("limit-resume:"),
+ );
+ }),
+ );
+
it.effect("projects one shared provider session into multiple thread bindings", () =>
Effect.gen(function* () {
const projectionStore = yield* ProjectionStore.ProjectionStoreV2;
diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.ts
index f1816349226..e87b91218fd 100644
--- a/apps/server/src/orchestration-v2/ProjectionStore.ts
+++ b/apps/server/src/orchestration-v2/ProjectionStore.ts
@@ -3974,6 +3974,13 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =
${booleanInt(options.autoResume)}
OR (${booleanInt(options.snooze)} AND julianday(json_extract(item.payload_json, '$.failure.resetAt')) > julianday(${DateTime.formatIso(options.now)}))
)
+ -- A subagent of a settled parent is abandoned work; resuming it would wake the parent.
+ AND NOT EXISTS (
+ SELECT 1 FROM orchestration_v2_projection_threads parent
+ WHERE json_extract(t.payload_json, '$.lineage.relationshipToParent') = 'subagent'
+ AND parent.thread_id = json_extract(t.payload_json, '$.lineage.parentThreadId')
+ AND json_extract(parent.payload_json, '$.settledOverride') = 'settled'
+ )
)
)
AND NOT EXISTS (