TypeScript
fix(server): usage-limit auto-resume no longer arms children of settled parents (#27951)
Julius Marminge95414cf1ad5a
diff --git a/apps/server/src/orchestration-v2/ProjectionStore.test.ts b/apps/server/src/orchestration-v2/ProjectionStore.test.tsindex 0fd9724ae9d..4efe34dd306 100644--- a/apps/server/src/orchestration-v2/ProjectionStore.test.ts+++ b/apps/server/src/orchestration-v2/ProjectionStore.test.ts@@ -48,6 +48,7 @@ import {THREAD_HISTORY_PAGE_POLICY,THREAD_HISTORY_TURN_SNAPSHOT_ROW_LIMIT,} from "./threadHistoryPaging.ts";+import { limitRecoveryCommand } from "./UsageLimitRecoveryWorker.ts";const layerTest = Layer.mergeAll(ProjectionStore.layer.pipe(Layer.provideMerge(SqlitePersistence.layerMemory)),@@ -5457,6 +5458,183 @@ it.layer(layerTest)("ProjectionStoreV2", (it) => {}),);+ it.effect("does not freshly arm limit recovery for a subagent of a settled parent", () =>+ Effect.gen(function* () {+ const store = yield* ProjectionStore.ProjectionStoreV2;+ const now = yield* DateTime.now;+ const resetAt = DateTime.formatIso(DateTime.add(now, { hours: 1 }));+ const threadPayload = (+ threadId: ThreadId,+ parentThreadId: ThreadId | null,+ settledOverride: "settled" | null,+ ) =>+ ({+ createdBy: "user",+ creationSource: "web",+ id: threadId,+ projectId: ProjectId.make("project:settled-parent-limit"),+ title: "Settled parent limit recovery",+ providerInstanceId,+ modelSelection,+ runtimeMode: "full-access",+ interactionMode: "default",+ branch: null,+ worktreePath: null,+ activeProviderThreadId: null,+ lineage: {+ parentThreadId,+ relationshipToParent: parentThreadId === null ? null : "subagent",+ rootThreadId: parentThreadId ?? threadId,+ },+ forkedFrom: null,+ createdAt: now,+ updatedAt: now,+ archivedAt: null,+ settledOverride,+ settledAt: settledOverride === null ? null : now,+ lastVisitedAt: null,+ deletedAt: null,+ }) as const;+ const createThread = (+ threadId: ThreadId,+ parentThreadId: ThreadId | null,+ settledOverride: "settled" | null,+ ) =>+ store.apply({+ id: EventId.make(`event:${threadId}:created`),+ type: "thread.created",+ threadId,+ occurredAt: now,+ payload: threadPayload(threadId, parentThreadId, settledOverride),+ });+ const addLimitedChild = Effect.fnUntraced(function* (name: string, parentSettled: boolean) {+ const parentId = ThreadId.make(`thread:settled-parent-limit:${name}:parent`);+ const childId = ThreadId.make(`thread:settled-parent-limit:${name}:child`);+ const runId = RunId.make(`run:settled-parent-limit:${name}`);+ const rootNodeId = NodeId.make(`node:settled-parent-limit:${name}`);+ yield* createThread(parentId, null, parentSettled ? "settled" : null);+ yield* createThread(childId, parentId, null);+ const run = {+ id: runId,+ threadId: childId,+ ordinal: 1,+ providerInstanceId,+ modelSelection,+ providerThreadId: null,+ userMessageId: MessageId.make(`message:settled-parent-limit:${name}`),+ rootNodeId,+ activeAttemptId: null,+ status: "failed" as const,+ requestedAt: now,+ startedAt: now,+ completedAt: now,+ checkpointId: null,+ contextHandoffId: null,+ };+ yield* store.apply({+ id: EventId.make(`event:settled-parent-limit:${name}:run`),+ type: "run.created",+ threadId: childId,+ runId,+ nodeId: rootNodeId,+ driver,+ providerInstanceId,+ occurredAt: now,+ payload: run,+ });+ yield* store.apply({+ id: EventId.make(`event:settled-parent-limit:${name}:error`),+ type: "turn-item.updated",+ threadId: childId,+ occurredAt: now,+ payload: {+ id: TurnItemId.make(`item:settled-parent-limit:${name}:error`),+ threadId: childId,+ runId,+ nodeId: rootNodeId,+ providerThreadId: null,+ providerTurnId: null,+ nativeItemRef: null,+ parentItemId: null,+ ordinal: 1,+ status: "failed",+ title: "Usage limit reached",+ startedAt: now,+ completedAt: now,+ updatedAt: now,+ type: "error",+ failure: {+ class: "usage_limit",+ message: "Plan limit reached.",+ resetAt,+ code: "usageLimitExceeded",+ retryable: null,+ },+ },+ });+ return childId;+ });+ const settledParentChild = yield* addLimitedChild("settled", true);+ const activeParentChild = yield* addLimitedChild("active", false);+ for (const options of [+ { now, autoResume: true, snooze: false },+ { now, autoResume: false, snooze: true },+ ]) {+ const armed = (yield* store.getLimitRecoveryCandidates(options))+ .filter((row) => row.id === settledParentChild || row.id === activeParentChild)+ .filter(+ (row) =>+ limitRecoveryCommand(+ row,+ options.autoResume,+ DateTime.toEpochMillis(now),+ options.snooze,+ )?.type === "thread.metadata.update",+ )+ .map((row) => row.id);+ assert.deepEqual(+ armed,+ [activeParentChild],+ `only the child of the unsettled parent is armed (${JSON.stringify(options)})`,+ );+ }+ // A recovery armed before the parent settled still resumes at its reset.+ yield* store.apply({+ id: EventId.make("event:settled-parent-limit:settled:armed"),+ type: "thread.metadata-updated",+ threadId: settledParentChild,+ occurredAt: now,+ payload: {+ ...threadPayload(+ settledParentChild,+ ThreadId.make("thread:settled-parent-limit:settled:parent"),+ null,+ ),+ limitRecovery: {+ runId: RunId.make("run:settled-parent-limit:settled"),+ resetAt,+ autoResume: true,+ },+ },+ });+ const afterReset = DateTime.add(now, { hours: 2 });+ const resumed = (yield* store.getLimitRecoveryCandidates({+ now: afterReset,+ autoResume: true,+ snooze: false,+ })).find((row) => row.id === settledParentChild);+ assert.isDefined(resumed);+ assert.isTrue(+ limitRecoveryCommand(+ resumed,+ true,+ DateTime.toEpochMillis(afterReset),+ false,+ )?.commandId.startsWith("limit-resume:"),+ );+ }),+ );+it.effect("projects one shared provider session into multiple thread bindings", () =>Effect.gen(function* () {const projectionStore = yield* ProjectionStore.ProjectionStoreV2;diff --git a/apps/server/src/orchestration-v2/ProjectionStore.ts b/apps/server/src/orchestration-v2/ProjectionStore.tsindex f1816349226..e87b91218fd 100644--- a/apps/server/src/orchestration-v2/ProjectionStore.ts+++ b/apps/server/src/orchestration-v2/ProjectionStore.ts@@ -3974,6 +3974,13 @@ export const layer: Layer.Layer<ProjectionStoreV2, never, SqlClient.SqlClient> =${booleanInt(options.autoResume)}OR (${booleanInt(options.snooze)} AND julianday(json_extract(item.payload_json, '$.failure.resetAt')) > julianday(${DateTime.formatIso(options.now)})))+ -- A subagent of a settled parent is abandoned work; resuming it would wake the parent.+ AND NOT EXISTS (+ SELECT 1 FROM orchestration_v2_projection_threads parent+ WHERE json_extract(t.payload_json, '$.lineage.relationshipToParent') = 'subagent'+ AND parent.thread_id = json_extract(t.payload_json, '$.lineage.parentThreadId')+ AND json_extract(parent.payload_json, '$.settledOverride') = 'settled'+ )))AND NOT EXISTS (