TypeScript
fix(server): clean up child worktrees after their parent settles (#30026)
Lakshmi Tanmayb6307ee88a72
diff --git a/apps/server/src/storageCleanup.test.ts b/apps/server/src/storageCleanup.test.tsindex 937af138d9d..b4c9ac3e689 100644--- a/apps/server/src/storageCleanup.test.ts+++ b/apps/server/src/storageCleanup.test.ts@@ -778,6 +778,162 @@ it.live("preserves late tracked edits when submodules were never initialized", ();describe("subagent worktree ownership", () => {+ it.effect.each([+ "child",+ "grandchild",+ "immediate",+ "active-parent",+ "recent-settle",+ "settle-grace",+ "recent-child-activity",+ "running-child",+ "pending-child",+ "running-parent",+ "pending-parent",+ "unsettled-child",+ "pinned-child",+ "fork",+ "missing-parent",+ "cycle",+ ] as const)("checks isolated subagent settlement (%s)", (scenario) =>+ Effect.gen(function* () {+ yield* TestClock.setTime(NOW_MS);+ const fixture = yield* cleanupFixture({+ ...DEFAULT_SERVER_SETTINGS.storageCleanup,+ worktreeSettledAfterDays: scenario === "immediate" ? 0 : 8,+ });+ const root = shell({+ settledAt:+ scenario === "active-parent"+ ? null+ : scenario === "recent-settle" || scenario === "settle-grace"+ ? at(-30_000)+ : scenario === "immediate"+ ? at(-120_000)+ : at(-20 * DAY_MS),+ settledOverride: scenario === "active-parent" ? null : "settled",+ updatedAt: scenario === "settle-grace" ? at(-30_000) : at(-10 * DAY_MS),+ });+ const child = shell({+ id: ThreadId.make("child"),+ branch: "feature",+ worktreePath: fixture.worktree,+ lineage: {+ parentThreadId:+ scenario === "missing-parent"+ ? ThreadId.make("missing")+ : scenario === "cycle"+ ? ThreadId.make("child")+ : ["grandchild", "running-parent", "pending-parent"].includes(scenario)+ ? ThreadId.make("middle")+ : root.id,+ rootThreadId: root.id,+ relationshipToParent: scenario === "fork" ? "fork" : "subagent",+ },+ status: scenario === "running-child" ? "running" : "completed",+ pendingBackgroundTasks:+ scenario === "pending-child" ? [{ taskId: "task", kind: "command" }] : [],+ latestRunCompletedAt: scenario === "recent-child-activity" ? at(-2 * DAY_MS) : null,+ settledOverride: scenario === "unsettled-child" ? "active" : null,+ pinnedAt: scenario === "pinned-child" ? at(-DAY_MS) : null,+ });+ const middle = shell({+ id: ThreadId.make("middle"),+ status: scenario === "running-parent" ? "running" : "idle",+ pendingBackgroundTasks:+ scenario === "pending-parent" ? [{ taskId: "task", kind: "command" }] : [],+ lineage: {+ parentThreadId: root.id,+ rootThreadId: root.id,+ relationshipToParent: "subagent",+ },+ });+ fixture.setThreads([child, middle, root]);+ const report = yield* fixture.service.runNow;+ const removed = ["child", "grandchild", "immediate"].includes(scenario);+ expect(yield* fixture.fs.exists(fixture.worktree)).toBe(!removed);+ expect(report.entries[0]).toMatchObject({+ threadId: child.id,+ outcome: removed ? "removed" : "kept",+ reason: removed+ ? scenario === "immediate"+ ? "Settled for 0 days"+ : "Settled for 20 days"+ : scenario === "settle-grace"+ ? "Settled moments ago"+ : scenario === "recent-settle"+ ? "Settled for 0 of 8 days"+ : scenario === "recent-child-activity"+ ? "Settled for 2 of 8 days"+ : scenario === "running-child" || scenario === "pending-child"+ ? "Thread is running or has pending work"+ : "Not settled",+ });+ }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped),+ );++ it.live.each([+ "parent-unsettled",+ "parent-resettled",+ "child-unsettled",+ "intermediate-running",+ "intermediate-pending",+ ] as const)("cancels inherited settlement during measurement (%s)", (change) =>+ runCleanupTest(+ Effect.gen(function* () {+ const fixture = yield* pausedMeasurementFixture({+ ...DEFAULT_SERVER_SETTINGS.storageCleanup,+ worktreeSettledAfterDays: 0,+ });+ const root = shell({ settledAt: at(-20 * DAY_MS), settledOverride: "settled" });+ const middle = shell({+ id: ThreadId.make("middle"),+ lineage: {+ parentThreadId: root.id,+ rootThreadId: root.id,+ relationshipToParent: "subagent",+ },+ });+ const child = shell({+ id: ThreadId.make("child"),+ branch: "feature",+ worktreePath: fixture.worktree,+ lineage: {+ parentThreadId: middle.id,+ rootThreadId: root.id,+ relationshipToParent: "subagent",+ },+ });+ fixture.setThreads([child, middle, root]);+ const run = yield* fixture.service.runNow.pipe(Effect.forkChild);+ yield* Deferred.await(fixture.measuring);+ fixture.setThreads([+ { ...child, settledOverride: change === "child-unsettled" ? "active" : null },+ {+ ...middle,+ status: change === "intermediate-running" ? "running" : "idle",+ pendingBackgroundTasks:+ change === "intermediate-pending" ? [{ taskId: "task", kind: "command" }] : [],+ },+ change === "child-unsettled" || change.startsWith("intermediate-")+ ? root+ : {+ ...root,+ settledAt: change === "parent-resettled" ? at(0) : null,+ settledOverride: change === "parent-resettled" ? "settled" : "active",+ updatedAt: at(0),+ },+ ]);+ yield* Deferred.succeed(fixture.resume, undefined);+ expect((yield* Fiber.join(run)).entries[0]).toMatchObject({+ outcome: "kept",+ reason: "Parent settlement changed since check",+ });+ expect(yield* fixture.fs.exists(fixture.worktree)).toBe(true);+ }),+ ),+ );+it.effect.each(["idle","grandchild",diff --git a/apps/server/src/storageCleanup.ts b/apps/server/src/storageCleanup.tsindex cf3759519c6..9714d9175af 100644--- a/apps/server/src/storageCleanup.ts+++ b/apps/server/src/storageCleanup.ts@@ -14,6 +14,7 @@ import type {WorktreeCleanupRules,StorageCleanupReport,StorageCleanupReportEntry,+ ThreadId,} from "@t3tools/contracts";import { resolveWorktreeCleanup } from "@t3tools/shared/projectSettings";import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker";@@ -124,9 +125,16 @@ export function storageCleanupThreadIdle(thread: OrchestrationV2ThreadShell,now: number,settings: LimitRecoverySettings,+): boolean {+ return thread.worktreePath !== null && threadIdle(thread, now, settings);+}++function threadIdle(+ thread: OrchestrationV2ThreadShell,+ now: number,+ settings: LimitRecoverySettings,): boolean {return (- thread.worktreePath !== null &&thread.activeRunId === null &&(thread.status === "idle" ||thread.status === "completed" ||@@ -213,6 +221,36 @@ function worktreeOwner<T extends Pick<OrchestrationV2ThreadShell, "id" | "lineagreturn owner;}+function worktreeSettlementChain(+ thread: OrchestrationV2ThreadShell,+ byId: ReadonlyMap<ThreadId, OrchestrationV2ThreadShell>,+ now: number,+ settings: LimitRecoverySettings,+) {+ const chain = [thread];+ let current = thread;+ while (current.settledAt === null) {+ if (+ current.lineage.relationshipToParent !== "subagent" ||+ current.settledOverride === "active" ||+ current.pinnedAt != null+ )+ return [];+ const parent = byId.get(current.lineage.parentThreadId!);+ if (+ parent === undefined ||+ chain.some((member) => member.id === parent.id) ||+ parent.projectId !== thread.projectId ||+ parent.lineage.rootThreadId !== thread.lineage.rootThreadId ||+ !threadIdle(parent, now, settings)+ )+ return [];+ chain.push(parent);+ current = parent;+ }+ return chain;+}+/** Squash/rebase merges require the merged PR to name this exact head commit. */export function storageCleanupPullRequestMerged(pullRequest: Pick<@@ -775,6 +813,7 @@ const make = Effect.gen(function* () {),)).filter((thread) => thread.worktreePath !== null && thread.branch !== null);const snapshot = yield* readThreads();+ const threadsById = new Map(snapshot.threads.map((thread) => [thread.id, thread]));const refreshedDefaultRefs = new Map<string, Set<string>>();const defaultBranches = new Map<string, string | null>();const groups = Map.groupBy(@@ -808,6 +847,10 @@ const make = Effect.gen(function* () {const group = groups.get(worktreePath) ?? [];const activityAt = Math.max(...group.map(storageCleanupActivityAt));const deleted = "workspaceRoot" in thread;+ const settlementChain = deleted+ ? []+ : worktreeSettlementChain(thread, threadsById, now, serverSettings);+ const settledThread = settlementChain.at(-1);const members = deleted ? [thread, ...group] : group;const project = deleted? { workspaceRoot: thread.workspaceRoot }@@ -862,7 +905,7 @@ const make = Effect.gen(function* () {const changes = yield* localChangesReason(worktreePath, settings);if (changes !== null) return keep(changes);if (- group.some(+ [...group, ...settlementChain.slice(1)].some((member) =>member.settledAt !== null &&now - Math.max(DateTime.toEpochMillis(member.updatedAt), activityAt) <@@ -876,9 +919,9 @@ const make = Effect.gen(function* () {settings.worktreeAfterDays !== null &&activityAt < now - settings.worktreeAfterDays * DAY_MS;const settledFor =- deleted || thread.settledAt === null+ settledThread?.settledAt == null? null- : now - Math.max(DateTime.toEpochMillis(thread.settledAt), activityAt);+ : now - Math.max(DateTime.toEpochMillis(settledThread.settledAt), activityAt);const settledDays = settledFor === null ? null : Math.floor(settledFor / DAY_MS);const settled =settledFor !== null &&@@ -1045,6 +1088,27 @@ const make = Effect.gen(function* () {}))return waitForStop("thread activity or shared worktree changed since check");+ if (settlementChain.length > 1) {+ const ancestors = yield* Effect.forEach(settlementChain.slice(1), (ancestor) =>+ projections.getThreadShell(ancestor.id),+ );+ const latestSettlement = worktreeSettlementChain(+ latestGroup.find((shell) => shell.id === thread.id)!,+ new Map(+ [...latestGroup, ...ancestors.filter((ancestor) => ancestor !== null)].map(+ (ancestor) => [ancestor.id, ancestor],+ ),+ ),+ now,+ serverSettings,+ ).at(-1);+ if (+ latestSettlement?.id !== settledThread?.id ||+ !Equal.equals(latestSettlement?.settledAt, settledThread?.settledAt) ||+ !Equal.equals(latestSettlement?.updatedAt, settledThread?.updatedAt)+ )+ return "parent settlement changed since check";+ }// Sessions can outlive their run and can be shared across app threads.const sessionRows = yield* sql<{ payload_json: string }>`SELECT payload_json FROM orchestration_v2_projection_provider_sessions