Skip to content
TypeScript

fix(server): clean up child worktrees after their parent settles (#30026)

Lakshmi Tanmayb6307ee88a72
diff --git a/apps/server/src/storageCleanup.test.ts b/apps/server/src/storageCleanup.test.ts
index 937af138d9d..b4c9ac3e689 100644
--- a/apps/server/src/storageCleanup.test.ts
+++ b/apps/server/src/storageCleanup.test.ts
@@ -778,6 +778,162 @@ it.live("preserves late tracked edits when submodules were never initialized", (
);
describe("subagent worktree ownership", () => {
+ it.effect.each([
+ "child",
+ "grandchild",
+ "immediate",
+ "active-parent",
+ "recent-settle",
+ "settle-grace",
+ "recent-child-activity",
+ "running-child",
+ "pending-child",
+ "running-parent",
+ "pending-parent",
+ "unsettled-child",
+ "pinned-child",
+ "fork",
+ "missing-parent",
+ "cycle",
+ ] as const)("checks isolated subagent settlement (%s)", (scenario) =>
+ Effect.gen(function* () {
+ yield* TestClock.setTime(NOW_MS);
+ const fixture = yield* cleanupFixture({
+ ...DEFAULT_SERVER_SETTINGS.storageCleanup,
+ worktreeSettledAfterDays: scenario === "immediate" ? 0 : 8,
+ });
+ const root = shell({
+ settledAt:
+ scenario === "active-parent"
+ ? null
+ : scenario === "recent-settle" || scenario === "settle-grace"
+ ? at(-30_000)
+ : scenario === "immediate"
+ ? at(-120_000)
+ : at(-20 * DAY_MS),
+ settledOverride: scenario === "active-parent" ? null : "settled",
+ updatedAt: scenario === "settle-grace" ? at(-30_000) : at(-10 * DAY_MS),
+ });
+ const child = shell({
+ id: ThreadId.make("child"),
+ branch: "feature",
+ worktreePath: fixture.worktree,
+ lineage: {
+ parentThreadId:
+ scenario === "missing-parent"
+ ? ThreadId.make("missing")
+ : scenario === "cycle"
+ ? ThreadId.make("child")
+ : ["grandchild", "running-parent", "pending-parent"].includes(scenario)
+ ? ThreadId.make("middle")
+ : root.id,
+ rootThreadId: root.id,
+ relationshipToParent: scenario === "fork" ? "fork" : "subagent",
+ },
+ status: scenario === "running-child" ? "running" : "completed",
+ pendingBackgroundTasks:
+ scenario === "pending-child" ? [{ taskId: "task", kind: "command" }] : [],
+ latestRunCompletedAt: scenario === "recent-child-activity" ? at(-2 * DAY_MS) : null,
+ settledOverride: scenario === "unsettled-child" ? "active" : null,
+ pinnedAt: scenario === "pinned-child" ? at(-DAY_MS) : null,
+ });
+ const middle = shell({
+ id: ThreadId.make("middle"),
+ status: scenario === "running-parent" ? "running" : "idle",
+ pendingBackgroundTasks:
+ scenario === "pending-parent" ? [{ taskId: "task", kind: "command" }] : [],
+ lineage: {
+ parentThreadId: root.id,
+ rootThreadId: root.id,
+ relationshipToParent: "subagent",
+ },
+ });
+ fixture.setThreads([child, middle, root]);
+ const report = yield* fixture.service.runNow;
+ const removed = ["child", "grandchild", "immediate"].includes(scenario);
+ expect(yield* fixture.fs.exists(fixture.worktree)).toBe(!removed);
+ expect(report.entries[0]).toMatchObject({
+ threadId: child.id,
+ outcome: removed ? "removed" : "kept",
+ reason: removed
+ ? scenario === "immediate"
+ ? "Settled for 0 days"
+ : "Settled for 20 days"
+ : scenario === "settle-grace"
+ ? "Settled moments ago"
+ : scenario === "recent-settle"
+ ? "Settled for 0 of 8 days"
+ : scenario === "recent-child-activity"
+ ? "Settled for 2 of 8 days"
+ : scenario === "running-child" || scenario === "pending-child"
+ ? "Thread is running or has pending work"
+ : "Not settled",
+ });
+ }).pipe(Effect.provide(cleanupTestLayer), Effect.scoped),
+ );
+
+ it.live.each([
+ "parent-unsettled",
+ "parent-resettled",
+ "child-unsettled",
+ "intermediate-running",
+ "intermediate-pending",
+ ] as const)("cancels inherited settlement during measurement (%s)", (change) =>
+ runCleanupTest(
+ Effect.gen(function* () {
+ const fixture = yield* pausedMeasurementFixture({
+ ...DEFAULT_SERVER_SETTINGS.storageCleanup,
+ worktreeSettledAfterDays: 0,
+ });
+ const root = shell({ settledAt: at(-20 * DAY_MS), settledOverride: "settled" });
+ const middle = shell({
+ id: ThreadId.make("middle"),
+ lineage: {
+ parentThreadId: root.id,
+ rootThreadId: root.id,
+ relationshipToParent: "subagent",
+ },
+ });
+ const child = shell({
+ id: ThreadId.make("child"),
+ branch: "feature",
+ worktreePath: fixture.worktree,
+ lineage: {
+ parentThreadId: middle.id,
+ rootThreadId: root.id,
+ relationshipToParent: "subagent",
+ },
+ });
+ fixture.setThreads([child, middle, root]);
+ const run = yield* fixture.service.runNow.pipe(Effect.forkChild);
+ yield* Deferred.await(fixture.measuring);
+ fixture.setThreads([
+ { ...child, settledOverride: change === "child-unsettled" ? "active" : null },
+ {
+ ...middle,
+ status: change === "intermediate-running" ? "running" : "idle",
+ pendingBackgroundTasks:
+ change === "intermediate-pending" ? [{ taskId: "task", kind: "command" }] : [],
+ },
+ change === "child-unsettled" || change.startsWith("intermediate-")
+ ? root
+ : {
+ ...root,
+ settledAt: change === "parent-resettled" ? at(0) : null,
+ settledOverride: change === "parent-resettled" ? "settled" : "active",
+ updatedAt: at(0),
+ },
+ ]);
+ yield* Deferred.succeed(fixture.resume, undefined);
+ expect((yield* Fiber.join(run)).entries[0]).toMatchObject({
+ outcome: "kept",
+ reason: "Parent settlement changed since check",
+ });
+ expect(yield* fixture.fs.exists(fixture.worktree)).toBe(true);
+ }),
+ ),
+ );
+
it.effect.each([
"idle",
"grandchild",
diff --git a/apps/server/src/storageCleanup.ts b/apps/server/src/storageCleanup.ts
index cf3759519c6..9714d9175af 100644
--- a/apps/server/src/storageCleanup.ts
+++ b/apps/server/src/storageCleanup.ts
@@ -14,6 +14,7 @@ import type {
WorktreeCleanupRules,
StorageCleanupReport,
StorageCleanupReportEntry,
+ ThreadId,
} from "@t3tools/contracts";
import { resolveWorktreeCleanup } from "@t3tools/shared/projectSettings";
import { makeDrainableWorker } from "@t3tools/shared/DrainableWorker";
@@ -124,9 +125,16 @@ export function storageCleanupThreadIdle(
thread: OrchestrationV2ThreadShell,
now: number,
settings: LimitRecoverySettings,
+): boolean {
+ return thread.worktreePath !== null && threadIdle(thread, now, settings);
+}
+
+function threadIdle(
+ thread: OrchestrationV2ThreadShell,
+ now: number,
+ settings: LimitRecoverySettings,
): boolean {
return (
- thread.worktreePath !== null &&
thread.activeRunId === null &&
(thread.status === "idle" ||
thread.status === "completed" ||
@@ -213,6 +221,36 @@ function worktreeOwner<T extends Pick<OrchestrationV2ThreadShell, "id" | "lineag
return owner;
}
+function worktreeSettlementChain(
+ thread: OrchestrationV2ThreadShell,
+ byId: ReadonlyMap<ThreadId, OrchestrationV2ThreadShell>,
+ now: number,
+ settings: LimitRecoverySettings,
+) {
+ const chain = [thread];
+ let current = thread;
+ while (current.settledAt === null) {
+ if (
+ current.lineage.relationshipToParent !== "subagent" ||
+ current.settledOverride === "active" ||
+ current.pinnedAt != null
+ )
+ return [];
+ const parent = byId.get(current.lineage.parentThreadId!);
+ if (
+ parent === undefined ||
+ chain.some((member) => member.id === parent.id) ||
+ parent.projectId !== thread.projectId ||
+ parent.lineage.rootThreadId !== thread.lineage.rootThreadId ||
+ !threadIdle(parent, now, settings)
+ )
+ return [];
+ chain.push(parent);
+ current = parent;
+ }
+ return chain;
+}
+
/** Squash/rebase merges require the merged PR to name this exact head commit. */
export function storageCleanupPullRequestMerged(
pullRequest: Pick<
@@ -775,6 +813,7 @@ const make = Effect.gen(function* () {
),
)).filter((thread) => thread.worktreePath !== null && thread.branch !== null);
const snapshot = yield* readThreads();
+ const threadsById = new Map(snapshot.threads.map((thread) => [thread.id, thread]));
const refreshedDefaultRefs = new Map<string, Set<string>>();
const defaultBranches = new Map<string, string | null>();
const groups = Map.groupBy(
@@ -808,6 +847,10 @@ const make = Effect.gen(function* () {
const group = groups.get(worktreePath) ?? [];
const activityAt = Math.max(...group.map(storageCleanupActivityAt));
const deleted = "workspaceRoot" in thread;
+ const settlementChain = deleted
+ ? []
+ : worktreeSettlementChain(thread, threadsById, now, serverSettings);
+ const settledThread = settlementChain.at(-1);
const members = deleted ? [thread, ...group] : group;
const project = deleted
? { workspaceRoot: thread.workspaceRoot }
@@ -862,7 +905,7 @@ const make = Effect.gen(function* () {
const changes = yield* localChangesReason(worktreePath, settings);
if (changes !== null) return keep(changes);
if (
- group.some(
+ [...group, ...settlementChain.slice(1)].some(
(member) =>
member.settledAt !== null &&
now - Math.max(DateTime.toEpochMillis(member.updatedAt), activityAt) <
@@ -876,9 +919,9 @@ const make = Effect.gen(function* () {
settings.worktreeAfterDays !== null &&
activityAt < now - settings.worktreeAfterDays * DAY_MS;
const settledFor =
- deleted || thread.settledAt === null
+ settledThread?.settledAt == null
? null
- : now - Math.max(DateTime.toEpochMillis(thread.settledAt), activityAt);
+ : now - Math.max(DateTime.toEpochMillis(settledThread.settledAt), activityAt);
const settledDays = settledFor === null ? null : Math.floor(settledFor / DAY_MS);
const settled =
settledFor !== null &&
@@ -1045,6 +1088,27 @@ const make = Effect.gen(function* () {
})
)
return waitForStop("thread activity or shared worktree changed since check");
+ if (settlementChain.length > 1) {
+ const ancestors = yield* Effect.forEach(settlementChain.slice(1), (ancestor) =>
+ projections.getThreadShell(ancestor.id),
+ );
+ const latestSettlement = worktreeSettlementChain(
+ latestGroup.find((shell) => shell.id === thread.id)!,
+ new Map(
+ [...latestGroup, ...ancestors.filter((ancestor) => ancestor !== null)].map(
+ (ancestor) => [ancestor.id, ancestor],
+ ),
+ ),
+ now,
+ serverSettings,
+ ).at(-1);
+ if (
+ latestSettlement?.id !== settledThread?.id ||
+ !Equal.equals(latestSettlement?.settledAt, settledThread?.settledAt) ||
+ !Equal.equals(latestSettlement?.updatedAt, settledThread?.updatedAt)
+ )
+ return "parent settlement changed since check";
+ }
// Sessions can outlive their run and can be shared across app threads.
const sessionRows = yield* sql<{ payload_json: string }>`
SELECT payload_json FROM orchestration_v2_projection_provider_sessions