Skip to content

armgit / qm

Public Imported from GitHub
0

Multiplayer agent harness for work.

28.8 KB
Raw Blame
import { hashId } from "../../util/crypto.ts";
import type { Principal, Resolution, ScopeId, Session } from "../../types.ts";
import { personalScope } from "../../types.ts";
import { intersectEgressPolicies } from "../../resolution/egress-policy.ts";
import { isOpenScopeMember } from "../../resolution/sharing-access.ts";
import type { GapPhase } from "../../sessions/session-store.ts";
import { type SandboxHandle, supportsProcessSessions, SandboxProvisionCleanupError } from "../../sandbox/sandbox.ts";
import { accessConversationSandbox, type SandboxAccessPlan } from "../../sandbox/sandbox-resources.ts";
import { reconcileProcesses } from "../../processes/reconcile.ts";
import { leaseLapsed } from "../../runs/run-store.ts";
import { finishProcessCredentials } from "../../credentials/execute-files.ts";
import {
  materializeSkillTree as laySkillTree,
  packRoot,
  rehomeSkillPaths,
  renderSkillBody,
  skillDir,
  SKILLS_ROOT,
} from "../../skills/materialize.ts";
import { safeSkillFilePath, type SkillResolution } from "../../skills/skill-store.ts";
import { isSafeSkillName } from "../../skills/skill-name.ts";
import type { SkillResult } from "../../tools/primitives.ts";
import { TURN_FILES_DIR } from "../attachments.ts";
import { errMessage, swallow, swallowAs } from "../../util/errors.ts";
import { sleep } from "../../util/async.ts";
import { loadActiveBundles } from "./turn-helpers.ts";
import type { OrchestratorDeps, OrchestratorInput } from "./types.ts";

const TURN_FILES_MAX_AGE_MS = 24 * 60 * 60_000;

export interface TurnSandboxContext {
  deps: OrchestratorDeps;
  input: OrchestratorInput;
  actor: Principal;
  session: Session;
  resolution: Resolution;
  scopeId: ScopeId;
  memoryScopeId: ScopeId;
  transferId: string;
  turnSessionDir: string;
  turnFilesDir: string;
  connectorEnv: Record<string, string>;
  egressTokenForTurn: string | undefined;
  egressTokenForPolicy?: (policy: Resolution["egress"]) => Promise<string | undefined>;
  openSpeakerKeychain?: boolean;
  openResourceAccess?: boolean;
  ownerAuthAvailable: boolean;
  visibleSkillsForTurn: (sources?: readonly string[]) => Promise<SkillResolution[]>;
  emitGapWork: (phase: GapPhase, start: number, end: number) => void;
}

export function createTurnSandboxes(ctx: TurnSandboxContext) {
  const {
    deps,
    input,
    actor,
    session,
    resolution,
    scopeId,
    memoryScopeId,
    transferId,
    turnSessionDir,
    turnFilesDir,
    connectorEnv,
    egressTokenForTurn,
    egressTokenForPolicy,
    openSpeakerKeychain,
    openResourceAccess,
    ownerAuthAvailable,
    visibleSkillsForTurn,
    emitGapWork,
  } = ctx;

  let ownerAuthCommand: ((command: string, env?: Record<string, string>) => string) | undefined;
  const brokerEnvKeys = [
    "AWS_ACCESS_KEY_ID",
    "AWS_SECRET_ACCESS_KEY",
    "AWS_SESSION_TOKEN",
    "AWS_REGION",
    "AWS_DEFAULT_REGION",
  ];
  const unsetBrokerEnv = (env: Record<string, string>): string => {
    const keys = brokerEnvKeys.filter((key) => !(key in env));
    return keys.length ? `unset ${keys.join(" ")}; ` : "";
  };
  if (ownerAuthAvailable) {
    ownerAuthCommand = (command, env = {}) => {
      if (openSpeakerKeychain)
        deps.auditLog.record({
          at: Date.now(),
          principalId: actor.id,
          action: "keychain.open_speaker_use",
          resource: "isolated owner execution",
          scopeLabel: scopeId,
        });
      return `unset AGENT_API_TOKEN AGENT_OAUTH_CONSENT_TOKEN AGENT_CREDENTIAL_TOKEN; ${unsetBrokerEnv(env)}${command}`;
    };
  }
  const box: {
    handle: SandboxHandle | null;
    pending: SandboxHandle | null;
    used: boolean;
    provisionMs?: number;
  } = { handle: null, pending: null, used: false };
  const scratchBox: { handle: SandboxHandle | null; pending: SandboxHandle | null; provisionMs?: number } = {
    handle: null,
    pending: null,
  };
  let scratchProvisionInFlight: Promise<SandboxHandle> | null = null;
  const scratchKey = () => `turn:${session.id}:${transferId}`;
  let scratchStartedAt: number | undefined;
  let scratchReadyAt: number | undefined;
  const recordScratchLifecycle = (
    action: string,
    handle?: SandboxHandle,
    extra?: { releasedAt?: number; cleanupMs?: number; error?: string },
  ): void => {
    deps.auditLog?.record({
      at: Date.now(),
      principalId: actor.id,
      action: `sandbox.scratch.${action}`,
      resource: scratchKey(),
      scopeLabel: scopeId,
      detail: JSON.stringify({
        runId: input.runId,
        sessionId: session.id,
        backend: handle?.backend,
        sandboxId: handle?.id,
        startedAt: scratchStartedAt,
        readyAt: scratchReadyAt,
        provisionMs: scratchBox.provisionMs,
        ...extra,
      }),
    });
  };
  const ownerAuthBox: { handle: SandboxHandle | null; pending: SandboxHandle | null } = {
    handle: null,
    pending: null,
  };
  let ownerAuthProvisionInFlight: Promise<SandboxHandle> | null = null;
  const destroyEphemeralHandle = async (handle: SandboxHandle): Promise<void> => {
    const errors: unknown[] = [];
    for (let attempt = 1; attempt <= 3; attempt++) {
      try {
        await deps.sandbox.teardown(handle, { destroy: true });
        return;
      } catch (error) {
        errors.push(error);
        if (attempt < 3) await sleep(50 * attempt);
      }
    }
    throw new AggregateError(errors, "Disposable sandbox destruction failed");
  };
  let sandboxStatusSeq = 2_000_000;
  const onSandboxStatus =
    input.runId && deps.runActivity
      ? (text: string): void => {
          void deps
            .runActivity!.append(input.runId!, {
              seq: sandboxStatusSeq++,
              parentSeq: null,
              type: "sandbox_status",
              payload: { text },
              createdAt: Date.now(),
            })
            .catch(swallowAs("orchestrator: sandbox status append", undefined));
        }
      : undefined;
  const resourceHandles = new Map<string, SandboxHandle>();
  const resourcePolicy = new Map<string, string>();
  const resourcePendingHandles = new Map<string, SandboxHandle>();
  const resourcePending = new Map<string, Promise<SandboxHandle>>();
  let provisionInFlight: Promise<SandboxHandle> | null = null;
  const holdResource = async (handle: SandboxHandle): Promise<void> => {
    if (handle.resourceId && deps.runs && input.runId)
      await deps.sandboxResources?.hold(handle.resourceId, input.runId);
  };
  const releaseHold = async (handle: SandboxHandle): Promise<boolean> => {
    const runs = deps.runs;
    if (!handle.resourceId || !runs || !deps.sandboxResources) return false;
    return deps.sandboxResources.release(handle.resourceId, input.runId, async (runId) => {
      const run = await runs.get(runId);
      return run?.status === "running" && !leaseLapsed(run, Date.now());
    });
  };
  const provision = (eager = false): Promise<SandboxHandle> => {
    if (!eager) box.used = true;
    provisionInFlight ??= doProvision(eager ? () => {} : emitGapWork, eager).catch((err) => {
      provisionInFlight = null;
      throw err;
    });
    return provisionInFlight;
  };
  const doProvision = async (emit: typeof emitGapWork, eager: boolean): Promise<SandboxHandle> => {
    const provisionStart = Date.now();
    if (input.externalSlack) {
      const resource = await deps.sandboxResources?.resolve(memoryScopeId);
      if (resource && resource.ownerScopeId !== scopeId) throw new Error("External Slack requires its own sandbox.");
    }
    const swarmBinding = await deps.swarms?.binding(input);
    const handle = await deps.sandbox.provision(resolution.layers, {
      ...(swarmBinding?.sandboxId ? { sandboxId: swarmBinding.sandboxId } : {}),
      env: connectorEnv,
      egress: resolution.egress,
      ...(egressTokenForTurn ? { egressToken: egressTokenForTurn } : {}),
      ...(onSandboxStatus ? { onStatus: onSandboxStatus } : {}),
    });
    box.pending = handle;
    await holdResource(handle);
    emit("provision", provisionStart, Date.now());
    box.provisionMs = Date.now() - provisionStart;
    deps.auditLog?.record({
      at: Date.now(),
      principalId: actor.id,
      action: "sandbox.scoped.provision_ready",
      resource: scratchKey(),
      scopeLabel: scopeId,
      detail: JSON.stringify({
        runId: input.runId,
        sessionId: session.id,
        backend: handle.backend,
        sandboxId: handle.id,
        provisionMs: box.provisionMs,
        coldStart: handle.coldStart,
        eager,
      }),
    });
    const dirCleanupStart = Date.now();
    await prepareTurnFiles(handle);
    emit("dir_cleanup", dirCleanupStart, Date.now());
    if (deps.processes && supportsProcessSessions(deps.sandbox)) {
      const procReconcileStart = Date.now();
      try {
        await reconcileProcesses(deps.sandbox, handle, deps.processes, memoryScopeId, (exited, processId) =>
          finishProcessCredentials(
            {
              sandbox: deps.sandbox,
              processes: deps.processes!,
              ...(deps.keychain ? { keychain: deps.keychain } : {}),
            },
            exited,
            processId,
          ),
        );
      } catch (err) {
        deps.errors?.record(
          {
            category: "process_session",
            code: "reconcile_failed",
            message: errMessage(err),
            scopeLabel: scopeId,
            sessionId: session.id,
          },
          err,
        );
      } finally {
        emit("proc_reconcile", procReconcileStart, Date.now());
      }
    }
    box.handle = handle;
    return handle;
  };
  const skillsRoot = (r: SkillResolution) =>
    `${SKILLS_ROOT}/${turnSessionDir.split("/").at(-1)}/${hashId([r.skill!.id, String(r.skill!.version)], 16)}`;
  const laidTrees = new Set<string>();
  const materializeSkillTree = async (handle: SandboxHandle, r: SkillResolution): Promise<void> => {
    const treeKey = `${handle.resourceId ?? handle.id}:${skillDir(skillsRoot(r), r)}`;
    if (laidTrees.has(treeKey)) return;
    const start = Date.now();
    try {
      const bundles = r.screenedBundles ?? (deps.skillBundles ? await loadActiveBundles(deps.skillBundles, [r]) : []);
      await laySkillTree(deps.sandbox, handle, skillsRoot(r), r, bundles);
      laidTrees.add(treeKey);
    } catch (err) {
      deps.errors?.record(
        {
          category: "skills",
          code: "tree_materialize_failed",
          message: errMessage(err),
          scopeLabel: scopeId,
          sessionId: session.id,
        },
        err,
      );
      throw err;
    } finally {
      emitGapWork("skills_materialize", start, Date.now());
    }
  };
  const useSkill = async (name: string, file: string, sandboxId?: string, source?: string): Promise<SkillResult> => {
    const missing = { content: null, sourceScopeId: null };
    if (!isSafeSkillName(name)) return missing;
    try {
      if (safeSkillFilePath(file) !== file) return missing;
    } catch {
      return missing;
    }
    const resolution = (await visibleSkillsForTurn(source ? [source] : undefined)).find(
      (r) => r.skill?.manifest.name === name,
    );
    if (!resolution?.skill) return missing;
    const shipsFiles = (resolution.skill.manifest.files?.length ?? 0) > 0 || resolution.skill.pack !== undefined;
    const asset = resolution.skill.manifest.files?.find((f) => {
      try {
        return safeSkillFilePath(f.path) === file;
      } catch {
        return false;
      }
    })?.content;
    if (file !== "SKILL.md" && asset === undefined) return missing;
    if (deps.skills)
      void deps.skills.recordUse(resolution.skill.id).catch((e) => swallow("orchestrator: skill recordUse", e));
    const render = (root?: string) =>
      file === "SKILL.md"
        ? renderSkillBody(resolution, root)
        : rehomeSkillPaths(resolution, asset!, root ?? skillsRoot(resolution));
    if (!shipsFiles) return { content: render(), sourceScopeId: resolution.skill.scopeId };
    const access = sandboxId !== undefined ? await accessResource(sandboxId) : undefined;
    if (access?.crossScope) return { content: render(skillsRoot(resolution)), sourceScopeId: resolution.skill.scopeId };
    const handle = access ? await provisionResource(access) : await provision();
    await materializeSkillTree(handle, resolution);
    const root = `${handle.rootDir}/${skillsRoot(resolution)}`;
    const pack = packRoot(root, resolution);
    return {
      content: render(root),
      sourceScopeId: resolution.skill.scopeId,
      dir: skillDir(root, resolution),
      ...(pack ? { packDir: pack } : {}),
    };
  };
  const writableScopeId = resolution.layers.find((layer) => layer.mode === "rw")?.scopeId;
  const canUseSandboxScope = async (target: ScopeId): Promise<boolean> => {
    if (target === writableScopeId || target === scopeId) return true;
    if (!openResourceAccess || actor.type !== "internal" || !deps.config || !deps.isCurrentSharedScopeMember)
      return false;
    const personal = personalScope(actor.id);
    for (const scope of new Set([scopeId, target])) {
      if (scope === personal) {
        if ((await deps.config.resolveSharingPostureDurable(personal, scope)) !== "open") return false;
      } else if (
        !(await isOpenScopeMember({
          actorId: actor.id,
          scope,
          config: deps.config,
          isCurrentSharedScopeMember: deps.isCurrentSharedScopeMember,
        }))
      )
        return false;
    }
    return true;
  };
  const accessResource = async (id: string): Promise<SandboxAccessPlan> => {
    const resource = await accessConversationSandbox(deps.sandboxResources, actor.id, id, canUseSandboxScope);
    const crossScope = resource.ownerScopeId !== writableScopeId && resource.ownerScopeId !== scopeId;
    if (!crossScope) return { resource, crossScope: false, egress: resolution.egress, commandPolicy: null };
    await deps.config!.refreshSecurity([resource.ownerScopeId]);
    const credentialScopeId = resource.ownerScopeId === personalScope(actor.id) ? resource.ownerScopeId : undefined;
    return {
      resource,
      crossScope: true,
      egress: intersectEgressPolicies(resolution.egress, deps.config!.getEgress(resource.ownerScopeId)),
      commandPolicy: deps.config!.getCommandPolicy(resource.ownerScopeId),
      ...(credentialScopeId ? { credentialScopeId } : {}),
    };
  };
  const provisionResource = async (
    input: string | SandboxAccessPlan,
    authorize?: (access: SandboxAccessPlan) => void,
  ): Promise<SandboxHandle> => {
    const access = typeof input === "string" ? await accessResource(input) : input;
    const { resource, crossScope, egress, credentialScopeId } = access;
    const id = resource.id;
    const pending = resourcePending.get(id);
    if (pending) {
      await pending;
      return provisionResource(id, authorize);
    }
    authorize?.(access);
    const policyKey = JSON.stringify({ egress, credentialScopeId });
    const existing = resourceHandles.get(id);
    if (existing && resourcePolicy.get(id) === policyKey) {
      return existing;
    }
    const provisioned = (async () => {
      const layers = crossScope
        ? resolution.layers
            .filter((layer) => layer.mode === "rw" || layer.mountPath === "global")
            .map((layer) => (layer.mode === "rw" ? { ...layer, scopeId: resource.ownerScopeId } : layer))
        : resolution.layers;
      if (crossScope && egressTokenForTurn && !egressTokenForPolicy)
        throw new Error("target sandbox egress authorization unavailable");
      const egressToken = crossScope ? await egressTokenForPolicy?.(egress) : egressTokenForTurn;
      const handle = await deps.sandbox.provision(layers, {
        sandboxId: id,
        ...(!crossScope ? { env: connectorEnv } : {}),
        ...(access.env ? { env: { ...access.env } } : {}),
        egress,
        ...(egressToken ? { egressToken } : {}),
      });
      resourcePendingHandles.set(id, handle);
      await holdResource(handle);
      if (!crossScope) {
        await prepareTurnFiles(handle);
      }
      resourceHandles.set(id, handle);
      resourcePolicy.set(id, policyKey);
      resourcePendingHandles.delete(id);
      return handle;
    })().finally(() => {
      resourcePending.delete(id);
    });
    resourcePending.set(id, provisioned);
    return provisioned;
  };

  const provisionScratch = (): Promise<SandboxHandle> => {
    if (scratchBox.pending)
      return Promise.reject(new Error("Disposable sandbox initialization cleanup is still pending"));
    if (scratchBox.handle) return Promise.resolve(scratchBox.handle);
    scratchProvisionInFlight ??= (async () => {
      const provisionStart = Date.now();
      scratchStartedAt = provisionStart;
      recordScratchLifecycle("provision_started");
      const handle = await deps.sandbox.provision(
        resolution.layers.filter((l) => l.mode === "ro" && l.mountPath === "global"),
        {
          env: connectorEnv,
          egress: resolution.egress,
          ...(egressTokenForTurn ? { egressToken: egressTokenForTurn } : {}),
          scratch: { key: scratchKey() },
          routeScopeId: memoryScopeId,
          ...(onSandboxStatus ? { onStatus: onSandboxStatus } : {}),
        },
      );
      scratchBox.provisionMs = Date.now() - provisionStart;
      scratchBox.handle = handle;
      scratchReadyAt = Date.now();
      recordScratchLifecycle("provision_ready", handle);
      return handle;
    })().catch((error) => {
      scratchProvisionInFlight = null;
      if (error instanceof SandboxProvisionCleanupError) scratchBox.pending = error.handle;
      recordScratchLifecycle("provision_failed", scratchBox.pending ?? undefined, { error: errMessage(error) });
      throw error;
    });
    return scratchProvisionInFlight;
  };
  const provisionOwnerAuth = ownerAuthAvailable
    ? async (): Promise<SandboxHandle> => {
        // Recheck each command, including commands reusing this turn's isolated computer.
        if (
          openSpeakerKeychain &&
          (!(await deps.isCurrentSharedScopeMember?.(actor.id, scopeId)) ||
            (await deps.config?.resolveSharingPostureDurable(personalScope(actor.id), scopeId)) !== "open")
        )
          throw new Error("Open speaker keychain access is no longer authorized");
        if (ownerAuthBox.handle) return ownerAuthBox.handle;
        if (ownerAuthBox.pending && !ownerAuthProvisionInFlight) {
          return Promise.reject(new Error("owner-auth box initialization failed and cleanup is still pending"));
        }
        ownerAuthProvisionInFlight ??= (async () => {
          const handle = await deps.sandbox.provision(
            resolution.layers.filter((l) => l.mode === "ro" && l.mountPath === "global"),
            {
              egress: resolution.egress,
              ...(egressTokenForTurn ? { egressToken: egressTokenForTurn } : {}),
              scratch: { key: `owner-auth:${session.id}:${transferId}` },
              routeScopeId: memoryScopeId,
              ...(onSandboxStatus ? { onStatus: onSandboxStatus } : {}),
            },
          );
          ownerAuthBox.pending = handle;
          ownerAuthBox.handle = handle;
          return handle;
        })().catch(async (err) => {
          ownerAuthProvisionInFlight = null;
          if (err instanceof SandboxProvisionCleanupError) ownerAuthBox.pending = err.handle;
          const pendingHandle = ownerAuthBox.pending;
          if (pendingHandle) {
            try {
              await destroyEphemeralHandle(pendingHandle);
              if (ownerAuthBox.pending === pendingHandle) ownerAuthBox.pending = null;
            } catch (cleanupErr) {
              deps.errors?.record(
                {
                  category: "sandbox",
                  code: "owner_auth_init_cleanup_failed",
                  message: errMessage(cleanupErr),
                  scopeLabel: scopeId,
                  sessionId: session.id,
                },
                cleanupErr,
              );
            }
          }
          throw err;
        });
        return ownerAuthProvisionInFlight;
      }
    : undefined;
  const reachBoxes = new Map<ScopeId, SandboxHandle>();
  const provisionForReach = async (target: ScopeId): Promise<SandboxHandle> => {
    const cached = reachBoxes.get(target);
    if (cached) return cached;
    const handle = await deps.sandbox.provision(
      [
        { scopeId: resolution.orgScopeId, mountPath: "global", mode: "ro" },
        { scopeId: target, mountPath: "", mode: "rw" },
      ],
      {
        egress: resolution.egress,
        ...(egressTokenForTurn ? { egressToken: egressTokenForTurn } : {}),
        ...(onSandboxStatus ? { onStatus: onSandboxStatus } : {}),
      },
    );
    reachBoxes.set(target, handle);
    await holdResource(handle);
    return handle;
  };
  const clearTurnFiles = async (handle: SandboxHandle): Promise<void> => {
    for (let attempt = 1; attempt <= 3; attempt++) {
      try {
        await deps.sandbox.removeDir(handle, turnFilesDir);
        return;
      } catch (err) {
        if (attempt === 3) swallow("orchestrator: turn file cleanup", err);
        else await sleep(50);
      }
    }
  };
  const pruneSkills = async (handle: SandboxHandle): Promise<void> => {
    const visible = (await visibleSkillsForTurn()).flatMap((r) => (r.skill ? [r.skill, ...r.shadowed] : []));
    const keep = new Set(visible.map((skill) => hashId([skill.id, String(skill.version)], 16)));
    const stale = new Set<string>();
    const listed = await deps.sandbox
      .listDir(handle, SKILLS_ROOT)
      .catch(swallowAs("orchestrator: list skills to prune", [] as string[]));
    for (const path of listed) {
      const [root, conversation, identity] = path.split("/");
      if (root === SKILLS_ROOT && conversation && identity && !keep.has(identity))
        stale.add(`${SKILLS_ROOT}/${conversation}/${identity}`);
    }
    await Promise.all(
      [...stale].map((dir) =>
        deps.sandbox.removeDir(handle, dir).catch(swallowAs("orchestrator: stale skill cleanup", undefined)),
      ),
    );
  };
  const clearStaleTurnFiles = async (handle: SandboxHandle): Promise<void> => {
    const cutoff = Date.now() - TURN_FILES_MAX_AGE_MS;
    const paths = deps.sandbox.removeDirAndList
      ? await deps.sandbox.removeDirAndList(handle, turnSessionDir, TURN_FILES_DIR)
      : await (async () => {
          await deps.sandbox.removeDir(handle, turnSessionDir);
          return deps.sandbox
            .listDir(handle, TURN_FILES_DIR)
            .catch(swallowAs("orchestrator: list stale turn files", [] as string[]));
        })();
    const stale = new Set<string>();
    for (const path of paths) {
      const parts = path.split("/");
      const startedAt = Number.parseInt(parts[2]?.split("-")[0] ?? "", 36);
      if (parts[0] === TURN_FILES_DIR && parts[1] && parts[2] && Number.isFinite(startedAt) && startedAt < cutoff) {
        stale.add(`${TURN_FILES_DIR}/${parts[1]}/${parts[2]}`);
      }
    }
    await Promise.all(
      [...stale].map((dir) =>
        deps.sandbox.removeDir(handle, dir).catch(swallowAs("orchestrator: stale turn file cleanup", undefined)),
      ),
    );
  };
  const prepareTurnFiles = async (handle: SandboxHandle): Promise<void> => {
    await Promise.all([pruneSkills(handle), clearStaleTurnFiles(handle)]);
  };
  const hasLiveProcesses = async (handle: SandboxHandle): Promise<boolean> => {
    if (!deps.processes || !handle.resourceId) return false;
    return (await deps.processes.listLive()).some((process) => process.sandboxId === handle.resourceId);
  };
  const reclaimBox = async (): Promise<void> => {
    let ownerCleanupError: unknown;
    let scratchCleanupError: unknown;
    if (ownerAuthProvisionInFlight)
      await ownerAuthProvisionInFlight.catch(swallowAs("orchestrator: owner auth provision", undefined));
    ownerAuthProvisionInFlight = null;
    const reachHandles = [...reachBoxes.values()];
    reachBoxes.clear();
    await Promise.all(
      reachHandles.map(async (h) => {
        let keepReachWarm = await releaseHold(h).catch(swallowAs("orchestrator: reach hold release", true));
        if (!keepReachWarm && deps.processes && supportsProcessSessions(deps.sandbox)) {
          try {
            keepReachWarm = await hasLiveProcesses(h);
          } catch (e) {
            swallow("orchestrator: reach live process check", e);
            keepReachWarm = false;
          }
        }
        if (keepReachWarm) {
          await deps.sandbox
            .teardown(h, { keepWarm: true })
            .catch(swallowAs("orchestrator: keep warm teardown", undefined));
          return;
        }
        await deps.sandbox.teardown(h).catch(swallowAs("orchestrator: teardown", undefined));
      }),
    );
    const ownerHandle = ownerAuthBox.handle ?? ownerAuthBox.pending;
    if (ownerHandle) {
      try {
        await destroyEphemeralHandle(ownerHandle);
        ownerAuthBox.handle = null;
        ownerAuthBox.pending = null;
      } catch (err) {
        ownerCleanupError = err;
        deps.errors?.record(
          {
            category: "sandbox",
            code: "owner_auth_destroy_failed",
            message: errMessage(err),
            scopeLabel: scopeId,
            sessionId: session.id,
          },
          err,
        );
      }
    }
    if (scratchProvisionInFlight)
      await scratchProvisionInFlight.catch(swallowAs("orchestrator: scratch provision", undefined));
    scratchProvisionInFlight = null;
    const scratchHandle = scratchBox.handle ?? scratchBox.pending;
    if (scratchHandle) {
      const cleanupStart = Date.now();
      try {
        await destroyEphemeralHandle(scratchHandle);
        scratchBox.handle = null;
        scratchBox.pending = null;
        recordScratchLifecycle("released", scratchHandle, {
          releasedAt: Date.now(),
          cleanupMs: Date.now() - cleanupStart,
        });
      } catch (error) {
        scratchCleanupError = error instanceof Error ? error : new Error(errMessage(error));
        recordScratchLifecycle("release_failed", scratchHandle, {
          releasedAt: Date.now(),
          cleanupMs: Date.now() - cleanupStart,
          error: errMessage(error),
        });
        deps.errors?.record(
          {
            category: "sandbox",
            code: "scratch_destroy_failed",
            message: errMessage(error),
            scopeLabel: scopeId,
            sessionId: session.id,
          },
          scratchCleanupError,
        );
      }
    }
    await Promise.allSettled(resourcePending.values());
    const released = new Set<string>();
    const current = box.handle ?? box.pending;
    const releases: Promise<void>[] = [];
    for (const handle of [...resourceHandles.values(), ...resourcePendingHandles.values()]) {
      const key = `${handle.backend}:${handle.id}`;
      if (released.has(key) || (current?.id === handle.id && current.backend === handle.backend)) continue;
      released.add(key);
      releases.push(
        (async () => {
          try {
            if (handle.scopeId === undefined || handle.scopeId === writableScopeId || handle.scopeId === scopeId)
              await clearTurnFiles(handle);
          } finally {
            await deps.sandbox.teardown(handle, {
              keepWarm:
                (await releaseHold(handle).catch(swallowAs("orchestrator: resource hold release", true))) ||
                (await hasLiveProcesses(handle).catch(swallowAs("orchestrator: live process check", true))),
            });
          }
        })(),
      );
    }
    const releasedResults = await Promise.allSettled(releases);
    for (const result of releasedResults)
      if (result.status === "rejected") swallow("resource sandbox release", result.reason);
    resourceHandles.clear();
    resourcePendingHandles.clear();
    if (provisionInFlight) await provisionInFlight.catch(swallowAs("orchestrator: provision", undefined));
    provisionInFlight = null;
    const handle = box.handle ?? box.pending;
    box.handle = null;
    box.pending = null;
    if (!handle) {
      if (ownerCleanupError) throw ownerCleanupError;
      if (scratchCleanupError) throw scratchCleanupError;
      return;
    }
    await clearTurnFiles(handle);
    let keepWarm = await releaseHold(handle).catch(swallowAs("orchestrator: hold release", true));
    if (!keepWarm && deps.processes && supportsProcessSessions(deps.sandbox)) {
      try {
        keepWarm = await hasLiveProcesses(handle);
      } catch (e) {
        swallow("orchestrator: live process check", e);
        keepWarm = false;
      }
    }
    await deps.sandbox.teardown(handle, {
      ...(keepWarm ? { keepWarm: true } : {}),
      ...(box.used ? {} : { homeUnchanged: true }),
    });
    if (ownerCleanupError) throw ownerCleanupError;
    if (scratchCleanupError) throw scratchCleanupError;
  };

  return {
    scratchBox,
    ownerAuthBox,
    ownerAuthCommand,
    provision,
    provisionScratch,
    accessResource,
    provisionResource,
    provisionOwnerAuth,
    useSkill,
    provisionForReach,
    reclaimBox,
    invalidateProvision: () => {
      const previous = box.handle ?? box.pending;
      if (previous)
        (box.handle ? resourceHandles : resourcePendingHandles).set(previous.resourceId ?? previous.id, previous);
      box.handle = null;
      box.pending = null;
      provisionInFlight = null;
      laidTrees.clear();
    },
  };
}