Skip to content

A desktop coding agent with free access to frontier models.

TypeScript
25.0 KB
Raw Blame
import { afterEach, describe, expect, test } from "bun:test";
import { mkdtemp, rm } from "node:fs/promises";
import { tmpdir } from "node:os";
import { join } from "node:path";

import { clearEnginePoolForConfig, setEnginePoolForConfig, type EnginePool } from "./engine-pool.js";
import { startServer } from "./server.js";
import type { ServerConfig } from "./types.js";

import { afterAll as afterAllUnlocked, beforeAll as beforeAllUnlocked } from "bun:test";
import { applyProviderLockAnswer, resetProviderLock } from "./provider-lock.js";

// These cover the app while the server leaves model providers unlocked
// (providers.locked false); provider-lock.test.ts covers the lock.
beforeAllUnlocked(() => { applyProviderLockAnswer({ providers: { locked: false } }); });
afterAllUnlocked(() => resetProviderLock());

/**
 * Regression tests for the "aborted messages / provider header-timeout storm"
 * class of bugs (support reports 2026-08-07):
 *  1. A managed engine reload must never dispose an engine with non-idle
 *     sessions — subagent children abort mid-turn ("The message was
 *     interrupted") when it does. Reloads defer and stay pending instead.
 *  2. A dispose wedged on live-session teardown must not freeze the provider
 *     sync queue: the engine answers /instance/dispose only after teardown,
 *     and an unbounded await froze lastRun at "applied" and jammed every
 *     later pass.
 */

const CLIENT_TOKEN = "owt_reload_guard_client";
const HOST_TOKEN = "owt_reload_guard_host";
const roots: string[] = [];
const stops: Array<() => void | Promise<void>> = [];
let previousRuntimeDb: string | undefined;
let previousEnvStore: string | undefined;
let previousDisposeTimeout: string | undefined;
let previousReloadRetry: string | undefined;

function isRecord(value: unknown): value is Record<string, unknown> {
  return typeof value === "object" && value !== null && !Array.isArray(value);
}

function hostHeaders() {
  return { "x-omnirush-host-token": HOST_TOKEN, "content-type": "application/json" };
}

function clientHeaders() {
  return { authorization: `Bearer ${CLIENT_TOKEN}`, "content-type": "application/json" };
}

async function readJsonObject(response: Response): Promise<Record<string, unknown>> {
  const payload: unknown = await response.json();
  if (!isRecord(payload)) throw new Error("Expected JSON object");
  return payload;
}

async function createTempRoot() {
  const root = await mkdtemp(join(tmpdir(), "omnirush-reload-guard-"));
  roots.push(root);
  previousRuntimeDb = process.env.OMNIRUSH_RUNTIME_DB;
  process.env.OMNIRUSH_RUNTIME_DB = join(root, "runtime.sqlite");
  previousEnvStore = process.env.OMNIRUSH_ENV_STORE;
  process.env.OMNIRUSH_ENV_STORE = join(root, "env.json");
  return root;
}

function serverConfig(root: string, baseUrl?: string): ServerConfig {
  return {
    host: "127.0.0.1",
    port: 0,
    configPath: join(root, "server.json"),
    token: CLIENT_TOKEN,
    hostToken: HOST_TOKEN,
    approval: { mode: "auto", timeoutMs: 1000 },
    corsOrigins: ["*"],
    workspaces: [{ id: "ws_1", name: "Workspace", path: root, preset: "starter", workspaceType: "local", baseUrl }],
    authorizedRoots: [root],
    readOnly: false,
    startedAt: Date.now(),
    tokenSource: "cli",
    hostTokenSource: "cli",
    logFormat: "pretty",
    logRequests: false,
  } satisfies ServerConfig;
}

interface FakeEngine {
  port: number;
  requests: string[];
  disposeCount: () => number;
  setBusy: (busy: boolean) => void;
  setDisposeHangs: (hangs: boolean) => void;
}

function startFakeEngine(): FakeEngine {
  const requests: string[] = [];
  let busy = false;
  let disposeHangs = false;
  const hangReleases: Array<() => void> = [];
  const engine = Bun.serve({
    port: 0,
    async fetch(request) {
      const url = new URL(request.url);
      requests.push(`${request.method} ${url.pathname}`);
      if (request.method === "GET" && url.pathname === "/session/status") {
        const statuses = busy ? { ses_live: { type: "busy" } } : {};
        return new Response(JSON.stringify(statuses), { headers: { "content-type": "application/json" } });
      }
      if (request.method === "POST" && url.pathname === "/instance/dispose") {
        if (disposeHangs) {
          // Mirrors the real engine wedged on live-session teardown: the
          // response arrives only when the test tears down.
          await new Promise<void>((resolve) => hangReleases.push(resolve));
        }
        return new Response(JSON.stringify({ ok: true }), { headers: { "content-type": "application/json" } });
      }
      if (request.method === "GET" && url.pathname === "/config") {
        return new Response(JSON.stringify({}), { headers: { "content-type": "application/json" } });
      }
      if (url.pathname.startsWith("/auth/")) {
        return new Response(JSON.stringify(true), { headers: { "content-type": "application/json" } });
      }
      return new Response(JSON.stringify({ error: "not_found" }), { status: 404, headers: { "content-type": "application/json" } });
    },
  });
  stops.push(() => {
    while (hangReleases.length) hangReleases.pop()?.();
    return engine.stop(true);
  });
  return {
    port: Number(engine.port),
    requests,
    disposeCount: () => requests.filter((entry) => entry === "POST /instance/dispose").length,
    setBusy: (value) => { busy = value; },
    setDisposeHangs: (value) => { disposeHangs = value; },
  };
}

/** A Den provider snapshot with the full config surface the engine reads. */
function stableDenProvider(): Record<string, unknown> {
  return {
    id: "lpr_steady",
    providerId: "openai-compatible",
    name: "Steady provider",
    source: "custom",
    updatedAt: "2026-08-04T10:00:00.000Z",
    providerConfig: {
      env: ["STEADY_PROVIDER_API_KEY"],
      npm: "@ai-sdk/openai-compatible",
      api: "https://models.example.test/api/v1",
      options: { baseURL: "https://models.example.test/api/v1" },
    },
    apiKey: "sk-steady-provider",
    apiKeys: null,
    models: [
      { id: "model-b", name: "Model B", config: {} },
      { id: "model-a", name: "Model A", config: {} },
    ],
  };
}

function startFakeDen(options?: { providers?: Record<string, unknown>[] }): { url: string; requests: string[] } {
  const providers = options?.providers ?? [];
  const requests: string[] = [];
  const den = Bun.serve({
    port: 0,
    fetch(request) {
      const url = new URL(request.url);
      requests.push(`${request.method} ${url.pathname}`);
      if (request.method === "GET" && url.pathname === "/v1/llm-providers") {
        return new Response(JSON.stringify({ llmProviders: providers }), { headers: { "content-type": "application/json" } });
      }
      const connect = url.pathname.match(/^\/v1\/llm-providers\/([^/]+)\/connect$/);
      if (request.method === "GET" && connect) {
        const provider = providers.find((entry) => entry.id === decodeURIComponent(connect[1] ?? ""));
        if (provider) {
          return new Response(JSON.stringify({ llmProvider: provider }), { headers: { "content-type": "application/json" } });
        }
      }
      return new Response(JSON.stringify({ error: "not_found" }), { status: 404, headers: { "content-type": "application/json" } });
    },
  });
  stops.push(() => den.stop(true));
  return { url: `http://127.0.0.1:${den.port}`, requests };
}

function guardProvider(): Record<string, unknown> {
  return {
    id: "lpr_guard",
    providerId: "anthropic",
    name: "Guard Provider",
    source: "custom",
    updatedAt: "2026-08-10T00:00:00.000Z",
    providerConfig: { env: ["GUARD_PROVIDER_API_KEY"], npm: "@ai-sdk/anthropic" },
    apiKey: "sk-guard",
    apiKeys: null,
    models: [{ id: "guard-model", name: "Guard Model", config: {} }],
  };
}

afterEach(async () => {
  while (stops.length) await stops.pop()?.();
  while (roots.length) {
    const root = roots.pop();
    if (root) await rm(root, { recursive: true, force: true });
  }
  if (previousRuntimeDb === undefined) delete process.env.OMNIRUSH_RUNTIME_DB;
  else process.env.OMNIRUSH_RUNTIME_DB = previousRuntimeDb;
  if (previousEnvStore === undefined) delete process.env.OMNIRUSH_ENV_STORE;
  else process.env.OMNIRUSH_ENV_STORE = previousEnvStore;
  if (previousDisposeTimeout === undefined) delete process.env.OMNIRUSH_ENGINE_DISPOSE_TIMEOUT_MS;
  else process.env.OMNIRUSH_ENGINE_DISPOSE_TIMEOUT_MS = previousDisposeTimeout;
  if (previousReloadRetry === undefined) delete process.env.OMNIRUSH_ENGINE_RELOAD_RETRY_MS;
  else process.env.OMNIRUSH_ENGINE_RELOAD_RETRY_MS = previousReloadRetry;
});

async function waitUntil(predicate: () => boolean, timeoutMs: number): Promise<boolean> {
  const deadline = Date.now() + timeoutMs;
  while (Date.now() < deadline) {
    if (predicate()) return true;
    await new Promise((resolve) => setTimeout(resolve, 50));
  }
  return predicate();
}

describe("engine reload guard", () => {
  test("provider sync defers pool rollover while a generation is draining", async () => {
    previousReloadRetry = process.env.OMNIRUSH_ENGINE_RELOAD_RETRY_MS;
    process.env.OMNIRUSH_ENGINE_RELOAD_RETRY_MS = "50";
    const root = await createTempRoot();
    const engine = startFakeEngine();
    const baseUrl = `http://127.0.0.1:${engine.port}`;
    const config = serverConfig(root, baseUrl);
    let draining = true;
    let rollovers = 0;
    const fakePool = {
      hasDrainingGeneration: () => draining,
      requestRollover: async () => {
        rollovers += 1;
        return { action: "rolled_over", drainingSessions: 0 };
      },
      connections: () => [],
      primaryUrl: () => baseUrl,
      routeRequest: () => null,
      reportRequestSuccess: () => undefined,
      reportRequestFailure: () => undefined,
      snapshot: () => ({ generations: [] }),
    } as unknown as EnginePool;
    setEnginePoolForConfig(config, fakePool);
    stops.push(() => clearEnginePoolForConfig(config));
    const server = await startServer(config);
    stops.push(() => server.stop());
    const base = `http://127.0.0.1:${server.port}`;
    const den = startFakeDen({ providers: [guardProvider()] });
    const put = await fetch(`${base}/den-session`, {
      method: "PUT",
      headers: hostHeaders(),
      body: JSON.stringify({ baseUrl: den.url, token: "den_token", orgId: "org_test" }),
    });
    expect(put.status).toBe(204);

    const run = await Promise.race([
      fetch(`${base}/cloud-provider-sync/run`, {
        method: "POST",
        headers: hostHeaders(),
        body: JSON.stringify({ reason: "draining" }),
      }).then(readJsonObject),
      new Promise<never>((_, reject) => setTimeout(() => reject(new Error("sync run timed out")), 5_000)),
    ]);
    expect(run.status === "applied" || run.status === "noop").toBe(true);
    expect(rollovers).toBe(0);
    const status = await readJsonObject(await fetch(`${base}/cloud-provider-sync/status`, { headers: clientHeaders() }));
    expect(status.reloadPending).toBe(true);

    draining = false;
    expect(await waitUntil(() => rollovers === 1, 3_000)).toBe(true);
  });

  test("global provider patch defers the reload while sessions are busy and applies it once idle", async () => {
    const root = await createTempRoot();
    const engine = startFakeEngine();
    const config = serverConfig(root, `http://127.0.0.1:${engine.port}`);
    const server = await startServer(config);
    stops.push(() => server.stop());
    const base = `http://127.0.0.1:${server.port}`;
    const den = startFakeDen();

    engine.setBusy(true);
    const deferred = await readJsonObject(await fetch(`${base}/runtime-config/providers`, {
      method: "PATCH",
      headers: hostHeaders(),
      body: JSON.stringify({ provider: { lpr_test: { id: "anthropic", name: "Anthropic", env: ["ANTHROPIC_API_KEY"] } } }),
    }));
    expect(deferred.ok).toBe(true);
    expect(deferred.changed).toBe(true);
    expect(deferred.reload).toBe("deferred");
    expect(engine.disposeCount()).toBe(0);

    // The parked reload applies through the provider sync once the engine idles.
    engine.setBusy(false);
    const put = await fetch(`${base}/den-session`, {
      method: "PUT",
      headers: hostHeaders(),
      body: JSON.stringify({ baseUrl: den.url, token: "den_token", orgId: "org_test" }),
    });
    expect(put.status).toBe(204);
    const run = await readJsonObject(await fetch(`${base}/cloud-provider-sync/run`, {
      method: "POST",
      headers: hostHeaders(),
      body: JSON.stringify({ reason: "test" }),
    }));
    expect(run.status === "applied" || run.status === "noop").toBe(true);
    expect(engine.disposeCount()).toBeGreaterThanOrEqual(1);
  });

  test("provider sync passes defer reloads while busy, then reload and converge to noop when idle", async () => {
    const root = await createTempRoot();
    const engine = startFakeEngine();
    const config = serverConfig(root, `http://127.0.0.1:${engine.port}`);
    const server = await startServer(config);
    stops.push(() => server.stop());
    const base = `http://127.0.0.1:${server.port}`;
    const den = startFakeDen();

    engine.setBusy(true);
    const put = await fetch(`${base}/den-session`, {
      method: "PUT",
      headers: hostHeaders(),
      body: JSON.stringify({ baseUrl: den.url, token: "den_token", orgId: "org_test" }),
    });
    expect(put.status).toBe(204);

    // The first pass (PUT /den-session enqueues one automatically) writes the
    // engine-visible runtime file -> a reload becomes pending, but the busy
    // engine must not be disposed.
    const busyRun = await readJsonObject(await fetch(`${base}/cloud-provider-sync/run`, {
      method: "POST",
      headers: hostHeaders(),
      body: JSON.stringify({ reason: "busy_pass" }),
    }));
    expect(busyRun.status === "applied" || busyRun.status === "noop").toBe(true);
    expect(engine.disposeCount()).toBe(0);
    const busyStatus = await readJsonObject(await fetch(`${base}/cloud-provider-sync/status`, { headers: clientHeaders() }));
    const busyLastRun = isRecord(busyStatus.lastRun) ? busyStatus.lastRun : {};
    const busyDetail = isRecord(busyLastRun.detail) ? busyLastRun.detail : {};
    expect(busyDetail.reloadDeferred).toBe(true);

    // Idle: the pending reload fires exactly once, and the sync settles.
    engine.setBusy(false);
    await readJsonObject(await fetch(`${base}/cloud-provider-sync/run`, {
      method: "POST",
      headers: hostHeaders(),
      body: JSON.stringify({ reason: "idle_pass" }),
    }));
    expect(engine.disposeCount()).toBe(1);
    const settled = await readJsonObject(await fetch(`${base}/cloud-provider-sync/run`, {
      method: "POST",
      headers: hostHeaders(),
      body: JSON.stringify({ reason: "converged_pass" }),
    }));
    expect(settled.status).toBe("noop");
    expect(engine.disposeCount()).toBe(1);
  });

  test("repeated passes over an unchanged Den snapshot never dispose the engine again", async () => {
    const root = await createTempRoot();
    const engine = startFakeEngine();
    const config = serverConfig(root, `http://127.0.0.1:${engine.port}`);
    const server = await startServer(config);
    stops.push(() => server.stop());
    const base = `http://127.0.0.1:${server.port}`;
    const provider = stableDenProvider();
    const den = startFakeDen({ providers: [provider] });

    const put = await fetch(`${base}/den-session`, {
      method: "PUT",
      headers: hostHeaders(),
      body: JSON.stringify({ baseUrl: den.url, token: "den_token", orgId: "org_test" }),
    });
    expect(put.status).toBe(204);

    // Materialize the provider. The engine cannot apply provider config on a
    // live instance (PUT /auth carries credentials only), so this legitimately
    // disposes -- but it must settle, not repeat.
    await readJsonObject(await fetch(`${base}/cloud-provider-sync/run`, {
      method: "POST",
      headers: hostHeaders(),
      body: JSON.stringify({ reason: "materialize" }),
    }));
    const settledDisposes = engine.disposeCount();
    expect(settledDisposes).toBeGreaterThan(0);
    expect(engine.requests.indexOf("PUT /auth/lpr_steady")).toBeLessThan(
      engine.requests.indexOf("POST /instance/dispose"),
    );

    // Every later pass sees byte-identical Den state. A pass that still reports
    // "applied" here is the runaway-dispose bug: on the 5-minute interval it
    // would tear down the engine forever, aborting whatever is running.
    for (let pass = 0; pass < 5; pass += 1) {
      const result = await readJsonObject(await fetch(`${base}/cloud-provider-sync/run`, {
        method: "POST",
        headers: hostHeaders(),
        body: JSON.stringify({ reason: `steady_${pass}` }),
      }));
      expect(result.status).toBe("noop");
      expect(engine.disposeCount()).toBe(settledDisposes);
    }

    // The credential push is fingerprint-guarded, so it must not re-deliver
    // on every pass either.
    expect(engine.requests.filter((entry) => entry === "PUT /auth/lpr_steady")).toHaveLength(1);

    // A credential rotation leaves the provider config byte-identical, but
    // the cached SDK client must still be replaced after the new auth lands.
    provider.apiKey = "sk-steady-provider-rotated";
    const beforeRotationDisposes = engine.disposeCount();
    const rotated = await readJsonObject(await fetch(`${base}/cloud-provider-sync/run`, {
      method: "POST",
      headers: hostHeaders(),
      body: JSON.stringify({ reason: "credential_rotated" }),
    }));
    expect(rotated.status).toBe("applied");
    expect(engine.requests.filter((entry) => entry === "PUT /auth/lpr_steady")).toHaveLength(2);
    expect(engine.disposeCount()).toBe(beforeRotationDisposes + 1);
    expect(engine.requests.lastIndexOf("PUT /auth/lpr_steady")).toBeLessThan(
      engine.requests.lastIndexOf("POST /instance/dispose"),
    );
  });

  test("a deferred reload lands by itself once the engine idles, even with no Den session", async () => {
    previousReloadRetry = process.env.OMNIRUSH_ENGINE_RELOAD_RETRY_MS;
    process.env.OMNIRUSH_ENGINE_RELOAD_RETRY_MS = "200";
    const root = await createTempRoot();
    const engine = startFakeEngine();
    const config = serverConfig(root, `http://127.0.0.1:${engine.port}`);
    const server = await startServer(config);
    stops.push(() => server.stop());
    const base = `http://127.0.0.1:${server.port}`;

    // No PUT /den-session on purpose: a signed-out desktop that patches
    // global providers must still see them reach the engine eventually.
    engine.setBusy(true);
    const deferred = await readJsonObject(await fetch(`${base}/runtime-config/providers`, {
      method: "PATCH",
      headers: hostHeaders(),
      body: JSON.stringify({ provider: { lpr_retry: { id: "anthropic", name: "Anthropic", env: ["ANTHROPIC_API_KEY"] } } }),
    }));
    expect(deferred.reload).toBe("deferred");
    expect(engine.disposeCount()).toBe(0);

    // Still busy: the retry poll must keep waiting, not dispose.
    await new Promise((resolve) => setTimeout(resolve, 500));
    expect(engine.disposeCount()).toBe(0);

    // Idle: the parked reload lands from the retry poll alone.
    engine.setBusy(false);
    const landed = await waitUntil(() => engine.disposeCount() >= 1, 3_000);
    expect(landed).toBe(true);
  });

  test("a reload failure mid-pass keeps the materialized providers visible and the failure loud", async () => {
    previousDisposeTimeout = process.env.OMNIRUSH_ENGINE_DISPOSE_TIMEOUT_MS;
    process.env.OMNIRUSH_ENGINE_DISPOSE_TIMEOUT_MS = "500";
    const root = await createTempRoot();
    const engine = startFakeEngine();
    const config = serverConfig(root, `http://127.0.0.1:${engine.port}`);
    const server = await startServer(config);
    stops.push(() => server.stop());
    const base = `http://127.0.0.1:${server.port}`;
    const den = startFakeDen({ providers: [guardProvider()] });

    engine.setDisposeHangs(true);
    const put = await fetch(`${base}/den-session`, {
      method: "PUT",
      headers: hostHeaders(),
      body: JSON.stringify({ baseUrl: den.url, token: "den_token", orgId: "org_test" }),
    });
    expect(put.status).toBe(204);

    const failed = await readJsonObject(await fetch(`${base}/cloud-provider-sync/run`, {
      method: "POST",
      headers: hostHeaders(),
      body: JSON.stringify({ reason: "reload_fails" }),
    }));
    expect(failed.status).toBe("failed");
    expect(String(failed.message ?? "")).toContain("did not complete in time");

    // The materialization itself landed (config + env writes), so the status
    // must list the provider instead of pretending nothing synced, and the
    // still-owed reload must be visible.
    const status = await readJsonObject(await fetch(`${base}/cloud-provider-sync/status`, { headers: clientHeaders() }));
    const providers = Array.isArray(status.providers) ? status.providers.filter(isRecord) : [];
    expect(providers.map((entry) => entry.cloudProviderId)).toContain("lpr_guard");
    expect(status.reloadPending).toBe(true);
    const lastRun = isRecord(status.lastRun) ? status.lastRun : {};
    expect(lastRun.status).toBe("failed");
    expect(String(lastRun.message ?? "").length).toBeGreaterThan(0);
  });

  test("a failed reload self-heals through the retry poll and settles the status", async () => {
    previousDisposeTimeout = process.env.OMNIRUSH_ENGINE_DISPOSE_TIMEOUT_MS;
    process.env.OMNIRUSH_ENGINE_DISPOSE_TIMEOUT_MS = "500";
    previousReloadRetry = process.env.OMNIRUSH_ENGINE_RELOAD_RETRY_MS;
    process.env.OMNIRUSH_ENGINE_RELOAD_RETRY_MS = "200";
    const root = await createTempRoot();
    const engine = startFakeEngine();
    const config = serverConfig(root, `http://127.0.0.1:${engine.port}`);
    const server = await startServer(config);
    stops.push(() => server.stop());
    const base = `http://127.0.0.1:${server.port}`;
    const den = startFakeDen({ providers: [guardProvider()] });

    engine.setDisposeHangs(true);
    const put = await fetch(`${base}/den-session`, {
      method: "PUT",
      headers: hostHeaders(),
      body: JSON.stringify({ baseUrl: den.url, token: "den_token", orgId: "org_test" }),
    });
    expect(put.status).toBe(204);
    const failed = await readJsonObject(await fetch(`${base}/cloud-provider-sync/run`, {
      method: "POST",
      headers: hostHeaders(),
      body: JSON.stringify({ reason: "reload_fails" }),
    }));
    expect(failed.status).toBe("failed");
    const disposesAfterFailure = engine.disposeCount();

    // No further /run calls: the parked reload must land from the retry poll
    // alone once the engine answers disposes again, and the status must settle
    // to a truthful applied instead of staying failed forever.
    engine.setDisposeHangs(false);
    const settled = await (async () => {
      const deadline = Date.now() + 5_000;
      while (Date.now() < deadline) {
        const status = await readJsonObject(await fetch(`${base}/cloud-provider-sync/status`, { headers: clientHeaders() }));
        const lastRun = isRecord(status.lastRun) ? status.lastRun : {};
        if (lastRun.status === "applied" && status.reloadPending === false) return status;
        await new Promise((resolve) => setTimeout(resolve, 100));
      }
      return null;
    })();
    expect(settled).not.toBeNull();
    expect(engine.disposeCount()).toBeGreaterThan(disposesAfterFailure);
  });

  test("a wedged dispose times out instead of freezing the sync queue", async () => {
    previousDisposeTimeout = process.env.OMNIRUSH_ENGINE_DISPOSE_TIMEOUT_MS;
    process.env.OMNIRUSH_ENGINE_DISPOSE_TIMEOUT_MS = "500";
    const root = await createTempRoot();
    const engine = startFakeEngine();
    const config = serverConfig(root, `http://127.0.0.1:${engine.port}`);
    const server = await startServer(config);
    stops.push(() => server.stop());
    const base = `http://127.0.0.1:${server.port}`;
    const den = startFakeDen();

    engine.setDisposeHangs(true);
    const put = await fetch(`${base}/den-session`, {
      method: "PUT",
      headers: hostHeaders(),
      body: JSON.stringify({ baseUrl: den.url, token: "den_token", orgId: "org_test" }),
    });
    expect(put.status).toBe(204);

    // The pass carries a pending reload (first runtime-file write); the hung
    // dispose must fail the pass in bounded time instead of hanging it.
    const first = await readJsonObject(await fetch(`${base}/cloud-provider-sync/run`, {
      method: "POST",
      headers: hostHeaders(),
      body: JSON.stringify({ reason: "wedged_pass" }),
    }));
    expect(first.status).toBe("failed");
    expect(String(first.message ?? "")).toContain("did not complete in time");

    // The queue is not jammed: the next pass runs (and retries the reload).
    const disposesBefore = engine.disposeCount();
    const second = await readJsonObject(await fetch(`${base}/cloud-provider-sync/run`, {
      method: "POST",
      headers: hostHeaders(),
      body: JSON.stringify({ reason: "post_wedge_pass" }),
    }));
    expect(second.status).toBe("failed");
    expect(engine.disposeCount()).toBe(disposesBefore + 1);

    // Recovery: once the engine answers disposes again, the reload lands.
    engine.setDisposeHangs(false);
    const recovered = await readJsonObject(await fetch(`${base}/cloud-provider-sync/run`, {
      method: "POST",
      headers: hostHeaders(),
      body: JSON.stringify({ reason: "recovered_pass" }),
    }));
    expect(recovered.status === "applied" || recovered.status === "noop").toBe(true);
  });
});