A desktop coding agent with free access to frontier models.
TypeScript
import { browserScript } from "@omnirush/testkit";
import { expect, onTestFinished } from "vitest";
import { observeText, textProgressFailures, control, denFetch, evalIn, go, readAvailableModels, selectModel, sendComposerMessage, waitFor } from "@omnirush/behaviors";
import type { DenSession, ModelFacts } from "@omnirush/behaviors";
import { screenshot, validate } from "@omnirush/test-evidence";
import { app, eventually, mcpMock, needs, server, sleep, test, unmetNeeds } from "@omnirush/testkit";
import type { TestNeeds } from "@omnirush/testkit";
/**
* ACCEPTANCE TEST for #3671: org-published LLM providers never finish syncing
* to member desktops — the settings row sits on "Syncing" forever (suspect:
* commit 316d364d0's engine-reload guard deferring reloads indefinitely).
*
* EXPECTED RED on dev as of 2026-08-11, at claim 1 or claim 2. Every failure
* message embeds the per-provider /cloud-provider-sync/status payload so a red
* run discriminates the three failure modes on its own:
* - stuck-syncing -> claim 1 times out; the payload shows which
* provider never reached a terminal outcome.
* - applied-but-engine-empty -> claim 1 green, claim 2 red; status lists the
* provider (fingerprint recorded as done) but
* the picker never listed its model — the
* settings row derives "Syncing" from exactly
* this gap (entitled && !available).
* - silent-skip -> a published provider is absent from status
* with no failure message anywhere (claims 1/3).
*
* Terminal per-provider outcome, derived from the real schema
* (apps/server/src/cloud-provider-sync.ts:33-69): CloudProviderSyncStatus has
* hasSession, lastRun { at, status: "applied"|"noop"|"failed", message?,
* detail? } and providers[] (CloudProviderSyncStatusProvider: cloudProviderId,
* providerId, sourceProviderId, name, source, updatedAt, modelIds, importedAt).
* A provider entry only materializes after a successful pass
* (updateProviderStatus, cloud-provider-sync.ts:781-801), so "terminal" here
* means: the provider's cloudProviderId is listed AND lastRun settled as
* applied|noop. Repeated "failed" (or absence) is permanently non-terminal.
*/
const ORGANIZATION_NAME = "Cloud Provider Sync Contract";
const CUSTOM_PROVIDER_NAME = "Sync Contract Custom Models";
const CUSTOM_PROVIDER_KEY = "sync-contract-custom-models";
const CUSTOM_MODEL_ID = "sync-contract-custom-model";
const CATALOG_PROVIDER_NAME = "Sync Contract Catalog Models";
const CATALOG_MODEL_ID = "gpt-5.4";
const THIRD_MODEL_ID = "sync-contract-third-model";
const REQUEST_TIMEOUT_MS = 10_000;
const TERMINAL_BUDGET_MS = 120_000;
const MODEL_BUDGET_MS = 30_000;
const V2_START_BUDGET_MS = 180_000;
const V2_MIRROR_BUDGET_MS = 60_000;
const THIRD_PROVIDER_BUDGET_MS = 360_000;
const QUIET_DELAY_MS = 30_000;
const OBSERVATION_WINDOW_MS = 60_000;
// Intended poll cadence: one sync pass per 5 minutes (defaultIntervalMs =
// 5 * 60 * 1_000, apps/server/src/cloud-provider-sync.ts:132; the testkit
// desktop sets no OMNIRUSH_CLOUD_PROVIDER_SYNC_INTERVAL_MS override, so the
// env branch at cloud-provider-sync.ts:481-484 stays on the default). Each
// pass issues exactly ONE GET /v1/llm-providers/:llmProviderId/connect per
// provider (fetchProviders, cloud-provider-sync.ts:294-306), so a 60s window
// holds at most one intended pass -> <=1 connect per provider. 3x headroom:
const CONNECT_BOUND_PER_PROVIDER = 3;
// den-api's access log normalizes the path to the parameterized route
// (ee/apps/den-api/src/observability/hono.ts:66-72), so connect calls cannot
// be split per provider id; every pass touches every provider exactly once,
// which makes per-provider rate = total / provider count.
const CONNECT_ROUTE = "/v1/llm-providers/:llmProviderId/connect";
const requirements: TestNeeds = { optIn: ["OMNIRUSH_EVAL_E2E_TESTS"] };
const missingRequirements = unmetNeeds(requirements, process.env);
const title = missingRequirements.length > 0
? `cloud provider sync contract skipped — needs: ${missingRequirements.join(", ")}`
: "an org-published provider's desktop sync reaches a terminal, truthful status";
interface PublishedProvider {
cloudId: string;
name: string;
modelId: string;
}
interface ProviderStatusEntry {
cloudProviderId: string;
providerId: string;
sourceProviderId: string;
name: string;
source: string;
modelIds: string[];
}
interface SyncStatusFacts {
hasSession: boolean;
lastRunStatus: "applied" | "noop" | "failed" | "none";
lastRunAt: string;
lastRunMessage: string;
reloadDeferred: boolean;
providers: ProviderStatusEntry[];
raw: Record<string, unknown>;
}
interface ConnectLogEntry {
timestamp: string;
}
function isRecord(value: unknown): value is Record<string, unknown> {
return typeof value === "object" && value !== null && !Array.isArray(value);
}
function auth(session: DenSession): Record<string, string> {
return { authorization: `Bearer ${session.token}` };
}
async function organizationId(session: DenSession): Promise<string> {
const result = await denFetch(session, "/v1/me/orgs", {
headers: auth(session),
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
});
const organizations = isRecord(result.body) && Array.isArray(result.body.orgs)
? result.body.orgs.filter(isRecord)
: [];
const organization = organizations.find((entry) => entry.name === ORGANIZATION_NAME);
const id = organization && typeof organization.id === "string" ? organization.id : "";
if (!result.response.ok || !id) {
throw new Error(`Finding the test organization failed: HTTP ${result.response.status} ${result.text.slice(0, 500)}`);
}
return id;
}
async function createProvider(admin: DenSession, orgId: string, body: Record<string, unknown>): Promise<string> {
const result = await denFetch(admin, "/v1/llm-providers", {
method: "POST",
headers: { ...auth(admin), "x-omnirush-org-id": orgId },
body: JSON.stringify(body),
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
});
const provider = isRecord(result.body) && isRecord(result.body.llmProvider) ? result.body.llmProvider : null;
const id = provider && typeof provider.id === "string" ? provider.id : "";
if (result.response.status !== 201 || !id) {
throw new Error(`Creating provider ${JSON.stringify(body.name)} failed: HTTP ${result.response.status} ${result.text.slice(0, 500)}`);
}
return id;
}
async function deleteProvider(admin: DenSession, orgId: string, providerId: string): Promise<void> {
const result = await denFetch(admin, `/v1/llm-providers/${encodeURIComponent(providerId)}`, {
method: "DELETE",
headers: { ...auth(admin), "x-omnirush-org-id": orgId },
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
});
if (!result.response.ok) throw new Error(`Deleting fixture provider failed: HTTP ${result.response.status}`);
}
async function memberVisibleProviderIds(member: DenSession, orgId: string): Promise<string[]> {
const result = await denFetch(member, "/v1/llm-providers", {
headers: { ...auth(member), "x-omnirush-org-id": orgId },
signal: AbortSignal.timeout(REQUEST_TIMEOUT_MS),
});
if (!result.response.ok) {
throw new Error(`Listing member-visible providers failed: HTTP ${result.response.status} ${result.text.slice(0, 500)}`);
}
const providers = isRecord(result.body) && Array.isArray(result.body.llmProviders)
? result.body.llmProviders.filter(isRecord)
: [];
return providers.flatMap((entry) => (typeof entry.id === "string" ? [entry.id] : []));
}
function parseSyncStatus(payload: Record<string, unknown>): SyncStatusFacts {
const lastRun = isRecord(payload.lastRun) ? payload.lastRun : {};
const detail = isRecord(lastRun.detail) ? lastRun.detail : {};
const lastRunStatus = lastRun.status === "applied" || lastRun.status === "noop" || lastRun.status === "failed"
? lastRun.status
: "none";
const providers = Array.isArray(payload.providers) ? payload.providers.filter(isRecord) : [];
return {
hasSession: payload.hasSession === true,
lastRunStatus,
lastRunAt: typeof lastRun.at === "string" ? lastRun.at : "",
lastRunMessage: typeof lastRun.message === "string" ? lastRun.message : "",
reloadDeferred: detail.reloadDeferred === true,
providers: providers.map((entry) => ({
cloudProviderId: typeof entry.cloudProviderId === "string" ? entry.cloudProviderId : "",
providerId: typeof entry.providerId === "string" ? entry.providerId : "",
sourceProviderId: typeof entry.sourceProviderId === "string" ? entry.sourceProviderId : "",
name: typeof entry.name === "string" ? entry.name : "",
source: typeof entry.source === "string" ? entry.source : "",
modelIds: Array.isArray(entry.modelIds)
? entry.modelIds.filter((id): id is string => typeof id === "string")
: [],
})),
raw: payload,
};
}
// The desktop local server's GET /cloud-provider-sync/status is registered
// with "client" auth (apps/server/src/server.ts:2108), so the renderer's own
// persisted credentials (localStorage omnirush.server.port/omnirush.server.token)
// reach it with a plain Bearer fetch to 127.0.0.1.
async function readSyncStatusPayload(surface: Parameters<typeof evalIn>[0]): Promise<Record<string, unknown>> {
const value = await evalIn(surface, async () => {
const port = localStorage.getItem("omnirush.server.port");
const token = localStorage.getItem("omnirush.server.token");
if (!port || !token) return { specProbeError: "missing local server credentials" };
const response = await fetch("http://127.0.0.1:" + port + "/cloud-provider-sync/status", {
headers: { Authorization: "Bearer " + token },
});
if (!response.ok) return { specProbeError: "HTTP " + response.status + " " + (await response.text()).slice(0, 200) };
return await response.json();
}, { awaitPromise: true, timeoutMs: 30_000 });
if (!isRecord(value)) {
throw new Error(`GET /cloud-provider-sync/status returned a non-object: ${JSON.stringify(value)}`);
}
if (typeof value.specProbeError === "string") {
throw new Error(`GET /cloud-provider-sync/status unreachable: ${value.specProbeError}`);
}
return value;
}
async function engineV2Status(surface: Parameters<typeof evalIn>[0], enabled?: boolean) {
const value = await evalIn(surface, browserScript(async (inputEnabled) => {
const port = localStorage.getItem("omnirush.server.port");
const token = localStorage.getItem("omnirush.server.token");
if (!port || !token) return { specProbeError: "missing local server credentials" };
const enabled = inputEnabled;
const response = await fetch("http://127.0.0.1:" + port + "/experimental/engine-v2-preview" + (enabled === undefined ? "/status" : ""), {
...(enabled === undefined ? {} : { method: "PUT", body: JSON.stringify({ enabled, chatRouting: enabled }) }),
headers: { Authorization: "Bearer " + token, "Content-Type": "application/json" },
});
if (!response.ok) return { specProbeError: "HTTP " + response.status + " " + (await response.text()).slice(0, 200) };
return await response.json();
}, [enabled]), { awaitPromise: true, timeoutMs: 30_000 });
if (!isRecord(value) || typeof value.specProbeError === "string"
|| typeof value.enabled !== "boolean" || typeof value.running !== "boolean") {
throw new Error(`Engine v2 preview status was invalid: ${JSON.stringify(value)}`);
}
const ids = (field: string): string[] => {
const entries = value[field];
if (!Array.isArray(entries) || !entries.every((entry) => typeof entry === "string")) {
throw new Error(`Engine v2 preview ${field} was invalid: ${JSON.stringify(entries)}`);
}
return entries;
};
return {
enabled: value.enabled,
running: value.running,
...(typeof value.pid === "number" ? { pid: value.pid } : {}),
mirroredProviderIds: ids("mirroredProviderIds"),
skippedProviderIds: ids("skippedProviderIds"),
catalogModelIds: ids("catalogModelIds"),
};
}
type EngineV2Status = Awaited<ReturnType<typeof engineV2Status>>;
function waitForEngineV2(
surface: Parameters<typeof evalIn>[0],
until: (status: EngineV2Status) => boolean, within: number, label: string,
): Promise<EngineV2Status> {
return eventually(() => engineV2Status(surface), { within, intervalMs: 1_000, label, until });
}
function connectLogEntries(logText: string): ConnectLogEntry[] {
const entries: ConnectLogEntry[] = [];
for (const line of logText.split(/\r?\n/)) {
// Local-lane api.log interleaves pnpm/tsx chatter with the JSON lines, so
// parse from the first brace and skip anything that is not a JSON object.
const start = line.indexOf("{");
if (start === -1) continue;
let parsed: unknown;
try {
parsed = JSON.parse(line.slice(start));
} catch {
continue;
}
if (!isRecord(parsed) || parsed.http_route !== CONNECT_ROUTE) continue;
entries.push({ timestamp: typeof parsed.timestamp === "string" ? parsed.timestamp : "" });
}
return entries;
}
test.skipIf(missingRequirements.length > 0)(title, { timeout: 30 * 60_000 }, async ({ evidence, place }) => {
needs(requirements);
const promptMarker = `NATIVE-SYNC-${Date.now()}`;
const finalReply = `NATIVE-REPLY-${Date.now()}`;
await using den = await server({
place,
mocks: { agent: mcpMock({
agentWorkloads: [{ promptMarker, finalReply, steps: [] }],
agentRequiredHeader: { name: "x-private-model-setting", value: "sk-omnirush-sync-contract-eval-only" },
}) },
org: {
name: ORGANIZATION_NAME,
admin: { name: "Sync Admin" },
members: { member: { name: "Sync Member" } },
},
});
const member = den.members.member;
if (!member) throw new Error("The testkit did not provision the organization member.");
const orgId = await organizationId(den.admin);
// Both org-wide providers are published BEFORE the desktop boots, so the
// very first sign-in sync owes the member both of them.
const customProviderId = await createProvider(den.admin, orgId, {
name: CUSTOM_PROVIDER_NAME,
source: "custom",
customConfig: {
id: CUSTOM_PROVIDER_KEY,
name: CUSTOM_PROVIDER_NAME,
npm: "@ai-sdk/openai",
options: { baseURL: `${den.mocks.agent.url}/v1` },
env: ["SYNC_CONTRACT_PROVIDER_API_KEY"],
models: [{ id: CUSTOM_MODEL_ID, name: "Sync Contract Custom Model", tool_call: true,
headers: { "x-private-model-setting": "sk-omnirush-sync-contract-eval-only" },
limit: { context: 1050000, output: 128000 }, modalities: { input: ["text", "image", "pdf"], output: ["text"] } }],
},
apiKey: "sk-omnirush-sync-contract-eval-only",
allMembers: true,
memberIds: [],
teamIds: [],
});
onTestFinished(async () => {
await deleteProvider(den.admin, orgId, customProviderId).catch(() => undefined);
});
const catalogProviderId = await createProvider(den.admin, orgId, {
name: CATALOG_PROVIDER_NAME,
source: "models_dev",
providerId: "openai",
modelIds: [CATALOG_MODEL_ID],
apiKey: "sk-omnirush-sync-contract-eval-only",
allMembers: true,
memberIds: [],
teamIds: [],
});
onTestFinished(async () => {
await deleteProvider(den.admin, orgId, catalogProviderId).catch(() => undefined);
});
const published: PublishedProvider[] = [
{ cloudId: customProviderId, name: CUSTOM_PROVIDER_NAME, modelId: CUSTOM_MODEL_ID },
{ cloudId: catalogProviderId, name: CATALOG_PROVIDER_NAME, modelId: CATALOG_MODEL_ID },
];
// Access-wiring gate, so a red claim 1 can only mean sync: Den itself must
// already list both providers for this member.
const grantedIds = await memberVisibleProviderIds(member, orgId);
for (const provider of published) {
if (!grantedIds.includes(provider.cloudId)) {
throw new Error(
`Setup failed before any sync claim: Den does not list ${provider.name} (${provider.cloudId}) for the member. `
+ `Member-visible provider ids: ${JSON.stringify(grantedIds)}`,
);
}
}
await using desktopApp = await app({ den, as: "member", place });
// ── Claim 1: terminal status ────────────────────────────────────────────
const isTerminal = (status: SyncStatusFacts): boolean =>
published.every((provider) => status.providers.some((entry) => entry.cloudProviderId === provider.cloudId))
&& (status.lastRunStatus === "applied" || status.lastRunStatus === "noop");
const terminalDeadline = Date.now() + TERMINAL_BUDGET_MS;
let latestStatus: SyncStatusFacts | null = null;
let terminalReached = false;
let lastStatusReadError = "";
while (Date.now() < terminalDeadline) {
try {
latestStatus = parseSyncStatus(await readSyncStatusPayload(desktopApp));
if (isTerminal(latestStatus)) {
terminalReached = true;
break;
}
} catch (error) {
lastStatusReadError = error instanceof Error ? error.message : String(error);
}
await sleep(3_000);
}
const terminalSettledAtMs = Date.now();
const settledStatus = latestStatus;
if (!settledStatus) {
throw new Error(
`GET /cloud-provider-sync/status never returned a readable payload within ${TERMINAL_BUDGET_MS / 1000}s of desktop sign-in`
+ `${lastStatusReadError ? ` (last error: ${lastStatusReadError})` : ""}; no sync claim was reached.`,
);
}
const absentFromStatus = published
.filter((provider) => !settledStatus.providers.some((entry) => entry.cloudProviderId === provider.cloudId))
.map((provider) => `${provider.name} (${provider.cloudId})`);
evidence.recordAssertionEvidence(
"Both org-published providers reach a terminal per-provider sync outcome within 120s",
`Published ${JSON.stringify(published.map((provider) => provider.cloudId))}; latest /cloud-provider-sync/status: ${JSON.stringify(settledStatus.raw)}`,
terminalReached,
);
expect(terminalReached, [
"Claim 1 (terminal status): every Den-granted provider must appear in /cloud-provider-sync/status providers[] with a settled run (lastRun applied|noop) within 120s; a provider may not stay absent or permanently non-terminal.",
`Still absent from status: ${JSON.stringify(absentFromStatus)}; lastRun status: ${settledStatus.lastRunStatus}${settledStatus.lastRunMessage ? ` (${settledStatus.lastRunMessage})` : ""}.`,
`Latest status payload: ${JSON.stringify(settledStatus.raw)}`,
].join("\n")).toBe(true);
// Claim 2 checks model-catalog visibility, not successful inference.
const syncedProviders = published.filter((provider) =>
settledStatus.providers.some((entry) => entry.cloudProviderId === provider.cloudId));
const expectedModelIds = syncedProviders.map((provider) => provider.modelId);
let lastModels: ModelFacts[] = [];
const modelsArrived = await eventually(async () => {
lastModels = await readAvailableModels(desktopApp);
return lastModels;
}, {
within: MODEL_BUDGET_MS,
intervalMs: 3_000,
label: "engine models for status-synced providers",
until: (models) => expectedModelIds.every((id) => models.some((model) => model.id === id)),
}).then(() => true, () => false);
const missingModelIds = expectedModelIds.filter((id) => !lastModels.some((model) => model.id === id));
evidence.recordAssertionEvidence(
"Every provider the sync status lists contributes its model to the picker within 30s",
`Status-synced providers claim ${JSON.stringify(expectedModelIds)}; engine picker ids: ${JSON.stringify(lastModels.map((model) => model.id))}`,
modelsArrived,
);
expect(modelsArrived, [
"Claim 2 (catalog availability): the sync status lists these providers, but the picker never listed their models within 30s.",
`Missing model ids: ${JSON.stringify(missingModelIds)}.`,
`Status entries for the published providers: ${JSON.stringify(settledStatus.providers.filter((entry) => published.some((provider) => provider.cloudId === entry.cloudProviderId)))}.`,
`Latest status payload: ${JSON.stringify(settledStatus.raw)}`,
`Engine picker models: ${JSON.stringify(lastModels)}`,
].join("\n")).toBe(true);
// readAvailableModels leaves the picker dialog open; close it through its
// own close control so later claims observe a calm, unobstructed app.
await evalIn(desktopApp, () => {
const close = document.querySelector<HTMLElement>('[data-slot="dialog-content"] [data-slot="dialog-close"]');
if (close instanceof HTMLElement) close.click();
return true;
});
await waitFor(desktopApp, () => (!document.querySelector<HTMLElement>('[data-slot="dialog-content"]')), {
timeoutMs: 15_000,
label: "model picker dialog closed",
});
await go(desktopApp, `/workspace/${desktopApp.workspaceId}/session`);
await selectModel(desktopApp, CATALOG_MODEL_ID, { provider: CATALOG_PROVIDER_NAME });
const chosenDefault = await evalIn(desktopApp, () => (localStorage.getItem("omnirush.defaultModel")));
expect(typeof chosenDefault).toBe("string");
expect(String(chosenDefault)).toContain(CATALOG_MODEL_ID);
await evalIn(desktopApp, () => {
window.__modelSelectionProof = { changes: 0, loops: 0, unavailable: document.body.innerText.includes("Model no longer available") };
window.addEventListener("omnirush.defaultModelChanged", () => window.__modelSelectionProof.changes++);
const original = console.error;
console.error = (...args) => {
if (args.some((arg) => String(arg).includes("Maximum update depth"))) window.__modelSelectionProof.loops++;
original.apply(console, args);
};
new MutationObserver(() => {
if (document.body.innerText.includes("Model no longer available")) window.__modelSelectionProof.unavailable = true;
}).observe(document.body, { childList: true, subtree: true });
});
expect(await evalIn(desktopApp, () => (window.__modelSelectionProof.unavailable))).toBe(false);
// ── Claim 3: loud skips — a dropped provider must name itself ───────────
// The schema (apps/server/src/cloud-provider-sync.ts:38-69) carries no
// per-provider skipped/errored state: its only error carrier is
// lastRun.status="failed" plus lastRun.message, and a skipped provider
// manifests as absence from providers[] (prepareMaterialization silently
// drops entries without env matches, cloud-provider-sync.ts:414-426). The
// loud-skip contract at this schema is therefore: a failed run must carry a
// non-empty message, and a published provider absent from status is only
// acceptable when such an explaining failure exists.
const currentStatus = parseSyncStatus(await readSyncStatusPayload(desktopApp));
const droppedNow = published
.filter((provider) => !currentStatus.providers.some((entry) => entry.cloudProviderId === provider.cloudId))
.map((provider) => `${provider.name} (${provider.cloudId})`);
const failureExplained = currentStatus.lastRunStatus !== "failed" || currentStatus.lastRunMessage.trim().length > 0;
const dropsExplained = droppedNow.length === 0
|| (currentStatus.lastRunStatus === "failed" && currentStatus.lastRunMessage.trim().length > 0);
const loudSkips = failureExplained && dropsExplained;
evidence.recordAssertionEvidence(
"No provider is skipped or errored silently: failures carry a reason and drops are explained",
`Dropped from status: ${JSON.stringify(droppedNow)}; lastRun: ${currentStatus.lastRunStatus}${currentStatus.lastRunMessage ? ` (${currentStatus.lastRunMessage})` : " (no message)"}; payload: ${JSON.stringify(currentStatus.raw)}`,
loudSkips,
);
expect(loudSkips, [
"Claim 3 (loud skips): a failed sync run must carry a non-empty message, and a published provider missing from status must be accompanied by such an explained failure — never silently dropped.",
`Dropped without explanation: ${JSON.stringify(droppedNow)}; lastRun status: ${currentStatus.lastRunStatus}; message: ${JSON.stringify(currentStatus.lastRunMessage)}.`,
`Latest status payload: ${JSON.stringify(currentStatus.raw)}`,
].join("\n")).toBe(true);
// ── Claim 4: calm loop — the settled sync must not hammer den-api ───────
// Observation window: >=30s after claim 1 settled, then 60s wide. The
// window is bounded by two den.apiLog() snapshots (count diffing), so the
// sandbox/driver clock skew cannot bend it; the JSON lines' own timestamps
// travel into the failure message to pin any ~500ms hammering bursts.
const sinceSettleMs = Date.now() - terminalSettledAtMs;
if (sinceSettleMs < QUIET_DELAY_MS) await sleep(QUIET_DELAY_MS - sinceSettleMs);
const connectsBefore = connectLogEntries(await den.apiLog());
await sleep(OBSERVATION_WINDOW_MS);
const connectsAfter = connectLogEntries(await den.apiLog());
const windowConnectCount = connectsAfter.length - connectsBefore.length;
const windowTimestamps = connectsAfter.slice(connectsBefore.length).map((entry) => entry.timestamp);
const connectBound = CONNECT_BOUND_PER_PROVIDER * published.length;
const calmLoop = windowConnectCount >= 0 && windowConnectCount <= connectBound;
const statusAfterWindow = parseSyncStatus(await readSyncStatusPayload(desktopApp));
evidence.recordAssertionEvidence(
"The settled sync loop polls at its intended cadence instead of hammering /connect",
`${windowConnectCount} ${CONNECT_ROUTE} calls in a ${OBSERVATION_WINDOW_MS / 1000}s window across ${published.length} providers `
+ `(bound ${connectBound} = ${CONNECT_BOUND_PER_PROVIDER}/provider from the 5-minute cadence at apps/server/src/cloud-provider-sync.ts:132); `
+ `window call timestamps: ${JSON.stringify(windowTimestamps)}`,
calmLoop,
);
expect(calmLoop, [
`Claim 4 (calm loop): a settled sync intends <=1 pass (one /connect per provider) per 60s window at the 5-minute cadence (apps/server/src/cloud-provider-sync.ts:132); observed ${windowConnectCount} connect calls against a 3x-headroom bound of ${connectBound} (log counts ${connectsBefore.length} -> ${connectsAfter.length}).`,
`Window call timestamps (pins list+connect rounds landing within ~500ms): ${JSON.stringify(windowTimestamps)}.`,
`Latest status payload: ${JSON.stringify(statusAfterWindow.raw)}`,
].join("\n")).toBe(true);
// ── Claim 5: the settings surface stands, bug present or fixed ──────────
const settingsRoute = `/workspace/${desktopApp.workspaceId}/settings/cloud-providers`;
await go(desktopApp, settingsRoute);
await waitFor(desktopApp, browserScript((value) => {
if (!window.location.hash.includes('/settings/cloud-providers')) {
window.location.hash = value;
return false;
}
return /provider/i.test(document.body.innerText);
}, [`#${settingsRoute}`]), { timeoutMs: 60_000, label: "cloud providers settings surface" });
const shot = await screenshot(desktopApp);
const seen = await validate(shot, [
"The workspace settings cloud providers surface is visible",
"No crash text such as 'Something went wrong' is visible",
]);
evidence.recordAssertionEvidence(
"The cloud providers settings surface renders without crashing",
`Validated at ${settingsRoute}: ${seen.ok ? "ok" : seen.why}`,
seen.ok,
);
expect(seen.ok, `${seen.why}\nLatest status payload: ${JSON.stringify(statusAfterWindow.raw)}`).toBe(true);
// ── Claim 6: Den provider writes hot-mirror into the live v2 sidecar ─────
const customRuntime = settledStatus.providers.find((entry) => entry.cloudProviderId === customProviderId);
const catalogRuntime = settledStatus.providers.find((entry) => entry.cloudProviderId === catalogProviderId);
if (!customRuntime || !catalogRuntime) throw new Error("Settled sync status lost an initially published provider.");
onTestFinished(async () => {
await engineV2Status(desktopApp, false).catch(() => undefined);
});
await engineV2Status(desktopApp, true);
const startedV2 = await waitForEngineV2(
desktopApp, (status) => status.enabled && status.running && typeof status.pid === "number",
V2_START_BUDGET_MS, "OpenCode v2 sidecar to install and start",
);
const pid0 = startedV2.pid;
if (pid0 === undefined) throw new Error("Running OpenCode v2 status omitted its pid.");
const initialV2 = await waitForEngineV2(
desktopApp,
(status) => status.mirroredProviderIds.includes(customRuntime.providerId)
&& status.catalogModelIds.includes(CUSTOM_MODEL_ID)
&& status.mirroredProviderIds.includes(catalogRuntime.providerId)
&& !status.skippedProviderIds.includes(catalogRuntime.providerId)
&& status.catalogModelIds.includes(CATALOG_MODEL_ID),
V2_MIRROR_BUDGET_MS, "custom and native Den providers to appear in the v2 catalog",
);
const catalogPrivacy = await evalIn(desktopApp, browserScript(async (value, providerId) => {
const base = "http://127.0.0.1:" + localStorage.getItem("omnirush.server.port");
const paths = value;
const results = [];
let model;
for (const path of paths) {
const response = await fetch(base + path, {
headers: { Authorization: "Bearer " + localStorage.getItem("omnirush.server.token") }
});
const text = await response.text();
if (path === paths[0] && response.ok) model = JSON.parse(text).data.find((entry: { providerID?: string }) => entry.providerID === providerId);
results.push({ status: response.status, leaksKey: text.includes("sk-omnirush-sync-contract-eval-only"), ...(response.ok ? {} : { error: text.replaceAll("sk-omnirush-sync-contract-eval-only", "[redacted]") }) });
}
return { results, model };
}, [[`/workspace/${desktopApp.workspaceId}/opencode2/api/model`, `/workspace/${desktopApp.workspaceId}/opencode2/api/model/default`, `/workspace/${desktopApp.workspaceId}/opencode2/api/provider`, `/workspace/${desktopApp.workspaceId}/opencode2/api/provider/${customRuntime.providerId}`], customRuntime.providerId]), { awaitPromise: true });
expect(catalogPrivacy).toMatchObject({
results: Array.from({ length: 4 }, () => ({ status: 200, leaksKey: false })),
model: { limit: { context: 1050000, output: 128000 }, capabilities: { tools: true, input: ["text", "image", "pdf"], output: ["text"] } },
});
evidence.recordAssertionEvidence("custom native OpenAI model limits, tools and modalities survive mirroring", "The fixture uses the native @ai-sdk/openai adapter; its live v2 catalog entry retained 1,050,000 context and 128,000 output limits, tool support, text/image/PDF inputs and text output.", true);
await go(desktopApp, `/workspace/${desktopApp.workspaceId}/session`);
await sleep(16_000); // includes the renderer's engine-routing refresh interval
expect(await evalIn(desktopApp, () => (localStorage.getItem("omnirush.defaultModel")))).toBe(chosenDefault);
expect(await evalIn(desktopApp, () => (window.__modelSelectionProof))).toEqual({ changes: 0, loops: 0, unavailable: false });
evidence.recordAssertionEvidence(
"native organization model selection survives v1 to v2 without false unavailability or a React loop",
"The selected catalog provider/model stayed unchanged through Settings, cloud refresh, native v2 mirroring and routing refresh; zero default writes, unavailable warnings or maximum-depth errors were observed.", true,
);
await selectModel(desktopApp, CUSTOM_MODEL_ID, { provider: CUSTOM_PROVIDER_NAME });
await using responseObservation = await observeText(desktopApp, '[data-message-role="assistant"] .prose');
await sendComposerMessage(desktopApp, `Reply to the request marked ${promptMarker}.`);
await waitFor(desktopApp, browserScript((finalReply) => (Array.from(document.querySelectorAll<HTMLElement>('[data-message-role="assistant"]')).some((row) => row.textContent.includes(finalReply))), [finalReply]), { timeoutMs: 120_000, label: "native provider reply in the v2 transcript" });
await sleep(2_000); // includes the streamed-to-persisted message reconciliation
const frames = await responseObservation.finish();
expect(textProgressFailures(frames, finalReply), JSON.stringify(frames)).toEqual([]);
evidence.recordAssertionEvidence("streamed assistant text does not flash a different answer before the saved response", "Observed frames included empty states and never shrank after text appeared; the independent expected reply completed without replacement, including two seconds after completion.", true);
const nativeRequests = await den.mocks.agent.agentRequests({ promptMarker, atLeast: 1, timeoutMs: 10_000 });
expect(nativeRequests.some((request) => request.model === CUSTOM_MODEL_ID && request.kind === "final")).toBe(true);
expect(nativeRequests.some((request) => request.model !== CUSTOM_MODEL_ID || request.kind === "error")).toBe(false);
expect((await den.mocks.agent.requests()).some((request) => request.path.endsWith("/responses"))).toBe(true);
evidence.recordAssertionEvidence(
"native OpenAI inference completes through the selected v2 model",
"The signed-in desktop rendered the independent reply nonce; the witness required the private model header and saw the selected model on the native Responses endpoint, with no wrong-model or authentication-error requests.", true,
);
const nativeSessionRoute = await evalIn(desktopApp, () => (location.hash.slice(1)));
if (typeof nativeSessionRoute !== "string") throw new Error("Native session route was unavailable");
await go(desktopApp, `/workspace/${desktopApp.workspaceId}/session`);
await selectModel(desktopApp, CATALOG_MODEL_ID, { provider: CATALOG_PROVIDER_NAME });
const changesBeforeSync = await evalIn(desktopApp, () => (window.__modelSelectionProof.changes));
const thirdPublishedAt = Date.now();
const thirdProviderId = await createProvider(den.admin, orgId, {
name: "Sync Contract Third Models",
source: "custom",
customConfig: {
id: "sync-contract-third-models",
name: "Sync Contract Third Models",
npm: "@ai-sdk/openai-compatible",
options: { baseURL: "https://third-gateway.example.test/v1" },
env: ["SYNC_CONTRACT_THIRD_PROVIDER_API_KEY"],
models: [{ id: THIRD_MODEL_ID, name: "Sync Contract Third Model" }],
},
apiKey: "sk-omnirush-sync-contract-third-eval-only",
allMembers: true,
memberIds: [],
teamIds: [],
});
onTestFinished(async () => {
await deleteProvider(den.admin, orgId, thirdProviderId).catch(() => undefined);
});
const thirdSync = await eventually(async () => parseSyncStatus(await readSyncStatusPayload(desktopApp)), {
within: THIRD_PROVIDER_BUDGET_MS,
intervalMs: 3_000,
label: "third Den provider to reach desktop runtime config",
until: (status) => status.providers.some((entry) => entry.cloudProviderId === thirdProviderId),
});
const thirdRuntime = thirdSync.providers.find((entry) => entry.cloudProviderId === thirdProviderId);
if (!thirdRuntime) throw new Error("Third provider disappeared from its settled sync status.");
const thirdV2 = await waitForEngineV2(
desktopApp,
(status) => status.mirroredProviderIds.includes(thirdRuntime.providerId)
&& status.catalogModelIds.includes(THIRD_MODEL_ID),
V2_MIRROR_BUDGET_MS, "third Den provider to hot-mirror into v2",
);
const mirrorLatencyMs = Date.now() - thirdPublishedAt;
const catalogOutcome = "mirrored with its declared model and native default endpoint";
const hotMirrored = initialV2.running && initialV2.pid === pid0
&& thirdV2.running && thirdV2.pid === pid0;
evidence.recordAssertionEvidence(
"Den-published providers hot-mirror into the OpenCode v2 sidecar without restarting it",
`baseURL providers ${customRuntime.providerId} and ${thirdRuntime.providerId} mirrored with models ${CUSTOM_MODEL_ID} and ${THIRD_MODEL_ID}; catalog provider ${catalogRuntime.providerId} was ${catalogOutcome}; third-provider latency=${mirrorLatencyMs}ms; sidecar stayed at pid ${pid0}.`,
hotMirrored,
);
expect(await evalIn(desktopApp, () => (localStorage.getItem("omnirush.defaultModel")))).toBe(chosenDefault);
expect(await evalIn(desktopApp, () => (window.__modelSelectionProof.changes))).toBe(changesBeforeSync);
// Remove the selected assignment in the fixture organization, then use the
// real server sync boundary. A missing model must settle into recovery.
await deleteProvider(den.admin, orgId, catalogProviderId);
const syncResult = await evalIn(desktopApp, async () => {
const info = await window.__OMNIRUSH_ELECTRON__.invokeDesktop("omnirushServerInfo");
if (!info.hostToken) throw new Error("Missing local server host token");
const port = localStorage.getItem("omnirush.server.port");
const response = await fetch("http://127.0.0.1:" + port + "/cloud-provider-sync/run", {
method: "POST", headers: { "x-omnirush-host-token": info.hostToken, "Content-Type": "application/json" }, body: "{}"
});
return response.status;
}, { awaitPromise: true, timeoutMs: 90_000 });
expect(syncResult).toBe(200);
await go(desktopApp, `/workspace/${desktopApp.workspaceId}/session`);
// Opening Models is the user-facing refresh action; server reconciliation
// alone does not update the renderer's completed catalog snapshot.
await control(desktopApp, "session.model_picker.open");
await waitFor(desktopApp, () => (document.body.innerText.includes("Model no longer available")), { timeoutMs: 90_000, label: "revoked model recovery" });
const changesAfterRevocation = await evalIn(desktopApp, () => (window.__modelSelectionProof.changes));
await sleep(5_000);
expect(await evalIn(desktopApp, () => (localStorage.getItem("omnirush.defaultModel")))).toBe(chosenDefault);
expect(await evalIn(desktopApp, () => (window.__modelSelectionProof.changes))).toBe(changesAfterRevocation);
expect(await evalIn(desktopApp, () => (window.__modelSelectionProof.loops))).toBe(0);
await selectModel(desktopApp, CUSTOM_MODEL_ID, { provider: CUSTOM_PROVIDER_NAME });
await waitFor(desktopApp, () => (!document.body.innerText.includes("Model no longer available")), { timeoutMs: 30_000, label: "explicit model recovery" });
const recoveredDefault = await evalIn(desktopApp, () => (localStorage.getItem("omnirush.defaultModel")));
evidence.recordAssertionEvidence("a genuinely revoked model settles and recovers only after an explicit selection", "Revoking the selected assignment surfaced recovery while preserving its identity for five seconds with no repeated default writes or React loop; selecting an available model cleared the warning.", true);
// Continue the existing conversation: creating a new task intentionally
// reconciles the organization credential, which would replace this isolated
// env-store rotation with the fixture organization's unchanged secret.
await go(desktopApp, nativeSessionRoute);
await waitFor(desktopApp, browserScript((finalReply) => (Array.from(document.querySelectorAll<HTMLElement>('[data-message-role="assistant"] .prose')).some((row) => row.textContent.trim() === finalReply)), [finalReply]), { timeoutMs: 30_000, label: "original reply restored before continuing the conversation" });
await using rotationObservation = await observeText(desktopApp, '[data-message-role="assistant"] .prose', { ignoreExistingMessages: true });
const rotatedMarker = `${promptMarker}-rotated`;
const rotatedReply = `ROTATED-REPLY-${Date.now()}`;
const rotatedKey = "sk-omnirush-sync-contract-rotated-eval-only";
const mockUpdate = await fetch(`${den.mocks.agent.url}/admin/agent-workloads`, {
method: "POST", headers: { "Content-Type": "application/json" },
body: JSON.stringify({ workloads: [{ promptMarker: rotatedMarker, finalReply: rotatedReply, steps: [] }], requiredHeader: { name: "authorization", value: `Bearer ${rotatedKey}` } }),
signal: AbortSignal.timeout(10_000),
});
expect(mockUpdate.ok).toBe(true);
const rotatedAt = new Date().toISOString();
const rotationStatus = await evalIn(desktopApp, browserScript(async (providerId, rotatedKey) => {
const info = await window.__OMNIRUSH_ELECTRON__.invokeDesktop("omnirushServerInfo");
if (!info.hostToken) throw new Error("Missing local server host token");
const base = "http://127.0.0.1:" + localStorage.getItem("omnirush.server.port");
const headers = { "x-omnirush-host-token": info.hostToken, "Content-Type": "application/json" };
const config = await (await fetch(base + "/runtime-config/providers", { headers })).json();
const names = config.provider[providerId]?.env;
if (!Array.isArray(names) || names.length !== 1) throw new Error("fixture provider must declare one scoped credential");
const response = await fetch(base + "/env", { method: "PUT", headers, body: JSON.stringify({ entries: [{ key: names[0], value: rotatedKey }] }) });
return response.status;
}, [customRuntime.providerId, rotatedKey]), { awaitPromise: true, timeoutMs: 30_000 });
expect(rotationStatus).toBe(200);
await sleep(3_000);
await sendComposerMessage(desktopApp, `Reply to the request marked ${rotatedMarker}.`);
await waitFor(desktopApp, browserScript((rotatedReply) => (Array.from(document.querySelectorAll<HTMLElement>('[data-message-role="assistant"]')).some((row) => row.textContent.includes(rotatedReply))), [rotatedReply]), { timeoutMs: 120_000, label: "reply after credential rotation" });
await sleep(2_000);
const rotationFrames = await rotationObservation.finish();
expect(textProgressFailures(rotationFrames, rotatedReply), JSON.stringify(rotationFrames)).toEqual([]);
expect(await evalIn(desktopApp, browserScript((finalReply) => (document.querySelectorAll<HTMLElement>('[data-message-role="assistant"]').length >= 2 && Array.from(document.querySelectorAll<HTMLElement>('[data-message-role="assistant"] .prose')).some((row) => row.textContent.trim() === finalReply)), [finalReply]))).toBe(true);
evidence.recordAssertionEvidence("a second assistant response never substitutes the previous answer or overwrites it", "Every new response frame matched only the independent second reply; after completion the first answer remained intact in a separate assistant message.", true);
const rotatedRequests = await den.mocks.agent.agentRequests({ promptMarker: rotatedMarker, sinceIso: rotatedAt, atLeast: 1 });
expect(rotatedRequests.some((request) => request.kind === "final" && request.model === CUSTOM_MODEL_ID)).toBe(true);
expect(rotatedRequests.some((request) => request.kind === "error" || request.model !== CUSTOM_MODEL_ID)).toBe(false);
expect((await engineV2Status(desktopApp)).pid).toBe(pid0);
evidence.recordAssertionEvidence("stored credential rotation reaches v2 without an engine restart", "Changing only the provider's scoped env-store credential produced an authenticated native reply with the replacement key and same model and sidecar pid; no wrong-key errors or wrong-model requests occurred.", true);
await engineV2Status(desktopApp, false);
await go(desktopApp, `/workspace/${desktopApp.workspaceId}/session`);
await sleep(16_000);
expect(await evalIn(desktopApp, () => (localStorage.getItem("omnirush.defaultModel")))).toBe(recoveredDefault);
expect(await evalIn(desktopApp, () => (window.__modelSelectionProof.loops))).toBe(0);
evidence.recordAssertionEvidence("selected model survives cloud sync and return to v1", "The exact selected provider/model remained after another Den provider was published, the normal cloud sync completed, and routing returned to v1; no React update loop occurred.", true);
expect(hotMirrored, `Initial v2 status: ${JSON.stringify(initialV2)}; third v2 status: ${JSON.stringify(thirdV2)}`).toBe(true);
});