Make your computer a home for your AI partner. Avibe is the local-first Agent OS: Vibey, its built-in AI partner, works out of the box, and Claude Code, Codex & OpenCode run unchanged beside it — reach them from your browser, phone, or any chat app.
Python agentagent-osaiai-agentsai-partnerchatopsclaudeclaude-codecodexdevtoolsdiscord-botlark-botllmlocal-firstlong-term-memoryopencodeslack-botvibe-codingvibeywechat
// Suite B — the Add API key dialog, end to end against a controllable upstream.
//
// Scenario IDs are from docs/plans/model-hub-e2e-test-plan.md §3.
//
// Every spec here needs an upstream whose answers it can choose, so every spec
// here skips without one. The mock is reached ONLY over HTTP at
// `VIBE_E2E_MOCK_UPSTREAM_URL`, and its URL enters the product the same way a
// user's would: typed into the dialog's Base URL field.
import type { HubApi } from './support/api';
import { hub as copy } from './support/copy';
import { E2E_SOURCE_PREFIX, mockBaseUrl } from './support/env';
import {
requireMockUpstream,
requireModelHub,
requireRuntimeRunning,
} from './support/fixtures';
// `gateway` rather than `fixtures` for the auto `hubSurface` fixture it carries:
// the Add API key button lives on the gateway surface, and an instance with no
// sources shows the direct home instead. See support/gateway.ts.
import { expect, test } from './support/gateway';
import { fillApiKeyForm } from './support/hub';
import { anthropicInventory } from './support/mock';
import { captureAgentChain, restoreAgentChain } from './support/restore';
// The shipped vendor catalog, read from the same tracked file the dialog and the
// server read. A spec that named an id and its pinned interface by hand would
// keep passing after the catalog moved underneath it.
import { CATALOG_VENDORS } from './support/vendors';
/**
* Runs the routed dry run until the source has settled on a verdict about its
* KEY, and returns that verdict's detail key.
*
* Both halves are load-bearing, and neither works alone:
*
* - drive with the probe, because a dry run is the only thing that reaches the
* upstream with a real request; a refetch reads the model list, which an
* upstream that rejects completions still serves.
* - read the SOURCE, because the first probe that fails also makes the chain
* unrunnable — every probe after it is refused `409 probe_no_candidate` and
* carries no verdict at all. A poll on the probe's own answer goes null
* forever, one round after the state it was waiting for arrived.
*
* A healthy instance answers on the first dry run. The loop exists for the one
* state that is not an answer: a cooldown, which is what an unreachable upstream
* settles as, and which the state itself dates in `retry_at`. Sleeping to that
* instant is the difference between one more attempt and a busy wait; the budget
* is what stops it from waiting forever on an upstream that is genuinely gone.
*/
const settleKeyVerdict = async (
api: HubApi,
sourceId: string,
route: { backend: string; model: string },
budgetMs = 90_000,
): Promise<string | null> => {
const deadline = Date.now() + budgetMs;
let last: string | null = null;
while (Date.now() < deadline) {
await api.probeAgent(route.backend, route.model);
const state = (await api.sources()).find((source) => source.id === sourceId)?.state;
last = (state?.detail_key as string | null) ?? null;
// `needs_action` is terminal by construction: nothing but the user acts on
// it. A cooldown is the opposite — it expires, and then the next dry run is
// the one that learns what is actually wrong.
if (state?.status === 'needs_action' || state?.status === 'error') return last;
const retryAt = typeof state?.retry_at === 'string' ? Date.parse(state.retry_at) : Number.NaN;
const resumeAt = Number.isNaN(retryAt) ? Date.now() + 1_000 : retryAt + 500;
const pause = Math.min(Math.max(resumeAt - Date.now(), 500), deadline - Date.now());
if (pause <= 0) break;
await new Promise((resolve) => setTimeout(resolve, pause));
}
return last;
};
// Custom providers persist the protocol the user selected, without observation.
const PROTOCOLS = [
{ id: 'anthropic', label: copy('addKey.protocol.anthropicMessages') },
{ id: 'openai_responses', label: copy('addKey.protocol.openaiResponses') },
{ id: 'openai_chat', label: copy('addKey.protocol.openaiChatCompletions') },
] as const;
test.describe('B · add an API-key source', () => {
test.beforeEach(async ({ api, mock }) => {
await requireModelHub(api);
await requireRuntimeRunning(api);
await requireMockUpstream(mock);
});
// Every spec adds sources; none of them may outlive the spec that made them.
// Teardown runs on failure too, which is the case that matters: a red test
// that also poisons the next five is two problems reported as six.
test.afterEach(async ({ api }) => {
await api.removeSuiteSources();
});
for (const protocol of PROTOCOLS) {
test('B1 · saves the selected ' + protocol.id + ' protocol without model invocation', async ({ hub, mock, api }) => {
await mock.configure({ auth: 'ok', protocol: protocol.id, models_endpoint: 'ok' });
const name = E2E_SOURCE_PREFIX + 'selected-' + protocol.id;
await hub.goto();
await hub.addApiKeyButton.click();
await fillApiKeyForm(hub.addKeyDialog, {
name, baseUrl: mockBaseUrl(), apiKey: 'e2e-add', protocol: protocol.label,
});
await mock.resetRequests();
await hub.addKeyDialog.getByRole('button', { name: copy('addKey.confirm'), exact: true }).click();
await expect(hub.addKeyDialog).toHaveCount(0, { timeout: 30_000 });
await expect(hub.sourceDetailDialog).toContainText(name);
const created = (await api.sources()).find((source) => source.display_name === name);
expect(created).toMatchObject({ vendor: 'custom', protocol: protocol.id, verification_pending: expect.any(String) });
expect((await mock.requests()).filter((request) => request.method === 'POST' && request.path.startsWith('/v1/'))).toEqual([]);
});
}
test('B1 · a catalog vendor saves under its own id and pinned interface', async ({ hub, mock, api }) => {
const preset = CATALOG_VENDORS[0];
await mock.configure({ auth: 'ok', protocol: preset.protocol, models_endpoint: 'ok' });
const name = E2E_SOURCE_PREFIX + 'catalog-' + preset.id;
await hub.goto();
await hub.addApiKeyButton.click();
await fillApiKeyForm(hub.addKeyDialog, { vendor: preset.label, name, baseUrl: mockBaseUrl(), apiKey: 'e2e-add' });
await expect(hub.addKeyDialog.getByRole('group', { name: copy('addKey.field.protocol') })).toHaveCount(0);
await expect(hub.addKeyDialog).toContainText(PROTOCOLS.find((item) => item.id === preset.protocol)!.label);
await hub.addKeyDialog.getByRole('button', { name: copy('addKey.confirm'), exact: true }).click();
await expect(hub.addKeyDialog).toHaveCount(0, { timeout: 30_000 });
expect((await api.sources()).find((source) => source.display_name === name)).toMatchObject({
vendor: preset.id, protocol: preset.protocol, verification_pending: expect.any(String),
});
});
test('B1 · upstream authentication failure cannot veto saving configuration', async ({ hub, mock, api }) => {
await mock.configure({ auth: '401', protocol: 'anthropic', models_endpoint: 'ok' });
const name = E2E_SOURCE_PREFIX + 'auth-unverified';
await hub.goto();
await hub.addApiKeyButton.click();
await fillApiKeyForm(hub.addKeyDialog, {
name, baseUrl: mockBaseUrl(), apiKey: 'e2e-bad-key', protocol: PROTOCOLS[0].label,
});
await hub.addKeyDialog.getByRole('button', { name: copy('addKey.confirm'), exact: true }).click();
await expect(hub.addKeyDialog).toHaveCount(0, { timeout: 30_000 });
expect((await api.sources()).find((source) => source.display_name === name)).toMatchObject({
protocol: 'anthropic', verification_pending: expect.any(String),
});
});
test('B2 · a declared protocol persists without claiming response proof', async ({ hub, mock, api }) => {
await mock.configure({ auth: 'ok', protocol: 'openai_chat', models_endpoint: 'ok' });
const name = E2E_SOURCE_PREFIX + 'mismatch';
await hub.goto();
await hub.addApiKeyButton.click();
await fillApiKeyForm(hub.addKeyDialog, {
name, baseUrl: mockBaseUrl(), apiKey: 'e2e-mismatch', protocol: PROTOCOLS[0].label,
});
await hub.addKeyDialog.getByRole('button', { name: copy('addKey.confirm'), exact: true }).click();
await expect(hub.addKeyDialog).toHaveCount(0, { timeout: 30_000 });
expect((await api.sources()).find((source) => source.display_name === name)).toMatchObject({
vendor: 'custom', protocol: 'anthropic', verification_pending: expect.any(String),
});
});
test('B3 · automatically fetches ordered inventory, then tests only the chosen model', async ({ hub, mock, api, page }) => {
const inventory = anthropicInventory(['e2e-alpha', 'e2e-beta', 'e2e-gamma']);
await mock.configure({ auth: 'ok', protocol: 'anthropic', models_endpoint: 'ok', models: inventory });
const name = E2E_SOURCE_PREFIX + 'inventory';
await hub.goto();
await hub.addApiKeyButton.click();
await fillApiKeyForm(hub.addKeyDialog, { name, baseUrl: mockBaseUrl(), apiKey: 'e2e-pull', protocol: PROTOCOLS[0].label });
await hub.addKeyDialog.getByRole('button', { name: copy('addKey.confirm'), exact: true }).click();
await expect(hub.sourceDetailDialog).toBeVisible({ timeout: 30_000 });
const created = (await api.sources()).find((source) => source.display_name === name);
expect(created?.models).toEqual(inventory.map((model) => ({
id: model.id, origin: 'discovered', display_name: null, reasoning_efforts: [],
reasoning_efforts_source: null, discovered_at: expect.any(String), retired: false,
})));
expect(created?.verification_pending).toEqual(expect.any(String));
await hub.sourceDetailDialog.getByRole('button', { name: copy('sourceTest.open'), exact: true }).click();
const dialog = page.getByRole('dialog', { name: copy('sourceTest.title', { source: name }), exact: true });
await expect(dialog.getByRole('combobox')).toContainText('e2e-alpha');
await dialog.getByRole('combobox').click();
await page.getByRole('option', { name: 'e2e-beta', exact: true }).click();
await mock.resetRequests();
await dialog.getByRole('button', { name: copy('sourceTest.run'), exact: true }).click();
await expect(dialog.getByRole('status')).toContainText('e2e-beta', { timeout: 75_000 });
await expect.poll(async () => (await api.sources()).find((source) => source.id === created?.id)?.verification_pending).toBeFalsy();
const requests = (await mock.requests()).filter((request) => request.method === 'POST' && request.path === '/v1/messages');
expect(requests).toHaveLength(1);
expect(requests[0].body).toMatchObject({ model: 'e2e-beta' });
});
test('B4 · unavailable inventory still saves an unverified provider', async ({ hub, mock, api }) => {
await mock.configure({ auth: 'ok', protocol: 'anthropic', models_endpoint: 'http_500' });
const name = E2E_SOURCE_PREFIX + 'no-inventory';
await hub.goto();
await hub.addApiKeyButton.click();
await fillApiKeyForm(hub.addKeyDialog, { name, baseUrl: mockBaseUrl(), apiKey: 'e2e-empty', protocol: PROTOCOLS[0].label });
await hub.addKeyDialog.getByRole('button', { name: copy('addKey.confirm'), exact: true }).click();
await expect(hub.sourceDetailDialog).toBeVisible({ timeout: 30_000 });
expect((await api.sources()).find((source) => source.display_name === name)).toMatchObject({
protocol: 'anthropic', models: [], verification_pending: expect.any(String),
});
});
test('B6 · replacing the key reuses the same dialog, then closes on a toast', async ({ hub, mock, api, gateway }) => {
// The only spec in the suite that may have to sit out a 30-second cooldown
// before its precondition even exists. See `settleKeyVerdict`.
test.setTimeout(300_000);
// A source offers to have its key replaced only once the key has actually
// been rejected: `repairAction` reads `needs_action.credential_revoked`, and
// that button is the dialog's ONLY entry point in replace mode. Nothing
// about a healthy source opens it, so the rejection has to be arranged —
// and arranged the way a real one happens, since a refetch will not do it.
// The upstream still serves its model list to an unauthorised caller; only a
// request that runs down the chain learns the key is dead.
let source = gateway.sources[0];
const supplied = source.models[0]?.id;
const arrange = async (): Promise<void> => {
expect(
supplied,
`The precondition source ${source.display_name} came back with no models, so no route can reach it.`,
).toBeDefined();
const arranged = await api.putAgentChain(gateway.backend, gateway.model, [
{ source_id: source.id, model_id: supplied! },
]);
expect(arranged, 'The instance refused the arranged route, so the key cannot be rejected.').toBe(true);
};
// The operator's chain only. The retry below deletes and recreates the
// precondition source, so a baseline carrying its hop would name a source id
// that no longer exists — and that refusal is not partial.
const original = await captureAgentChain(api, gateway);
try {
// The restoration boundary is entered BEFORE the arranged PUT (inside
// `arrange` below), not after it returns: a PUT whose response is lost
// or times out rejects that await with the chain already replaced
// server-side, and a finally outside it would leave the user's route
// swapped for the arrangement — then the fixture's source sweep empties
// it.
await arrange();
// `cooldown.server_error` is the one answer that is not an answer: the
// gateway engine fails the dry run itself, without the upstream ever
// receiving it, and every retry re-arms a fresh thirty-second cooldown on
// top of the last — roughly one run in three (#1818). Recreating the source
// sometimes clears it; waiting, restarting the engine, and cycling the whole
// gateway do not. So a sticky cooldown is retried by rebuilding the
// arrangement, bounded to three attempts.
//
// A defect that survives all three attempts is #1818's exact signature
// (5xx + the upstream received nothing + the cooldown re-arming), and the
// run then retires ITSELF via `test.fixme` naming #1818, so an
// intermittent defect never burns the suite red. Any OTHER verdict still
// fails hard below; a wrong classification is what the scenario exists to
// catch.
//
// `test.fixme`, not `test.fail`. `test.fail` sets the expected status of
// the whole TEST — its finally, the suite's afterEach, and the gateway
// fixture's teardown included — so a restoration that then failed would
// satisfy "expected to fail" and the run would report green over a
// displaced route chain. Measured, not assumed: a mid-body `test.fail`
// with a throwing finally reports PASSED, while `test.fixme` in the same
// shape reports the teardown failure. Fixme also states the outcome
// honestly — one skipped spec naming the issue, rather than a green tick
// over a scenario that was never reached.
// The #1818 signature is TWO facts together, and both are checked before
// the marker fires: the source in `cooldown.server_error`, AND the mock
// having received no DRY RUN while it got there. A cooldown with the dry
// run on the log is a different defect — the upstream answered 401 and
// something classified it badly — and it must fail hard, not ride #1818's
// marker to green. Two discrimination rules keep that line honest:
//
// - Only a POST down a protocol path counts (`/v1/messages`,
// `/v1/responses`, `/v1/chat/completions`): the 401 the marker
// exists to route travels only on the dry run, while a `GET
// /v1/models` is model-list traffic an engine reads at startup — and
// #1818's engine restarts mid-settle, so startup GETs land on a
// post-reset log without any completion ever being attempted.
// - The marker reads the FINAL attempt's log, not an OR across
// attempts: an earlier attempt's dial that still ended in a sticky
// cooldown says nothing about the attempt that produced the verdict
// being marked, and OR-ing them let #1818 ride in green under an
// earlier attempt's history.
const upstreamSawDryRun = async (): Promise<boolean> => {
const requests = await mock.requests();
return requests.some(
(request) => request.method === 'POST'
&& /^\/v1\/(messages|responses|chat\/completions)$/.test(request.path.split('?')[0]),
);
};
await mock.resetRequests();
await mock.configure({ auth: '401' });
let verdict = await settleKeyVerdict(api, source.id, gateway);
// Per-attempt, not cumulative: the marker at the end must be able to say
// "the attempt that produced THIS verdict never reached the upstream."
let upstreamReceived = await upstreamSawDryRun();
for (let attempt = 0; attempt < 3 && verdict === 'models.source.cooldown.server_error'; attempt += 1) {
// The recreation has to happen against a HEALTHY upstream — the create
// probes the source before committing it, and the mock is still set to
// 401 for the settle. The rejection is re-armed after the new source
// exists, by the same dry run as before.
await mock.configure({ auth: 'ok' });
await api.deleteSource(source.id);
const rebuilt = await api.createApiKeySource(source.display_name, mockBaseUrl());
if (!rebuilt) {
// The instance that cannot rebuild this arrangement is a product
// failure wearing the cooldown's clothes — named, not skipped, and not
// #1818 either.
throw new Error('The instance refused to recreate the precondition source.');
}
expect(
rebuilt.models.some((model) => model.id === supplied),
'The recreated source lost the model the route depends on.',
).toBe(true);
source = rebuilt;
await arrange();
await mock.configure({ auth: '401' });
// One cooldown window (plus slack) is enough for a retry attempt: the
// defect this loop exists for re-arms the cooldown on EVERY probe, so a
// second window after the first re-armed would only burn another 30 s on
// the way to the same verdict. The FIRST settle keeps the full budget,
// where a genuine one-off transient still gets its chance to clear.
await mock.resetRequests();
verdict = await settleKeyVerdict(api, source.id, gateway, 35_000);
upstreamReceived = await upstreamSawDryRun();
}
test.fixme(
verdict === 'models.source.cooldown.server_error' && !upstreamReceived,
'Engine 5xx on the routed dry run (#1818): the upstream never received the request and the '
+ 'cooldown re-armed on every retry, so the replace-key flow was never reachable this run.',
);
expect(verdict).toBe('models.source.needs_action.credential_revoked');
// The replacement has to be a key that works, or the dialog would be
// reporting the same failure over again under a different name.
await mock.configure({ auth: 'ok' });
await hub.goto();
await hub.openSource(source.id);
await expect(hub.sourceDetailDialog).toBeVisible();
// What a stopped row owes the user is one tap to the fix, not an error
// string. The attribute is the product's own statement of where that tap
// goes, and the label is what the user reads on it.
const repair = hub.sourceDetailDialog.locator('[data-repair-destination="replace_key_dialog"]');
await expect(repair).toHaveText(copy('repair.replaceKey'));
await repair.click();
const dialog = hub.addKeyDialog;
await expect(dialog).toBeVisible();
await expect(dialog).toContainText(copy('repair.replaceTitle', { name: source.display_name }));
await dialog.getByLabel(copy('repair.replaceLabel'), { exact: true }).fill('e2e-second-key');
await dialog.getByRole('button', { name: copy('repair.replaceSubmit'), exact: true }).click();
// The upstream still lists the same models, so the new key costs the route
// nothing: a plain repair, which closes the dialog and says so in a toast.
await expect(hub.toasts.filter({ hasText: copy('repair.repaired') })).toHaveCount(1, { timeout: 30_000 });
await expect(dialog).toHaveCount(0);
// And the row it was raised from is no longer stopped. A dialog that
// reports a repair over a source still marked blocked has reported a
// repair that did not happen.
await expect
.poll(
async () => (await api.sources()).find((s) => s.id === source.id)?.state.detail_key,
{ timeout: 15_000 },
)
.toBeNull();
} finally {
// Whatever happened above, the instance's chain for this model goes back
// to what it was — the arrangement was the scenario's, not the user's.
await restoreAgentChain(api, gateway, original);
}
});
});