Skip to content

A desktop coding agent with free access to frontier models.

TypeScript

Register the connector fixture model and wait for submission readiness

Slendywy948e298dbc0a
diff --git a/evals/specs/connector-tool-call-branding.e2e.test.ts b/evals/specs/connector-tool-call-branding.e2e.test.ts
index 4dd10995..8cbffc49 100644
--- a/evals/specs/connector-tool-call-branding.e2e.test.ts
+++ b/evals/specs/connector-tool-call-branding.e2e.test.ts
@@ -6,8 +6,15 @@ const test = spec.world(connectorBranding, { timeout: 420_000 });
test("connector-backed tool calls show first-class branding and human-readable labels", async ({ world, user, probe, step }) => {
const sinceIso = new Date().toISOString();
+ const runTask = async () => {
+ await probe.eventually(() => probe.eval(() => {
+ const button = document.querySelector<HTMLButtonElement>('button[aria-label="Run task"]');
+ return Boolean(button && !button.disabled);
+ }), { within: 30_000, label: "Run task enabled after model synchronization", until: value => value === true });
+ await user.click("Run task");
+ };
await user.type("composer", world.prompt);
- await user.click("Run task");
+ await runTask();
await step("the real search and connector action are readable while the tool runs", async () => {
await user.see({ text: /Searched your connections for.*Slack list_channels/ }, { timeoutMs: 60_000 });
@@ -51,7 +58,7 @@ test("connector-backed tool calls show first-class branding and human-readable l
await step("a failed connector action stays identifiable and is not shown as successful", async () => {
await user.type("composer", world.failurePrompt);
- await user.click("Run task");
+ await runTask();
await user.see({ text: /^Reading history$/ }, { timeoutMs: 30_000 });
await user.see({ role: "button", label: /Read history failed/ }, { timeoutMs: 60_000 });
await user.see("Run task");
diff --git a/evals/worlds/library.ts b/evals/worlds/library.ts
index 44d8b565..65fb7404 100644
--- a/evals/worlds/library.ts
+++ b/evals/worlds/library.ts
@@ -353,9 +353,31 @@ export async function connectorBranding(seed: Seed) {
] }),
});
if (!workloads.ok) throw new Error("Could not arrange connector model turns.");
- const providerId = "connector-display";
const modelId = "connector-display-model";
- const app = await seed.desktop({ den, as: "admin", model: `${providerId}/${modelId}` });
+ const provider = await seed.api(den.admin, "/v1/llm-providers", {
+ method: "POST",
+ body: JSON.stringify({
+ name: "Connector display model", source: "custom", allMembers: true, memberIds: [], teamIds: [],
+ apiKey: "sk-connector-display-fixture",
+ customConfig: {
+ id: "connector-display", name: "Connector display model", npm: "@ai-sdk/openai-compatible",
+ options: { baseURL: `${den.mocks.connector.url}/v1` }, env: ["CONNECTOR_DISPLAY_API_KEY"],
+ models: [{ id: modelId, name: "Connector display model", tool_call: true }],
+ },
+ }),
+ });
+ const llmProvider = isRecord(provider.body) && isRecord(provider.body.llmProvider) ? provider.body.llmProvider : null;
+ if (provider.response.status !== 201 || typeof llmProvider?.id !== "string") {
+ throw new Error(`Connector fixture model setup failed: HTTP ${provider.response.status}`);
+ }
+ const providerId = llmProvider.id;
+ const connected = await seed.api(den.admin, `/v1/llm-providers/${encodeURIComponent(providerId)}/connect`);
+ if (!connected.response.ok) throw new Error(`Connector fixture model connection failed: HTTP ${connected.response.status}`);
+ const app = await seed.desktop({
+ den, as: "admin", model: `${providerId}/${modelId}`,
+ // The isolated Den fixture has no production trace-upload account.
+ env: { OMNIRUSH_DEV_MODE: "1", OMNIRUSH_SESSION_UPLOAD_OPTIONAL: "1" },
+ });
const workspace = await seed.workspace(app, seed.tmpPath("connector-tool-call-branding"));
await seed.evalIn(app, browserScript(async () => {
const info = await window.__OMNIRUSH_ELECTRON__?.invokeDesktop?.("omnirushServerInfo");