Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

fix(setup): use the local Intelligence dashboard for browser login (#811)

Sam Julien33406427f99a
diff --git a/README.md b/README.md
index 09a6a682..a35e74ea 100644
--- a/README.md
+++ b/README.md
@@ -98,7 +98,12 @@ A Bot is any endpoint speaking [AG-UI](https://github.com/ag-ui-protocol/ag-ui),
For fresh self-hosted setup, set `INTELLIGENCE_API_URL` to that deployment's
HTTPS API origin (or `http://localhost` for a local one) and run the helper
without the two `npx` commands. It opens an isolated Chrome or Edge window for
- your usual sign-in and project choice; install either browser first. It writes
+ your usual sign-in and project choice; install either browser first. For CLI-managed
+ local evaluation installs, it discovers the matching dashboard with the installed or
+ cached CopilotKit CLI and uses that dashboard for login and session requests. Discovery
+ runs offline with Bun, without downloading the CLI or requiring `npx` on your PATH.
+ If no CLI is available, manually hosted deployments use their API origin for sign-in. Complete the local auth
+ approval shown in the browser when prompted. The runtime API URL stays unchanged. It writes
the API URL, the key and the container, not `INTELLIGENCE_GATEWAY_WS_URL`, so
set that one to the same deployment yourself. Once `openbot` exists, the server assigns it
as the default by itself unless [Admin → Automatic Learning](docs/automatic-learning.md)
diff --git a/scripts/local-intelligence-cli.test.ts b/scripts/local-intelligence-cli.test.ts
new file mode 100644
index 00000000..b8a8c3db
--- /dev/null
+++ b/scripts/local-intelligence-cli.test.ts
@@ -0,0 +1,156 @@
+import { expect, test } from "bun:test";
+import { mkdtemp, mkdir, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join } from "node:path";
+import {
+ localCliEntrypoint,
+ runLocalIntelligenceCli,
+} from "./local-intelligence-cli";
+
+async function fixture(
+ run: (
+ root: string,
+ install: (path: string, version?: string, code?: string) => Promise<string>,
+ ) => Promise<void>,
+) {
+ const root = await mkdtemp(join(tmpdir(), "openbot-cli-discovery-"));
+ try {
+ await run(
+ root,
+ async (
+ path,
+ version = "4.99.0",
+ code = 'console.log(JSON.stringify({status:"completed",stacks:[],args:process.argv.slice(2),secret:process.env.INTELLIGENCE_API_KEY ?? null}));',
+ ) => {
+ await mkdir(path, { recursive: true });
+ await writeFile(
+ join(path, "package.json"),
+ JSON.stringify({
+ name: "copilotkit",
+ version,
+ bin: { copilotkit: "index.js" },
+ }),
+ );
+ const entry = join(path, "index.js");
+ await writeFile(entry, code);
+ return entry;
+ },
+ );
+ } finally {
+ await rm(root, { recursive: true, force: true });
+ }
+}
+const minimalPath = "/usr/bin:/bin:/usr/sbin:/sbin";
+
+test("minimal-PATH child uses an existing npm cache offline without exposing credentials", async () => {
+ await fixture(async (root, install) => {
+ await install(join(root, ".npm/_npx/cache/node_modules/copilotkit"));
+ const environment = {
+ PATH: minimalPath,
+ HOME: root,
+ npm_config_registry: "http://127.0.0.1:1",
+ INTELLIGENCE_API_KEY: "private-value",
+ };
+ const result = await runLocalIntelligenceCli(
+ ["local", "list", "--json"],
+ root,
+ { home: root, globalRoots: [], environment },
+ );
+ expect(result.exitCode).toBe(0);
+ expect(JSON.parse(result.stdout)).toEqual({
+ status: "completed",
+ stacks: [],
+ args: ["local", "list", "--json"],
+ secret: null,
+ });
+ });
+});
+
+test("installed CLI wins over cache; otherwise newest cached version wins deterministically", async () => {
+ await fixture(async (root, install) => {
+ await install(join(root, ".npm/_npx/z/node_modules/copilotkit"), "4.9.0");
+ await install(
+ join(root, ".npm/_npx/pre/node_modules/copilotkit"),
+ "4.10.0-beta.1",
+ );
+ await install(
+ join(root, ".npm/_npx/broken/node_modules/copilotkit"),
+ "invalid-version",
+ );
+ const latest = await install(
+ join(root, ".npm/_npx/a/node_modules/copilotkit"),
+ "4.10.0",
+ );
+ expect(
+ await localCliEntrypoint(root, { PATH: minimalPath }, root, []),
+ ).toBe(latest);
+ const installed = await install(
+ join(root, "node_modules/copilotkit"),
+ "4.8.0",
+ );
+ expect(
+ await localCliEntrypoint(root, { PATH: minimalPath }, root, []),
+ ).toBe(installed);
+ });
+});
+
+test("missing and malformed cache entries are unavailable without a registry invocation", async () => {
+ await fixture(async (root) => {
+ const cache = join(root, ".npm/_npx/partial/node_modules/copilotkit");
+ await mkdir(cache, { recursive: true });
+ await writeFile(join(cache, "package.json"), "not json");
+ expect(
+ await runLocalIntelligenceCli([], root, {
+ home: root,
+ globalRoots: [],
+ environment: { PATH: minimalPath },
+ }),
+ ).toEqual({ exitCode: 127, stdout: "" });
+ });
+});
+
+test("runtime failure is retained and stderr is not returned", async () => {
+ await fixture(async (root, install) => {
+ await install(
+ join(root, "node_modules/copilotkit"),
+ "4.0.0",
+ 'console.error("private diagnostic"); process.exit(2);',
+ );
+ expect(
+ await runLocalIntelligenceCli([], root, {
+ home: root,
+ globalRoots: [],
+ environment: { PATH: minimalPath },
+ }),
+ ).toEqual({ exitCode: 2, stdout: "" });
+ });
+});
+
+test("cancellation terminates discovery and timeout is a runtime failure", async () => {
+ await fixture(async (root, install) => {
+ await install(
+ join(root, "node_modules/copilotkit"),
+ "4.0.0",
+ "setInterval(() => {}, 1000);",
+ );
+ const controller = new AbortController();
+ const pending = runLocalIntelligenceCli([], root, {
+ home: root,
+ globalRoots: [],
+ environment: { PATH: minimalPath },
+ signal: controller.signal,
+ });
+ setTimeout(() => controller.abort(new Error("cancelled")), 50);
+ await expect(pending).rejects.toThrow("cancelled");
+ expect(
+ (
+ await runLocalIntelligenceCli([], root, {
+ home: root,
+ globalRoots: [],
+ environment: { PATH: minimalPath },
+ timeoutMs: 50,
+ })
+ ).exitCode,
+ ).toBe(1);
+ });
+});
diff --git a/scripts/local-intelligence-cli.ts b/scripts/local-intelligence-cli.ts
new file mode 100644
index 00000000..83d471ae
--- /dev/null
+++ b/scripts/local-intelligence-cli.ts
@@ -0,0 +1,151 @@
+import { spawn } from "node:child_process";
+import { readdir, readFile, stat } from "node:fs/promises";
+import { homedir } from "node:os";
+import {
+ delimiter,
+ dirname,
+ isAbsolute,
+ join,
+ relative,
+ resolve,
+} from "node:path";
+import {
+ cliEnvironment,
+ type CliResult,
+} from "../server/src/learning/ensure-container";
+
+/** Resolve only installed packages. Discovery must never contact a package registry. */
+export async function localCliEntrypoint(
+ directory: string,
+ environment: NodeJS.ProcessEnv = process.env,
+ home = homedir(),
+ globalRoots = [
+ "/opt/homebrew/lib/node_modules/copilotkit",
+ "/usr/local/lib/node_modules/copilotkit",
+ ],
+): Promise<string | undefined> {
+ const entry = async (root: string) => {
+ try {
+ const pkg = JSON.parse(
+ await readFile(join(root, "package.json"), "utf8"),
+ );
+ const bin = typeof pkg.bin === "string" ? pkg.bin : pkg.bin?.copilotkit;
+ if (
+ pkg.name !== "copilotkit" ||
+ typeof bin !== "string" ||
+ typeof pkg.version !== "string"
+ )
+ return;
+ // Validate before sorting; stale metadata must not break other cached installs.
+ Bun.semver.order(pkg.version, pkg.version);
+ const path = resolve(root, bin);
+ if (
+ isAbsolute(relative(root, path)) ||
+ relative(root, path).startsWith("..") ||
+ !(await stat(path)).isFile()
+ )
+ return;
+ return {
+ path,
+ version: pkg.version,
+ };
+ } catch {
+ // Partial/stale package-cache entries are not installed CLIs.
+ return;
+ }
+ };
+ const npmPrefix =
+ environment.npm_config_prefix ?? environment.NPM_CONFIG_PREFIX;
+ const installed = [
+ join(directory, "node_modules/copilotkit"),
+ join(
+ environment.BUN_INSTALL ?? join(home, ".bun"),
+ "install/global/node_modules/copilotkit",
+ ),
+ ...(environment.APPDATA
+ ? [join(environment.APPDATA, "npm/node_modules/copilotkit")]
+ : []),
+ ...(npmPrefix ? [join(npmPrefix, "lib/node_modules/copilotkit")] : []),
+ ...globalRoots,
+ ...(environment.PATH ?? "")
+ .split(delimiter)
+ .filter(Boolean)
+ .map((bin) => join(dirname(bin), "lib/node_modules/copilotkit")),
+ ];
+ for (const root of installed) {
+ const found = await entry(root);
+ if (found) return found.path;
+ }
+ // npm's npx cache is a complete installation with its dependency graph. Bun's raw
+ // package cache is not, so only Bun's installed global package is considered above.
+ const cacheRoot = join(
+ environment.npm_config_cache ??
+ environment.NPM_CONFIG_CACHE ??
+ join(home, ".npm"),
+ "_npx",
+ );
+ const cached: { path: string; version: string }[] = [];
+ const names = await readdir(cacheRoot).catch(() => [] as string[]);
+ for (const name of names.sort()) {
+ const found = await entry(join(cacheRoot, name, "node_modules/copilotkit"));
+ if (found) cached.push(found);
+ }
+ cached.sort(
+ (a, b) =>
+ Bun.semver.order(b.version, a.version) || a.path.localeCompare(b.path),
+ );
+ return cached[0]?.path;
+}
+
+/** Execute the installed CLI with Desktop's already-resolved Bun, even with Finder's PATH. */
+export async function runLocalIntelligenceCli(
+ args: string[],
+ directory: string,
+ options: {
+ signal?: AbortSignal;
+ environment?: NodeJS.ProcessEnv;
+ home?: string;
+ timeoutMs?: number;
+ globalRoots?: string[];
+ } = {},
+): Promise<CliResult> {
+ options.signal?.throwIfAborted();
+ const environment = options.environment ?? process.env;
+ const entry = await localCliEntrypoint(
+ directory,
+ environment,
+ options.home,
+ options.globalRoots,
+ );
+ options.signal?.throwIfAborted();
+ if (!entry) return { exitCode: 127, stdout: "" };
+ return new Promise((resolveResult, reject) => {
+ const child = spawn(process.execPath, [entry, ...args], {
+ cwd: directory,
+ env: cliEnvironment(environment),
+ stdio: ["ignore", "pipe", "pipe"],
+ });
+ let stdout = "";
+ const abort = () => child.kill("SIGKILL");
+ options.signal?.addEventListener("abort", abort, { once: true });
+ if (options.signal?.aborted) child.kill("SIGKILL");
+ let timedOut = false;
+ const timer = setTimeout(() => {
+ timedOut = true;
+ child.kill("SIGKILL");
+ }, options.timeoutMs ?? 10_000);
+ child.stdout.setEncoding("utf8");
+ child.stdout.on("data", (chunk: string) => {
+ stdout = (stdout + chunk).slice(0, 1_048_576);
+ });
+ child.stderr.resume();
+ const finish = (exitCode: number) => {
+ clearTimeout(timer);
+ options.signal?.removeEventListener("abort", abort);
+ if (options.signal?.aborted) reject(options.signal.reason);
+ else resolveResult({ exitCode: timedOut ? 1 : exitCode, stdout });
+ };
+ child.on("error", () => finish(127));
+ child.on("close", (code) => finish(code ?? 1));
+ });
+}
diff --git a/scripts/self-hosted-learning.test.ts b/scripts/self-hosted-learning.test.ts
index c0c182c6..80da9a2f 100644
--- a/scripts/self-hosted-learning.test.ts
+++ b/scripts/self-hosted-learning.test.ts
@@ -1,6 +1,7 @@
import { expect, test } from "bun:test";
import {
intelligenceApiOrigin,
+ learningSessionOrigin,
type SetupBrowser,
setupSelfHostedLearning,
} from "./self-hosted-learning";
@@ -14,6 +15,7 @@ type Response = { status: number; body: unknown };
function fixture(
override?: (url: string, method: string) => Response | undefined,
+ sessionOrigin = apiUrl,
) {
const calls: {
url: string;
@@ -27,8 +29,8 @@ function fixture(
navigations.push(url);
},
request: async (url, options) => {
- expect(new URL(url).origin).toBe(apiUrl);
- expect(options.headers.Origin).toBe(apiUrl);
+ expect(new URL(url).origin).toBe(sessionOrigin);
+ expect(options.headers.Origin).toBe(sessionOrigin);
expect(options.headers).not.toHaveProperty("Authorization");
expect(options.maxRedirects).toBe(0);
calls.push({ url, method: options.method, data: options.data });
@@ -255,3 +257,137 @@ test("API origins reject credentials, paths, and insecure non-loopback servers",
expect(() => intelligenceApiOrigin(value)).toThrow();
}
});
+
+const localApi = "http://localhost:61860";
+const localDashboard = "http://cpk.localhost:61861";
+const localStatus = (api = localApi, dashboard = localDashboard) => ({
+ status: "completed",
+ stacks: [{ urls: { api, dashboard } }],
+});
+
+test("local setup signs in and provisions through the dashboard while preserving the runtime API", async () => {
+ const f = fixture(undefined, localDashboard);
+ const connection = await setupSelfHostedLearning({
+ ...f.options,
+ apiUrl: localApi,
+ existingKey: "cpk-existing-runtime-key",
+ runCli: async (args) => {
+ expect(args).toEqual(["local", "list", "--json"]);
+ return { exitCode: 0, stdout: JSON.stringify(localStatus()) };
+ },
+ });
+ const login = new URL(f.navigations[0]);
+ expect(login.origin).toBe(localDashboard);
+ expect(login.pathname).toBe("/auth/signin");
+ expect(login.searchParams.get("callbackUrl")).toBe(`${localDashboard}/`);
+ expect(connection).toEqual({
+ apiUrl: localApi,
+ apiKey: "cpk-existing-runtime-key",
+ learningContainerId: "openbot",
+ });
+ expect(
+ f.calls.some(
+ (call) =>
+ call.url === `${localDashboard}${path}` && call.method === "POST",
+ ),
+ ).toBe(true);
+ expect(f.closed()).toBe(1);
+});
+
+test.each([
+ localStatus(localApi, "https://unrelated.example"),
+ localStatus(localApi, "http://cpk.localhost.evil.example:61861"),
+ localStatus(localApi, "http://user:[email protected]:61861"),
+ localStatus(localApi, `${localDashboard}/unexpected`),
+ { status: "error" },
+])(
+ "invalid or mismatched local status fails before opening a browser: %j",
+ async (status) => {
+ const f = fixture();
+ await expect(
+ setupSelfHostedLearning({
+ ...f.options,
+ apiUrl: localApi,
+ runCli: async () => ({ exitCode: 0, stdout: JSON.stringify(status) }),
+ }),
+ ).rejects.toThrow("Could not identify the local Intelligence dashboard");
+ expect(f.navigations).toHaveLength(0);
+ expect(f.calls).toHaveLength(0);
+ },
+);
+
+test("remote setup does not run the local CLI", async () => {
+ expect(
+ await learningSessionOrigin(apiUrl, async () => {
+ throw new Error("must not run");
+ }),
+ ).toBe(apiUrl);
+});
+
+test.each([
+ { exitCode: 1, stdout: "private diagnostic" },
+ { exitCode: 0, stdout: "invalid JSON" },
+])(
+ "local discovery errors are actionable and do not disclose CLI output",
+ async (result) => {
+ await expect(
+ learningSessionOrigin(localApi, async () => result),
+ ).rejects.toThrow("Could not identify the local Intelligence dashboard");
+ },
+);
+
+test("an unrelated local stack does not change a manually hosted deployment", async () => {
+ expect(
+ await learningSessionOrigin(localApi, async () => ({
+ exitCode: 0,
+ stdout: JSON.stringify(localStatus("http://localhost:9999")),
+ })),
+ ).toBe(localApi);
+});
+
+test("multiple stacks are matched by API origin, not selection or list order", async () => {
+ expect(
+ await learningSessionOrigin(localApi, async () => ({
+ exitCode: 0,
+ stdout: JSON.stringify({
+ status: "completed",
+ stacks: [
+ ...localStatus("http://localhost:9999", "http://cpk.localhost:9998")
+ .stacks,
+ ...localStatus().stacks,
+ ],
+ }),
+ })),
+ ).toBe(localDashboard);
+});
+
+test("unavailable CLI preserves a manually hosted loopback login", async () => {
+ expect(
+ await learningSessionOrigin(localApi, async () => ({
+ exitCode: 127,
+ stdout: "",
+ })),
+ ).toBe(localApi);
+});
+
+test.each([
+ "http://127.0.0.1:61860",
+ "http://[::1]:61860",
+ "http://LOCALHOST:61860/",
+])("matches loopback aliases: %s", async (origin) => {
+ expect(
+ await learningSessionOrigin(origin, async () => ({
+ exitCode: 0,
+ stdout: JSON.stringify(localStatus()),
+ })),
+ ).toBe(localDashboard);
+});
+
+test("matching preserves protocol and port", async () => {
+ expect(
+ await learningSessionOrigin("https://127.0.0.1:61860", async () => ({
+ exitCode: 0,
+ stdout: JSON.stringify(localStatus()),
+ })),
+ ).toBe("https://127.0.0.1:61860");
+});
diff --git a/scripts/self-hosted-learning.ts b/scripts/self-hosted-learning.ts
index 4e76b2a3..867f6487 100644
--- a/scripts/self-hosted-learning.ts
+++ b/scripts/self-hosted-learning.ts
@@ -2,6 +2,8 @@ import { createInterface } from "node:readline";
import { setTimeout as delay } from "node:timers/promises";
import { parseArgs } from "node:util";
import { chromium } from "playwright-core";
+import type { RunCli } from "../server/src/learning/ensure-container";
+import { runLocalIntelligenceCli } from "./local-intelligence-cli";
export type SetupProject = { id: string; name: string };
export type LearningConnection = {
@@ -59,6 +61,69 @@ export function intelligenceApiOrigin(value: string): string {
return url.origin;
}
+function loopbackOrigin(value: string): string {
+ const url = new URL(intelligenceApiOrigin(value));
+ if (["localhost", "127.0.0.1", "[::1]"].includes(url.hostname))
+ url.hostname = "localhost";
+ return url.origin;
+}
+
+/** Local evaluation authenticates and proxies session requests on its dashboard origin. */
+export async function learningSessionOrigin(
+ apiUrl: string,
+ run: RunCli = runLocalIntelligenceCli,
+): Promise<string> {
+ if (!["localhost", "127.0.0.1", "[::1]"].includes(new URL(apiUrl).hostname))
+ return apiUrl;
+ try {
+ const result = await run(["local", "list", "--json"], process.cwd());
+ if (result.exitCode === 127) return apiUrl;
+ const status: unknown = JSON.parse(result.stdout);
+ if (
+ result.exitCode !== 0 ||
+ !object(status) ||
+ status.status !== "completed" ||
+ !Array.isArray(status.stacks)
+ )
+ throw new Error();
+ const matches = status.stacks.filter(
+ (stack: unknown) =>
+ object(stack) &&
+ object(stack.urls) &&
+ typeof stack.urls.api === "string" &&
+ loopbackOrigin(stack.urls.api) === loopbackOrigin(apiUrl),
+ );
+ // A manually hosted local deployment has no CLI record; retain its existing API login flow.
+ if (matches.length === 0) return apiUrl;
+ if (matches.length !== 1) throw new Error();
+ const stack: unknown = matches[0];
+ if (
+ !object(stack) ||
+ !object(stack.urls) ||
+ typeof stack.urls.dashboard !== "string"
+ )
+ throw new Error();
+ const dashboard = new URL(stack.urls.dashboard);
+ if (
+ dashboard.protocol !== "http:" ||
+ !["localhost", "cpk.localhost", "127.0.0.1", "[::1]"].includes(
+ dashboard.hostname,
+ ) ||
+ dashboard.username ||
+ dashboard.password ||
+ dashboard.pathname !== "/" ||
+ dashboard.search ||
+ dashboard.hash
+ )
+ throw new Error();
+ return dashboard.origin;
+ } catch {
+ throw new SetupError(
+ "Could not identify the local Intelligence dashboard for this API. Check that the local Intelligence installation is available and that its API URL is correct, then try again.",
+ );
+ }
+}
+
async function openBrowser(signal: AbortSignal): Promise<SetupBrowser> {
signal.throwIfAborted();
const missingBrowser = (error: unknown) =>
@@ -139,6 +204,7 @@ export async function setupSelfHostedLearning(options: {
) => Promise<string>;
signal?: AbortSignal;
openBrowser?: (signal: AbortSignal) => Promise<SetupBrowser>;
+ runCli?: RunCli;
pollIntervalMs?: number;
timeoutMs?: number;
}): Promise<LearningConnection> {
@@ -164,6 +230,13 @@ export async function setupSelfHostedLearning(options: {
};
signal.addEventListener("abort", cancel, { once: true });
try {
+ signal.throwIfAborted();
+ const sessionOrigin = await learningSessionOrigin(
+ apiUrl,
+ options.runCli ??
+ ((args, directory) =>
+ runLocalIntelligenceCli(args, directory, { signal })),
+ );
signal.throwIfAborted();
session = await (options.openBrowser ?? openBrowser)(signal);
signal.throwIfAborted();
@@ -174,16 +247,16 @@ export async function setupSelfHostedLearning(options: {
data?: Record<string, string>,
) => {
signal.throwIfAborted();
- return browser.request(`${apiUrl}${path}`, {
+ return browser.request(`${sessionOrigin}${path}`, {
method,
- headers: { Origin: apiUrl, Accept: "application/json" },
+ headers: { Origin: sessionOrigin, Accept: "application/json" },
...(data ? { data } : {}),
maxRedirects: 0,
signal,
});
};
- const signIn = new URL("/auth/signin", apiUrl);
- signIn.searchParams.set("callbackUrl", `${apiUrl}/`);
+ const signIn = new URL("/auth/signin", sessionOrigin);
+ signIn.searchParams.set("callbackUrl", `${sessionOrigin}/`);
await session.navigate(signIn.toString());
while (true) {
const me = await request("/api/me");