Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

14.1 KB
Raw Blame
/**
 * Real HTTP validation of the production native emitter and the installed runtime.
 * Build the native example separately, then run:
 * bun --no-env-file --no-install desktop/scripts/validate-telemetry.ts /absolute/path/to/telemetry_probe
 * No credentials, containers, model calls, or production telemetry endpoints are used.
 */
import assert from "node:assert/strict";
import { existsSync } from "node:fs";
import { mkdtemp, readFile, rm } from "node:fs/promises";
import { createRequire } from "node:module";
import { tmpdir } from "node:os";
import { join, resolve } from "node:path";
import { desktopTelemetryProperties } from "../../server/src/desktop-telemetry";

const root = resolve(import.meta.dir, "../..");
const fromServer = createRequire(join(root, "server/package.json"));
const uuid =
  /^[0-9a-f]{8}-[0-9a-f]{4}-4[0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/;
type Env = Record<string, string>;
type Captured = { identity: string | null; body: Record<string, unknown> };
type Queued = { event_id: string; event_name: string; occurred_at_ms: number };
type State = {
  install_id: string;
  queue: Queued[];
  activated: boolean;
  last_step: string | null;
};

function object(value: unknown): Record<string, unknown> {
  assert.ok(value && typeof value === "object" && !Array.isArray(value));
  return value as Record<string, unknown>;
}

function lastObject(stdout: string): Record<string, unknown> {
  const line = stdout.trim().split("\n").at(-1);
  assert.ok(line, "child did not return a JSON report");
  return object(JSON.parse(line));
}

function loopback(value: string): URL {
  const url = new URL(value);
  assert.equal(url.protocol, "http:");
  assert.equal(url.hostname, "127.0.0.1");
  return url;
}

/** Fresh process: the SDK snapshots opt-out and sampling before any runtime import. */
async function runtimeChild(endpoint: string) {
  loopback(endpoint);
  assert.equal(process.env.COPILOTKIT_TELEMETRY_URL, endpoint);
  const fetchHttp = globalThis.fetch;
  const sends: Promise<Response>[] = [];
  const blocked: string[] = [];
  const guardedFetch: typeof fetch = Object.assign(
    (
      input: Parameters<typeof fetch>[0],
      init?: Parameters<typeof fetch>[1],
    ) => {
      const url = new URL(input instanceof Request ? input.url : String(input));
      if (url.protocol !== "http:" || url.hostname !== "127.0.0.1") {
        blocked.push(url.origin);
        return Promise.reject(new Error("Validation forbids external fetches"));
      }
      const result = fetchHttp(input, init);
      if (url.href === endpoint) sends.push(result);
      return result;
    },
    { preconnect: fetchHttp.preconnect },
  );
  globalThis.fetch = guardedFetch;

  const runtimePackage = fromServer("@copilotkit/runtime/package.json");
  assert.equal(
    runtimePackage.version,
    "1.70.1",
    "recheck the wire contract before changing the installed version",
  );
  // Match production-entry.ts's preload and the server's public ESM imports. Requiring the
  // CJS runtime selects a different dependency graph and does not validate the shipped path.
  await import(Bun.resolveSync("eventsource", join(root, "server")));
  const { CopilotRuntime } = await import(
    Bun.resolveSync("@copilotkit/runtime/v2", join(root, "server"))
  );
  const { createCopilotHonoHandler } = await import(
    Bun.resolveSync("@copilotkit/runtime/v2/hono", join(root, "server"))
  );
  const runtime = new CopilotRuntime({
    agents: {},
    telemetryProperties: desktopTelemetryProperties(),
  });
  const handler = createCopilotHonoHandler({
    runtime,
    basePath: "/api/copilotkit",
  });
  const info = await handler.fetch(
    new Request("http://127.0.0.1/api/copilotkit/info"),
  );
  assert.equal(info.status, 200);
  await info.text();
  // Handler creation queues instance_created in a promise continuation. Drain it before
  // waiting on the real fetches; no transport is mocked and no polling timeout proves success.
  await new Promise<void>((done) => setImmediate(done));
  await Promise.all(sends);
  assert.deepEqual(blocked, []);
  console.log(
    JSON.stringify({
      runtimeVersion: runtimePackage.version,
      requests: sends.length,
    }),
  );
}

async function validate(nativePath: string) {
  assert.ok(
    existsSync(nativePath),
    `Build the native telemetry_probe example first: ${nativePath}`,
  );
  const scratch = await mkdtemp(
    join(tmpdir(), "openbot-telemetry-validation-"),
  );
  const received: Captured[] = [];
  const server = Bun.serve({
    hostname: "127.0.0.1",
    port: 0,
    async fetch(request) {
      assert.equal(request.method, "POST");
      assert.equal(new URL(request.url).pathname, "/ingest");
      const raw = await request.text();
      assert.ok(Buffer.byteLength(raw) <= 16 * 1024);
      received.push({
        identity: request.headers.get("X-CopilotKit-Telemetry-Id"),
        body: object(JSON.parse(raw)),
      });
      return Response.json({ ok: true }, { status: 202 });
    },
  });
  const endpoint = `http://127.0.0.1:${server.port}/ingest`;
  // Reserve and close a second local socket: a real connection-refused flush, not a fake transport.
  const offline = Bun.serve({
    hostname: "127.0.0.1",
    port: 0,
    fetch: () => new Response(),
  });
  const offlineEndpoint = `http://127.0.0.1:${offline.port}/ingest`;
  await offline.stop(true);

  const cleanEnv: Env = {
    PATH: process.env.PATH ?? "",
    NODE_ENV: "production",
    NO_COLOR: "1",
  };
  // NODE_PATH is an explicit caller-selected resolution aid for existing installations;
  // this script never installs packages or silently rewires their dependency graph.
  for (const key of ["SystemRoot", "WINDIR", "TEMP", "TMP", "NODE_PATH"]) {
    const value = process.env[key];
    if (value) cleanEnv[key] = value;
  }
  async function processRun(command: string[], env: Env, cwd = scratch) {
    const child = Bun.spawn(command, {
      cwd,
      env: { ...cleanEnv, ...env },
      stdout: "pipe",
      stderr: "pipe",
    });
    const timeout = setTimeout(() => child.kill(), 15_000);
    try {
      const [stdout, stderr, code] = await Promise.all([
        new Response(child.stdout).text(),
        new Response(child.stderr).text(),
        child.exited,
      ]);
      return { stdout, stderr, code };
    } finally {
      clearTimeout(timeout);
    }
  }
  function lastJson(stdout: string): Env {
    const parsed = lastObject(stdout);
    assert.ok(
      Object.values(parsed).every((value) => typeof value === "string"),
    );
    return parsed as Env;
  }
  async function native(
    directory: string,
    action: string,
    env: Env = {},
    url = endpoint,
  ) {
    const result = await processRun([nativePath, directory, url, action], env);
    assert.equal(
      result.code,
      0,
      `${action} failed: ${result.stderr.slice(0, 500)}`,
    );
    return lastJson(result.stdout);
  }
  async function state(directory: string): Promise<State> {
    return JSON.parse(
      await readFile(join(directory, "telemetry-state.json"), "utf8"),
    );
  }
  async function runtime(env: Env) {
    const result = await processRun(
      [
        process.execPath,
        "--no-env-file",
        "--no-install",
        import.meta.path,
        "--runtime-child",
        endpoint,
      ],
      { ...env, COPILOTKIT_TELEMETRY_URL: endpoint },
      join(root, "server"),
    );
    assert.equal(
      result.code,
      0,
      `runtime child failed: ${result.stderr.slice(0, 1000)}`,
    );
    return lastObject(result.stdout);
  }
  const checks: string[] = [];
  try {
    const recoveredDir = join(scratch, "recovered");
    const nativeEnv = await native(recoveredDir, "record", {}, offlineEndpoint);
    const original = await state(recoveredDir);
    assert.match(original.install_id, uuid);
    assert.equal(original.queue.length, 1);
    assert.equal(original.queue[0].event_name, "oss.desktop.step_viewed");
    assert.equal(nativeEnv.CPK_TELEMETRY_ID, original.install_id);
    assert.equal(nativeEnv.COPILOTKIT_TELEMETRY_SAMPLE_RATE, "1");
    const failed = await processRun(
      [nativePath, recoveredDir, offlineEndpoint, "flush"],
      {},
    );
    assert.notEqual(failed.code, 0, "offline connection must fail");
    const queued = await state(recoveredDir);
    assert.equal(queued.install_id, original.install_id);
    assert.equal(queued.queue.length, 2);
    assert.equal(queued.queue[0].event_id, original.queue[0].event_id);
    assert.equal(queued.queue[1].event_name, "oss.desktop.setup_abandoned");
    assert.equal(received.length, 0);
    const relaunchedEnv = await native(recoveredDir, "flush");
    assert.deepEqual(relaunchedEnv, nativeEnv);
    assert.deepEqual(
      received.map((request) => request.body.event_id),
      queued.queue.map((event) => event.event_id),
    );
    assert.ok(
      received.every((request) => request.identity === original.install_id),
    );
    assert.deepEqual(
      received.map((request) => request.body.ts),
      queued.queue.map((event) => Math.floor(event.occurred_at_ms / 1000)),
    );
    assert.equal((await state(recoveredDir)).queue.length, 0);
    await native(recoveredDir, "flush");
    assert.equal(
      received.length,
      2,
      "successful flush must not replay acknowledged events",
    );
    checks.push(
      "offline queue replay: stable installation and event IDs; one recovered abandonment",
    );

    const quitStart = received.length;
    await native(join(scratch, "quit"), "quit");
    assert.deepEqual(
      received.slice(quitStart).map((request) => request.body.event),
      ["oss.desktop.step_viewed", "oss.desktop.setup_abandoned"],
    );
    checks.push("quit flushes current step and abandonment");

    const activationStart = received.length;
    const activatedDir = join(scratch, "activated");
    await native(activatedDir, "activate");
    await native(activatedDir, "activate");
    assert.deepEqual(
      received.slice(activationStart).map((request) => request.body.event),
      ["oss.desktop.activated"],
    );
    assert.equal((await state(activatedDir)).activated, true);
    checks.push(
      "activation emitted once across duplicate calls and process relaunch",
    );

    const nativeRequests = received.length;
    for (const request of received) {
      assert.deepEqual(Object.keys(request.body).sort(), [
        "event",
        "event_id",
        "global_properties",
        "package",
        "properties",
        "ts",
      ]);
      assert.match(String(request.body.event_id), uuid);
      assert.deepEqual(request.body.package, {
        name: "openbot-desktop",
        version: "0.0.9",
      });
      assert.deepEqual(object(request.body.global_properties), {
        ...desktopTelemetryProperties(nativeEnv),
        runtime_env: "test",
        sampleRate: 1,
        sampleWeight: 1,
        sampleRateAdjustmentFactor: 0,
      });
    }
    checks.push(
      "native wire envelope uses closed metadata, seconds and full sampling",
    );

    const runtimeResult = await runtime({
      ...nativeEnv,
      OPENBOT_UNKNOWN: "synthetic-private-value",
    });
    assert.equal(runtimeResult.requests, 1);
    const runtimeRequest = received.at(-1);
    assert.ok(runtimeRequest);
    assert.equal(received.length, nativeRequests + 1);
    assert.equal(runtimeRequest.identity, original.install_id);
    assert.equal(runtimeRequest.body.event, "oss.runtime.instance_created");
    assert.deepEqual(runtimeRequest.body.package, {
      name: "@copilotkit/runtime",
      version: "1.70.1",
    });
    assert.deepEqual(runtimeRequest.body.properties, {
      actionsAmount: 0,
      endpointTypes: [],
      endpointsAmount: 0,
      agentsAmount: 0,
      "cloud.api_key_provided": false,
    });
    assert.deepEqual(runtimeRequest.body.global_properties, {
      ...desktopTelemetryProperties(nativeEnv),
      sampleRate: 1,
      sampleWeight: 1,
      sampleRateAdjustmentFactor: 0,
      telemetry_identified: false,
      telemetry_emitter: "v2-runtime",
      telemetry_transport: "lambda",
    });
    assert.ok(
      Number(runtimeRequest.body.ts) > 1_000_000_000 &&
        Number(runtimeRequest.body.ts) < 10_000_000_000,
    );
    checks.push(
      "installed runtime1.70.1 Hono boot event: same native UUID, closed metadata, full sampling",
    );

    for (const variable of ["COPILOTKIT_TELEMETRY_DISABLED", "DO_NOT_TRACK"]) {
      for (const value of ["true", "1"]) {
        const before: number = received.length;
        const directory = join(scratch, `${variable}-${value}`);
        await native(directory, "record", {}, offlineEndpoint);
        assert.ok((await state(directory)).queue.length > 0);
        const optedOutEnv = await native(directory, "quit", {
          [variable]: value,
        });
        assert.deepEqual(optedOutEnv, { COPILOTKIT_TELEMETRY_DISABLED: "1" });
        assert.equal(
          existsSync(join(directory, "telemetry-state.json")),
          false,
        );
        const fresh = join(scratch, `${variable}-${value}-fresh`);
        assert.deepEqual(
          await native(fresh, "env", { [variable]: value }),
          optedOutEnv,
        );
        assert.equal(existsSync(join(fresh, "telemetry-state.json")), false);
        // Test the SDK's own override too, even if a caller retained an older enabled ID.
        assert.equal(
          (await runtime({ ...nativeEnv, [variable]: value })).requests,
          0,
        );
        assert.equal(received.length, before);
        checks.push(
          `${variable}=${value}: queue purged, no identity or HTTP sends, runtime also disabled`,
        );
      }
    }
    assert.equal(
      (await runtime({ COPILOTKIT_TELEMETRY_DISABLED: "1" })).requests,
      0,
    );
    console.log(
      JSON.stringify(
        {
          ok: true,
          checks,
          nativeRequests,
          runtimeRequests: 1,
          runtimeVersion: "1.70.1",
          externalRequests: 0,
        },
        null,
        2,
      ),
    );
  } finally {
    await server.stop(true);
    await rm(scratch, { recursive: true, force: true });
  }
}

if (process.argv[2] === "--runtime-child") {
  await runtimeChild(process.argv[3]);
} else {
  const nativePath =
    process.argv[2] ??
    join(root, "desktop/src-tauri/target/release/examples/telemetry_probe");
  await validate(resolve(nativePath));
}