Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

18.2 KB
Raw Blame
import { describe, expect, test } from "bun:test";
import {
  type CopilotKitIntelligence,
  LearnedSkillsError,
} from "@copilotkit/runtime/v2";
import {
  isLearningContainerId,
  type LearningSettings,
} from "@openbot/shared/learning";
import type { LearningSpaceState } from "@openbot/shared/recorder-api";
import type { LearningContainerSpec } from "../src/learning/ensure-container";
import {
  classifyLearningSpaceProbe,
  createPersonalLearningSpaces,
  type PersonalLearningSpaces,
  type PersonSpaceStore,
  personalLearningContainerId,
} from "../src/learning/personal-space";
import { createTrajectoryLearningSelector } from "../src/recorder/trajectory-gate";
import { skillSnapshot } from "./fixtures/learned-skills";

type Snapshots = Pick<CopilotKitIntelligence, "getLearnedSkillsSnapshot">;
type Call = { containerId: string; signal?: AbortSignal };

const enabled: LearningSettings = {
  enabled: true,
  defaultTarget: { containerId: "openbot" },
  agents: {},
};
const ADA = "person-ada";
const ADA_SPACE = personalLearningContainerId(ADA);
const wait = (ms: number) =>
  new Promise<void>((resolve) => setTimeout(resolve, ms));
const failWith = (code: string) => async () => {
  throw new LearnedSkillsError(code as never, false);
};

function memoryStore(): PersonSpaceStore & { rows: Map<string, string> } {
  const rows = new Map<string, string>();
  return {
    rows,
    confirmed: async (userId) => rows.has(userId),
    confirm: async (userId, containerId) => {
      if (!rows.has(userId)) rows.set(userId, containerId);
    },
    forget: async (userId) => {
      rows.delete(userId);
    },
  };
}

function harness(
  options: {
    settings?: LearningSettings | (() => Promise<LearningSettings>);
    probe?: (call: Call) => Promise<unknown>;
    create?: (
      container: LearningContainerSpec,
    ) => Promise<"created" | "present" | "skipped">;
    probeTimeoutMs?: number;
  } = {},
) {
  const probes: Call[] = [];
  const creates: LearningContainerSpec[] = [];
  const logs: Record<string, unknown>[] = [];
  let clock = 1_000_000;
  const store = memoryStore();
  const settings = options.settings ?? enabled;
  const spaces = createPersonalLearningSpaces({
    readSettings:
      typeof settings === "function" ? settings : async () => settings,
    snapshots: {
      getLearnedSkillsSnapshot: async (call: Call) => {
        probes.push(call);
        return options.probe ? options.probe(call) : skillSnapshot();
      },
    } as unknown as Snapshots,
    store,
    ...(options.create
      ? {
          create: async (container: LearningContainerSpec) => {
            creates.push(container);
            return (options.create as NonNullable<typeof options.create>)(
              container,
            );
          },
        }
      : {}),
    nameFor: async (userId) => (userId === ADA ? "Ada Lovelace" : null),
    probeTimeoutMs: options.probeTimeoutMs,
    now: () => clock,
    log: (entry) => logs.push(entry),
  });
  return {
    spaces,
    probes,
    creates,
    logs,
    store,
    advance: (ms: number) => {
      clock += ms;
    },
  };
}

describe("a person's Learning space id", () => {
  test("is a valid Intelligence container id, derived from the person, the same on every call", () => {
    for (const userId of [
      "dev-local-user",
      "aB3dE9fGhIjKlMnOpQrStUvWxYz01234",
      "[email protected]",
      "x".repeat(500),
    ]) {
      const id = personalLearningContainerId(userId);
      expect(isLearningContainerId(id)).toBe(true);
      expect(id).toMatch(/^user-[0-9a-f]{32}$/);
      expect(personalLearningContainerId(userId)).toBe(id);
    }
  });

  test("two people never share one, even when their ids differ only in case", () => {
    expect(personalLearningContainerId("AbC")).not.toBe(
      personalLearningContainerId("abc"),
    );
  });
});

describe("the person's Learning space", () => {
  test("Automatic Learning off is unassigned: nothing is asked or created", async () => {
    const h = harness({
      settings: { ...enabled, enabled: false },
      create: async () => "created",
    });
    expect(await h.spaces.state(ADA)).toEqual({
      state: "unassigned",
      containerId: null,
    });
    expect(h.probes).toHaveLength(0);
    expect(h.creates).toHaveLength(0);
  });

  test("a space Intelligence has is assigned, probed by the person's own id, and remembered", async () => {
    const h = harness();
    expect(await h.spaces.state(ADA)).toEqual({
      state: "assigned",
      containerId: ADA_SPACE,
    });
    expect(h.probes.map((call) => call.containerId)).toEqual([ADA_SPACE]);
    await wait(1);
    expect(h.store.rows.get(ADA)).toBe(ADA_SPACE);
  });

  test("a missing space is created on first need, named for the person, and then assigned", async () => {
    let exists = false;
    const h = harness({
      probe: async () => {
        if (!exists) throw new LearnedSkillsError("CONTAINER_NOT_FOUND", false);
        throw new LearnedSkillsError("REVISION_NOT_FOUND", false);
      },
      create: async () => {
        exists = true;
        return "created";
      },
    });
    expect(await h.spaces.state(ADA)).toEqual({
      state: "assigned",
      containerId: ADA_SPACE,
    });
    expect(h.creates).toEqual([
      { id: ADA_SPACE, name: "OpenBot: Ada Lovelace" },
    ]);
    expect(h.store.rows.get(ADA)).toBe(ADA_SPACE);
  });

  test("concurrent requests for one person create the space once", async () => {
    let exists = false;
    let release: () => void = () => {};
    const gate = new Promise<void>((resolve) => {
      release = resolve;
    });
    const h = harness({
      probe: async () => {
        if (!exists) throw new LearnedSkillsError("CONTAINER_NOT_FOUND", false);
        return skillSnapshot();
      },
      create: async () => {
        await gate;
        exists = true;
        return "created";
      },
    });
    const answers = Promise.all(
      Array.from({ length: 10 }, () => h.spaces.state(ADA)),
    );
    await wait(5);
    release();
    for (const answer of await answers)
      expect(answer).toEqual({ state: "assigned", containerId: ADA_SPACE });
    expect(h.creates).toHaveLength(1);
  });

  test("a space that is missing with no credential to make it is missing, and joins nothing", async () => {
    const h = harness({ probe: failWith("CONTAINER_NOT_FOUND") });
    expect(await h.spaces.state(ADA)).toEqual({
      state: "missing",
      containerId: null,
    });
  });

  test("a failed creation is missing, is not tried again for a minute, then is", async () => {
    const h = harness({
      probe: failWith("CONTAINER_NOT_FOUND"),
      create: async () => {
        throw new Error("Intelligence refused the service token for Learning.");
      },
    });
    expect((await h.spaces.state(ADA)).state).toBe("missing");
    expect((await h.spaces.state(ADA)).state).toBe("missing");
    expect(h.creates).toHaveLength(1);
    expect(h.logs).toContainEqual(
      expect.objectContaining({
        type: "learning-personal-space",
        outcome: "create_failed",
        containerId: ADA_SPACE,
      }),
    );
    h.advance(60_001);
    await h.spaces.state(ADA);
    expect(h.creates).toHaveLength(2);
  });

  test("a creation still running when the caller stops waiting is unverified with no container, and finishes anyway", async () => {
    let exists = false;
    const h = harness({
      probe: async () => {
        if (!exists) throw new LearnedSkillsError("CONTAINER_NOT_FOUND", false);
        return skillSnapshot();
      },
      create: async () => {
        await wait(40);
        exists = true;
        return "created";
      },
    });
    expect(await h.spaces.state(ADA, AbortSignal.timeout(10))).toEqual({
      state: "unverified",
      containerId: null,
    });
    await wait(60);
    expect(h.store.rows.get(ADA)).toBe(ADA_SPACE);
    expect(await h.spaces.state(ADA)).toEqual({
      state: "assigned",
      containerId: ADA_SPACE,
    });
    expect(h.creates).toHaveLength(1);
  });

  test("a space made in another project is a failure, not a space", async () => {
    const h = harness({
      probe: failWith("CONTAINER_NOT_FOUND"),
      create: async () => "created",
    });
    expect((await h.spaces.state(ADA)).state).toBe("missing");
    expect(h.store.rows.has(ADA)).toBe(false);
    expect(h.logs).toContainEqual(
      expect.objectContaining({ outcome: "create_failed" }),
    );
  });

  test.each(["UNSUPPORTED_SERVER", "TIMEOUT"] as const)(
    "Intelligence unable to say (%s) still names the person's container, so a pass joins it when it is there",
    async (code) => {
      const h = harness({ probe: failWith(code) });
      expect(await h.spaces.state(ADA)).toEqual({
        state: classifyLearningSpaceProbe({
          code,
        }) as LearningSpaceState["state"],
        containerId: ADA_SPACE,
      });
    },
  );

  test("Intelligence unable to say, with a credential, makes the space (already there counts) and calls it assigned", async () => {
    const h = harness({
      probe: failWith("UNSUPPORTED_SERVER"),
      create: async () => "present",
    });
    expect(await h.spaces.state(ADA)).toEqual({
      state: "assigned",
      containerId: ADA_SPACE,
    });
    expect(h.store.rows.get(ADA)).toBe(ADA_SPACE);
  });

  test("a space seen before is named without asking a credential again", async () => {
    const h = harness({
      probe: failWith("UNSUPPORTED_SERVER"),
      create: async () => "created",
    });
    await h.store.confirm(ADA, ADA_SPACE);
    expect(await h.spaces.state(ADA)).toEqual({
      state: "learning-off",
      containerId: ADA_SPACE,
    });
    expect(h.creates).toHaveLength(0);
  });

  test("a space removed in Intelligence is forgotten and made again", async () => {
    let exists = false;
    const h = harness({
      probe: async () => {
        if (!exists) throw new LearnedSkillsError("CONTAINER_NOT_FOUND", false);
        return skillSnapshot();
      },
      create: async () => {
        exists = true;
        return "created";
      },
    });
    await h.store.confirm(ADA, "stale-row");
    expect((await h.spaces.state(ADA)).state).toBe("assigned");
    expect(h.store.rows.get(ADA)).toBe(ADA_SPACE);
  });

  test("an answer is cached for a minute; an unverified one is not", async () => {
    const h = harness();
    await h.spaces.state(ADA);
    await h.spaces.state(ADA);
    expect(h.probes).toHaveLength(1);
    h.advance(60_001);
    await h.spaces.state(ADA);
    expect(h.probes).toHaveLength(2);

    const flaky = harness({ probe: failWith("NETWORK_ERROR") });
    await flaky.spaces.state(ADA);
    await flaky.spaces.state(ADA);
    expect(flaky.probes).toHaveLength(2);
  });

  test("settings that cannot be read are unverified with no container", async () => {
    const h = harness({
      settings: async () => {
        throw new Error("database down");
      },
    });
    expect(await h.spaces.state(ADA)).toEqual({
      state: "unverified",
      containerId: null,
    });
  });

  test("the probe stops at its timeout, and when the caller's signal aborts", async () => {
    const hangs = (call: Call) =>
      new Promise((_, reject) => {
        call.signal?.addEventListener("abort", () =>
          reject(new LearnedSkillsError("TIMEOUT", true)),
        );
      });
    const timed = harness({ probe: hangs, probeTimeoutMs: 20 });
    expect((await timed.spaces.state(ADA)).state).toBe("unverified");
    const caller = new AbortController();
    const cut = harness({ probe: hangs, probeTimeoutMs: 10_000 });
    const answer = cut.spaces.state(ADA, caller.signal);
    await wait(5);
    caller.abort(new Error("stop"));
    expect((await answer).state).toBe("unverified");
  });

  test("kick makes the space in the background and never throws", async () => {
    let exists = false;
    const h = harness({
      probe: async () => {
        if (!exists) throw new LearnedSkillsError("CONTAINER_NOT_FOUND", false);
        return skillSnapshot();
      },
      create: async () => {
        exists = true;
        return "created";
      },
    });
    h.spaces.kick(ADA);
    await wait(5);
    expect(h.store.rows.get(ADA)).toBe(ADA_SPACE);
  });
});

describe("classifyLearningSpaceProbe", () => {
  test.each([
    ["REVISION_NOT_FOUND", "assigned"],
    ["DELIVERY_DISABLED", "assigned"],
    ["REVISION_REVOKED", "assigned"],
    ["CONTAINER_NOT_FOUND", "missing"],
    ["ENTITLEMENT_REQUIRED", "learning-off"],
    ["AUTHORIZATION_FAILED", "learning-off"],
    ["UNSUPPORTED_SERVER", "learning-off"],
    ["TIMEOUT", "unverified"],
    ["SOMETHING_NEW", "unverified"],
  ] as const)("%s is %s", (code, state) => {
    expect(classifyLearningSpaceProbe({ code })).toBe(state);
  });

  test("success is assigned", () => {
    expect(classifyLearningSpaceProbe("ok")).toBe("assigned");
  });
});

describe("the trajectory Learning space selector", () => {
  const TRAJECTORY = "0b6c1c0e-6f0a-4f6e-9a55-1d1c2f3f4a5b";
  const PERSON = { id: ADA, name: "Ada" };
  const OTHER = { id: "person-grace", name: "Grace" };
  type State = PersonalLearningSpaces["state"];
  const answers =
    (learning: LearningSpaceState): State =>
    async () =>
      learning;
  /** A check that keeps going until its signal aborts, then answers what it knows. */
  const stopsOnAbort =
    (learning: LearningSpaceState): State =>
    (_userId, signal) =>
      new Promise((resolve) => {
        signal?.addEventListener("abort", () => resolve(learning));
      });

  function selector(
    options: {
      state?: State;
      isBoundTo?: (trajectoryId: string, userId: string) => Promise<boolean>;
      budgetMs?: number;
    } = {},
  ) {
    const checks: { userId: string; signal?: AbortSignal }[] = [];
    const state =
      options.state ?? answers({ state: "assigned", containerId: ADA_SPACE });
    const select = createTrajectoryLearningSelector({
      isBoundTo:
        options.isBoundTo ??
        (async (trajectoryId, userId) =>
          userId === PERSON.id && trajectoryId === TRAJECTORY),
      personalSpace: {
        state: (userId, signal) => {
          checks.push({ userId, signal });
          return state(userId, signal);
        },
      },
      budgetMs: options.budgetMs,
    });
    const pick = async (signal?: AbortSignal, user = PERSON) =>
      select({ trajectoryId: TRAJECTORY, user }, signal);
    return { pick, checks };
  }

  test("returns the person's own container, whichever Bot they later use", async () => {
    const { pick, checks } = selector();
    expect(await pick()).toEqual([ADA_SPACE]);
    expect(checks.map((check) => check.userId)).toEqual([ADA]);
  });

  test("returns the person's container end to end, created on the spot by the real space check", async () => {
    let exists = false;
    const h = harness({
      probe: async () => {
        if (!exists) throw new LearnedSkillsError("CONTAINER_NOT_FOUND", false);
        return skillSnapshot();
      },
      create: async () => {
        exists = true;
        return "created";
      },
    });
    const { pick } = selector({ state: h.spaces.state });
    expect(await pick()).toEqual([ADA_SPACE]);
    expect(h.creates).toHaveLength(1);
  });

  test.each([
    { state: "unassigned", containerId: null },
    { state: "missing", containerId: null },
    { state: "unverified", containerId: null },
  ] as const)("$state with no container joins none", async (learning) => {
    const { pick } = selector({ state: answers(learning) });
    expect(await pick()).toBeNull();
  });

  test.each(["assigned", "learning-off", "unverified"] as const)(
    "%s with the container returns it, so a pass joins the same space whatever one probe found",
    async (state) => {
      const { pick } = selector({
        state: answers({ state, containerId: ADA_SPACE }),
      });
      expect(await pick()).toEqual([ADA_SPACE]);
    },
  );

  test("a trajectory bound to nobody, or to another person, returns null and is never checked", async () => {
    const { pick, checks } = selector();
    expect(await pick(undefined, OTHER)).toBeNull();
    await wait(5);
    expect(checks).toEqual([]);
  });

  test("a binding lookup that fails returns null", async () => {
    const { pick, checks } = selector({
      isBoundTo: async () => {
        throw new Error("database down");
      },
    });
    expect(await pick()).toBeNull();
    await wait(5);
    expect(checks).toEqual([]);
  });

  test("a check that throws still answers, with no container", async () => {
    const { pick } = selector({
      state: async () => {
        throw new Error("boom");
      },
    });
    expect(await pick()).toBeNull();
  });

  test("a check that stops when the budget nears its end still sends its container", async () => {
    const { pick, checks } = selector({
      state: stopsOnAbort({ state: "unverified", containerId: ADA_SPACE }),
      budgetMs: 30,
    });
    expect(await pick()).toEqual([ADA_SPACE]);
    expect(checks[0]?.signal?.aborted).toBe(true);
  });

  test("a check that never answers is cut at the budget with no container", async () => {
    const { pick, checks } = selector({
      state: () => new Promise(() => {}),
      budgetMs: 30,
    });
    const started = performance.now();
    expect(await pick()).toBeNull();
    expect(performance.now() - started).toBeGreaterThanOrEqual(25);
    expect(checks[0]?.signal?.aborted).toBe(true);
  });

  test("a binding lookup slower than the budget answers null and checks nothing", async () => {
    const { pick, checks } = selector({
      isBoundTo: async () => {
        await wait(60);
        return true;
      },
      budgetMs: 20,
    });
    expect(await pick()).toBeNull();
    await wait(80);
    expect(checks).toEqual([]);
  });

  test("the runtime giving up mid-check answers at once and stops the check", async () => {
    const runtime = new AbortController();
    const { pick, checks } = selector({
      state: stopsOnAbort({ state: "unverified", containerId: ADA_SPACE }),
      budgetMs: 10_000,
    });
    const answer = pick(runtime.signal);
    await wait(10);
    runtime.abort(new Error("deadline"));
    expect(await answer).toBeNull();
    expect(checks[0]?.signal?.aborted).toBe(true);
  });

  test("a signal already aborted answers without checking", async () => {
    const runtime = new AbortController();
    runtime.abort(new Error("gone"));
    const { pick, checks } = selector();
    expect(await pick(runtime.signal)).toBeNull();
    await wait(5);
    expect(checks).toEqual([]);
  });
});