Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

21.9 KB
Raw Blame
import {
  afterAll,
  afterEach,
  beforeAll,
  describe,
  expect,
  test,
} from "bun:test";
import type Docker from "dockerode";
import { legacyNamesFor, NAMESPACE, namesFor } from "../../src/names";

// Launched by docker.integration.test.ts in a fresh process: names.ts reads its namespace once.
const namespace = NAMESPACE;
if (
  !namespace.startsWith("supervisor-test-") ||
  namespace !== process.env.COMPUTER_NAMESPACE
) {
  throw new Error(
    "This fixture requires its own supervisor-test namespace before importing the supervisor.",
  );
}

const SOCKET = process.env.DOCKER_SOCKET ?? "/var/run/docker.sock";

const { default: DockerClient } = await import("dockerode");
const supervisor = await import("../../src/docker");
const runtime: { docker: Docker; supervisor: typeof supervisor } = {
  docker: new DockerClient({ socketPath: SOCKET }),
  supervisor,
};
if (!(await supervisor.reachable()))
  throw new Error("The parent verified Docker; the child must run every case.");
function withDocker() {
  return runtime;
}

const IMAGE = `openbot-supervisor-fixture:${namespace}-one`;
const OTHER = `openbot-supervisor-fixture:${namespace}-two`;
const fixtureImages: string[] = [];

const BOT = "supervisortestbot";
const result = namesFor("bot", BOT);
if (!result.ok) throw new Error(result.reason);
const names = result.names;

/** People and a Bot from before one computer per person, for the cases that need them. */
function named(kind: "person" | "bot", id: string) {
  const found = namesFor(kind, id);
  if (!found.ok) throw new Error(found.reason);
  return found.names;
}
const ADA = named("person", "ada");
const BO = named("person", "bo");
const LEGACY_BOT = "notesbot";
const legacyFound = legacyNamesFor(LEGACY_BOT);
if (!legacyFound.ok) throw new Error(legacyFound.reason);
const LEGACY = legacyFound.names;
/** What every computer of this supervisor's carried before kinds existed. */
const LEGACY_LABELS = {
  "openbot.supervisor": "true",
  "openbot.namespace": namespace,
  "openbot.bot-id": LEGACY_BOT,
};

function isMissing(error: unknown): boolean {
  return (
    typeof error === "object" &&
    error !== null &&
    "statusCode" in error &&
    error.statusCode === 404
  );
}

async function remove(container: string) {
  const docker = withDocker().docker;
  try {
    const info = await docker.getContainer(container).inspect();
    const labels = info.Config?.Labels;
    if (
      labels?.["openbot.test-fixture"] !== namespace &&
      !(
        labels?.["openbot.namespace"] === namespace &&
        labels?.["openbot.bot-id"] === BOT
      )
    ) {
      throw new Error("Refusing to clean a container outside this fixture.");
    }
    await docker.getContainer(info.Id).remove({ force: true });
  } catch (error) {
    if (!isMissing(error)) throw error;
  }
}

function createdAt(volume: Docker.VolumeInspectInfo): string {
  // Engine API exposes CreatedAt, but the installed Dockerode declarations omit it.
  if ("CreatedAt" in volume && typeof volume.CreatedAt === "string")
    return volume.CreatedAt;
  throw new Error("The daemon did not return a volume creation time.");
}

async function removeVolumes() {
  for (const volume of [names.profileVolume, names.workspaceVolume]) {
    try {
      const info = await withDocker().docker.getVolume(volume).inspect();
      if (
        info.Labels?.["openbot.namespace"] !== namespace ||
        info.Labels?.["openbot.bot-id"] !== BOT
      ) {
        throw new Error("Refusing to clean a volume outside this fixture.");
      }
      await withDocker().docker.getVolume(volume).remove();
    } catch (error) {
      if (!isMissing(error)) throw error;
    }
  }
}

beforeAll(async () => {
  // Refuse even an improbable collision before any cleanup or creation.
  for (const resource of [
    withDocker().docker.getContainer(names.container),
    withDocker().docker.getVolume(names.profileVolume),
    withDocker().docker.getVolume(names.workspaceVolume),
  ]) {
    try {
      await resource.inspect();
    } catch (error) {
      if (isMissing(error)) continue;
      throw error;
    }
    throw new Error("A fixture resource name was already present.");
  }
  for (const [index, tag] of [IMAGE, OTHER].entries()) {
    const stream = await withDocker().docker.buildImage(
      { context: import.meta.dir, src: ["Dockerfile"] },
      {
        t: tag,
        buildargs: {
          FIXTURE_NAMESPACE: namespace,
          FIXTURE_VARIANT: String(index),
        },
      },
    );
    await new Promise<void>((resolve, reject) => {
      withDocker().docker.modem.followProgress(stream, (error) =>
        error ? reject(error) : resolve(),
      );
    });
    const image = await withDocker().docker.getImage(tag).inspect();
    fixtureImages.push(image.Id);
  }
  expect(fixtureImages[0]).not.toBe(fixtureImages[1]);
}, 180_000);

let completedCases = 0;
afterAll(async () => {
  await remove(names.container);
  await removeVolumes();
  await removeNamespace();
  for (const image of fixtureImages) {
    const info = await withDocker().docker.getImage(image).inspect();
    if (info.Config.Labels?.["openbot.test-fixture"] !== namespace)
      throw new Error("Foreign fixture image");
    await withDocker().docker.getImage(image).remove();
  }
  console.log(
    "SUPERVISOR_FIXTURE " +
      JSON.stringify({
        namespace,
        names,
        fixtureImages,
        completedCases,
        cleanup: "complete",
      }),
  );
});

async function plant(labels: Record<string, string>, health?: boolean) {
  await remove(names.container);
  await withDocker().docker.createContainer({
    name: names.container,
    Image: IMAGE,
    Labels: { ...labels, "openbot.test-fixture": namespace },
    ...(health
      ? {}
      : {
          Cmd: ["sleep", "600"],
          Healthcheck: {
            Test: ["CMD-SHELL", "exit 1"],
            Interval: 1_000_000_000,
            Retries: 1,
            StartPeriod: 0,
          },
        }),
  });
}

const OURS = {
  "openbot.supervisor": "true",
  "openbot.namespace": namespace,
  "openbot.computer-kind": "bot",
  "openbot.bot-id": BOT,
};

/**
 * Everything this run made in its own namespace, which is unique to it: people's computers, a Bot's
 * old computer and volume, and any copy helper left behind.
 */
async function removeNamespace() {
  const docker = withDocker().docker;
  const containers = await docker.listContainers({
    all: true,
    filters: { label: [`openbot.namespace=${namespace}`] },
  });
  for (const container of containers) {
    await docker
      .getContainer(container.Id)
      .remove({ force: true })
      .catch((error: unknown) => {
        if (!isMissing(error)) throw error;
      });
  }
  const { Volumes } = await docker.listVolumes({
    filters: { label: [`openbot.namespace=${namespace}`] },
  });
  for (const volume of Volumes ?? []) {
    await docker
      .getVolume(volume.Name)
      .remove()
      .catch((error: unknown) => {
        if (!isMissing(error)) throw error;
      });
  }
}

afterEach(async () => {
  await remove(names.container);
  await removeVolumes();
  await removeNamespace();
  supervisor.forgetCarryOvers();
  completedCases += 1;
});

/** Run a shell line in the fixture image with these volumes mounted, and return what it printed. */
async function inVolumes(binds: string[], script: string): Promise<string> {
  const docker = withDocker().docker;
  const container = await docker.createContainer({
    Image: IMAGE,
    Labels: {
      "openbot.test-fixture": namespace,
      "openbot.namespace": namespace,
    },
    Entrypoint: ["sh", "-c"],
    Cmd: [script],
    Tty: true,
    HostConfig: { Binds: binds },
  });
  try {
    await container.start();
    await container.wait();
    return (await container.logs({ stdout: true, stderr: true }))
      .toString()
      .trim();
  } finally {
    await container.remove({ force: true });
  }
}

describe("a name held by somebody else", () => {
  test("is refused rather than adopted, and never started", async () => {
    // The container this supervisor did not make. Ownership is checked everywhere else so that a
    // name collision reads as absent; starting it on a 409 was the path that adopted it instead,
    // and an adopted container receives the deployment's computer token.
    await plant({ "someone.else": "true" });

    await expect(
      withDocker().supervisor.ensure(names, { image: IMAGE, environment: [] }),
    ).rejects.toBeInstanceOf(withDocker().supervisor.NameHeldError);

    const info = await withDocker()
      .docker.getContainer(names.container)
      .inspect();
    expect(info.State?.Running).toBe(false);
  }, 90_000);

  test("but a container this supervisor owns is still started", async () => {
    // The other direction, and the reason the check is ownership rather than existence: `ensure` is
    // idempotent, so the container a previous call left stopped has to come back up.
    await plant(OURS, true);

    const state = await withDocker().supervisor.ensure(names, {
      image: IMAGE,
      environment: [],
    });

    expect(state.status).toBe("running");
  }, 90_000);
});

describe("a computer that never answers", () => {
  test("fails instead of being handed out as ready", async () => {
    // A wait that cannot fail is a sleep: every computer that never came up was reported ready, and
    // the caller learned otherwise by sending it the deployment's token and getting a transport
    // error back.
    await plant(OURS);

    await expect(
      withDocker().supervisor.ensure(names, {
        image: IMAGE,
        environment: [],
        readyTimeoutMs: 3_000,
      }),
    ).rejects.toBeInstanceOf(withDocker().supervisor.ComputerNotAnsweringError);
  }, 90_000);
});

describe("a computer built from an older image", () => {
  /*
   * The upgrade that never reached the computers.
   *
   * `ensure` reused any container with the right name whatever it was built from, so once a Bot had
   * a computer, rebuilding the image moved the tag and the container went on running the old one
   * indefinitely, with nothing to say so. `docker compose down` does not touch these either, because
   * the supervisor makes them rather than compose, so even a full teardown left them behind.
   *
   * Found by rebuilding every image, restarting the whole stack, and watching a Bot's computer
   * answer with in-memory state from an hour before: a handover prompt about a page from a previous
   * conversation, offered on a new one.
   *
   * Two different images rather than a rebuild of one, because what the code compares is the
   * resolved id on either side and two tags is the cheapest way to have two of those.
   */
  test("is replaced, and keeps its profile and workspace", async () => {
    // A computer this supervisor owns, made the way it makes them, on the wrong image.
    await withDocker().supervisor.ensure(names, {
      image: OTHER,
      environment: [],
    });
    const before = await withDocker()
      .docker.getContainer(names.container)
      .inspect();

    // Record the actual named volume identities before replacement.
    // Volumes outlive the container by not being removed with it; that is what makes replacing one
    // safe, and it is the whole reason this fix is allowed to be automatic.
    const volumes = await Promise.all(
      [names.profileVolume, names.workspaceVolume].map((volume) =>
        withDocker().docker.getVolume(volume).inspect(),
      ),
    );

    const state = await withDocker().supervisor.ensure(names, {
      image: IMAGE,
      environment: [],
    });
    const after = await withDocker()
      .docker.getContainer(names.container)
      .inspect();

    expect(state).not.toBeNull();
    // A different container, on the image asked for.
    expect(before.State.Health?.Status).toBe("healthy");
    expect(after.State.Health?.Status).toBe("healthy");
    expect(after.Id).not.toBe(before.Id);
    expect(after.Image).not.toBe(before.Image);

    const wanted = await withDocker().docker.getImage(IMAGE).inspect();
    expect(after.Image).toBe(wanted.Id);

    // The same volumes, not replacements: a Bot keeps its logins and its files across an upgrade.
    const kept = await Promise.all(
      [names.profileVolume, names.workspaceVolume].map((volume) =>
        withDocker().docker.getVolume(volume).inspect(),
      ),
    );
    expect(kept.map(createdAt)).toEqual(volumes.map(createdAt));
  }, 180_000);

  /*
   * The install that reached the containers but not the computers.
   *
   * A computer checks callers against the `COMPUTER_TOKEN` it was created with and keeps that one
   * for the life of the container. Setting a machine up again from nothing mints a new token:
   * compose rebuilds everything it owns with it, and the computers, which the supervisor makes
   * rather than compose, survive holding the old one. Every call to them is then a 401 that no
   * screen can account for, because the gateway allowed the action and the trail says it was
   * carried out.
   *
   * Found on a first run of v0.0.9 against a computer container the install before it had made.
   */
  test("is replaced when it holds a token this deployment no longer uses", async () => {
    await withDocker().supervisor.ensure(names, {
      image: IMAGE,
      environment: [`COMPUTER_TOKEN=${"old-token"}`],
    });
    const before = await withDocker()
      .docker.getContainer(names.container)
      .inspect();

    const state = await withDocker().supervisor.ensure(names, {
      image: IMAGE,
      environment: [`COMPUTER_TOKEN=${"new-token"}`],
    });
    const after = await withDocker()
      .docker.getContainer(names.container)
      .inspect();

    expect(state).not.toBeNull();
    expect(after.Id).not.toBe(before.Id);
    expect(after.Config.Env).toContain("COMPUTER_TOKEN=new-token");
    expect(after.State.Health?.Status).toBe("healthy");
  }, 180_000);

  test("is left alone when it holds the token asked for", async () => {
    /*
     * The half that keeps this from replacing a working browser on every request, which is the same
     * risk the image comparison beside it carries. A deployment that sets no token at all is also
     * not a mismatch: that is a choice the environment made, not something to act on.
     */
    const environment = ["COMPUTER_TOKEN=steady"];
    await withDocker().supervisor.ensure(names, { image: IMAGE, environment });
    const before = await withDocker()
      .docker.getContainer(names.container)
      .inspect();

    await withDocker().supervisor.ensure(names, { image: IMAGE, environment });
    const withNone = await withDocker().supervisor.ensure(names, {
      image: IMAGE,
      environment: [],
    });
    const after = await withDocker()
      .docker.getContainer(names.container)
      .inspect();

    expect(withNone).not.toBeNull();
    expect(after.Id).toBe(before.Id);
  }, 180_000);

  test("is left alone when it is already the image asked for", async () => {
    /*
     * The other half, and the one that keeps this from being a fix that restarts every computer on
     * every request. `ensure` is called whenever a computer is needed, so a comparison that ever
     * reported stale for a current container would throw away a Bot's browser mid-task.
     */
    const first = await withDocker().supervisor.ensure(names, {
      image: IMAGE,
      environment: [],
    });
    const before = await withDocker()
      .docker.getContainer(names.container)
      .inspect();

    const second = await withDocker().supervisor.ensure(names, {
      image: IMAGE,
      environment: [],
    });
    const after = await withDocker()
      .docker.getContainer(names.container)
      .inspect();

    // Both calls must reach the real health check and keep the same running container.
    expect(before.State.Health?.Status).toBe("healthy");
    expect(after.State.Health?.Status).toBe("healthy");
    expect(first).not.toBeNull();
    expect(second).not.toBeNull();
    expect(after.Id).toBe(before.Id);
  }, 180_000);
});

describe("one computer per person", () => {
  test("a person's computer is named and labelled for that person", async () => {
    const state = await withDocker().supervisor.ensure(ADA, {
      image: IMAGE,
      environment: [],
    });
    expect(state).toMatchObject({ kind: "person", id: "ada" });
    const info = await withDocker()
      .docker.getContainer(ADA.container)
      .inspect();
    expect(info.Config.Labels).toMatchObject({
      "openbot.computer-kind": "person",
      "openbot.person-id": "ada",
    });
    const listed = await withDocker().supervisor.listOwned();
    expect(listed).toContainEqual(
      expect.objectContaining({
        kind: "person",
        id: "ada",
        container: ADA.container,
      }),
    );
  }, 90_000);

  test("an old computer under a name today's scheme also derives is refused, not adopted", async () => {
    /*
     * The old computer of a Bot called `bot-supervisortestbot` was named exactly what an isolated Bot
     * called `supervisortestbot` is named now. Ours by owner and namespace, but not this computer:
     * adopting it would hand one Bot another's logins.
     */
    await plant(
      { ...OURS, "openbot.computer-kind": "", "openbot.bot-id": `bot-${BOT}` },
      true,
    );
    await expect(
      withDocker().supervisor.ensure(names, { image: IMAGE, environment: [] }),
    ).rejects.toBeInstanceOf(withDocker().supervisor.NameHeldError);
  }, 90_000);
});

describe("a private Bot's login, carried over to its owner", () => {
  async function oldProfile(contents: string) {
    await withDocker()
      .docker.createVolume({
        Name: LEGACY.profileVolume,
        Labels: LEGACY_LABELS,
      })
      .catch(() => undefined);
    await inVolumes(
      [`${LEGACY.profileVolume}:/v`],
      `mkdir -p /v/Default && printf %s '${contents}' > /v/Default/Cookies`,
    );
  }
  const cookiesIn = (person: typeof ADA) =>
    inVolumes(
      [`${person.profileVolume}:/p`],
      `cat /p/${LEGACY_BOT}/Default/Cookies 2>/dev/null || echo none`,
    );

  test("is copied into the owner's computer under that Bot, once, and the old volume kept", async () => {
    await oldProfile("ada-login");
    await withDocker().supervisor.ensure(ADA, {
      image: IMAGE,
      environment: [],
    });

    expect(
      await withDocker().supervisor.carryOverProfile(ADA, LEGACY_BOT, {
        image: IMAGE,
      }),
    ).toBe("copied");
    expect(await cookiesIn(ADA)).toBe("ada-login");

    // Run again from a fresh process's memory, after the old profile changed: the marker in the
    // owner's volume says it was done, so nothing is copied over what the owner has now.
    withDocker().supervisor.forgetCarryOvers();
    await oldProfile("stale");
    expect(
      await withDocker().supervisor.carryOverProfile(ADA, LEGACY_BOT, {
        image: IMAGE,
      }),
    ).toBe("done");
    expect(await cookiesIn(ADA)).toBe("ada-login");

    // The old volume is left exactly where it was.
    const kept = await withDocker()
      .docker.getVolume(LEGACY.profileVolume)
      .inspect();
    expect(kept.Name).toBe(LEGACY.profileVolume);
  }, 180_000);

  test("brings the Bot's old files into its folder on the owner's computer, once", async () => {
    await oldProfile("ada-login");
    await withDocker()
      .docker.createVolume({
        Name: LEGACY.workspaceVolume,
        Labels: LEGACY_LABELS,
      })
      .catch(() => undefined);
    await inVolumes(
      [`${LEGACY.workspaceVolume}:/v`],
      "mkdir -p /v/reports && printf %s 'march totals' > /v/reports/march.txt",
    );
    withDocker().supervisor.forgetCarryOvers();
    await withDocker().supervisor.ensure(ADA, {
      image: IMAGE,
      environment: [],
    });
    await withDocker().supervisor.carryOverProfile(ADA, LEGACY_BOT, {
      image: IMAGE,
    });
    // Where the Bot's shell and file tools start on its person's computer (agent-computer's workspaces.ts).
    const filesIn = () =>
      inVolumes(
        [`${ADA.workspaceVolume}:/w`],
        `cat /w/computers/${LEGACY_BOT}/reports/march.txt 2>/dev/null || echo none`,
      );
    expect(await filesIn()).toBe("march totals");

    // Once: a later change to the old volume is not copied over what the Bot has now.
    withDocker().supervisor.forgetCarryOvers();
    await inVolumes(
      [`${LEGACY.workspaceVolume}:/v`],
      "printf %s stale > /v/reports/march.txt",
    );
    await withDocker().supervisor.carryOverProfile(ADA, LEGACY_BOT, {
      image: IMAGE,
    });
    expect(await filesIn()).toBe("march totals");
    expect(
      (await withDocker().docker.getVolume(LEGACY.workspaceVolume).inspect())
        .Name,
    ).toBe(LEGACY.workspaceVolume);
  }, 180_000);

  test("never overwrites a profile the owner's computer already has for that Bot", async () => {
    await oldProfile("old-login");
    await withDocker().supervisor.ensure(BO, { image: IMAGE, environment: [] });
    await inVolumes(
      [`${BO.profileVolume}:/p`],
      `mkdir -p /p/${LEGACY_BOT}/Default && printf %s newer > /p/${LEGACY_BOT}/Default/Cookies`,
    );

    expect(
      await withDocker().supervisor.carryOverProfile(BO, LEGACY_BOT, {
        image: IMAGE,
      }),
    ).toBe("kept");
    expect(await cookiesIn(BO)).toBe("newer");
  }, 180_000);

  test("the old computer is stopped when the supervisor starts, and its volume kept", async () => {
    await oldProfile("x");
    await withDocker().docker.createContainer({
      name: LEGACY.container,
      Image: IMAGE,
      Labels: { ...LEGACY_LABELS, "openbot.test-fixture": namespace },
      HostConfig: { Binds: [`${LEGACY.profileVolume}:/profiles`] },
    });
    await withDocker().docker.getContainer(LEGACY.container).start();

    const stopped = await withDocker().supervisor.retireLegacyComputers();
    expect(stopped).toContain(LEGACY.container);
    const info = await withDocker()
      .docker.getContainer(LEGACY.container)
      .inspect();
    expect(info.State.Running).toBe(false);
    // Not listed as one of today's computers either.
    const listed = await withDocker().supervisor.listOwned();
    expect(listed.map((computer) => computer.container)).not.toContain(
      LEGACY.container,
    );
    expect(
      (await withDocker().docker.getVolume(LEGACY.profileVolume).inspect())
        .Name,
    ).toBe(LEGACY.profileVolume);
  }, 120_000);
});