Skip to content

Open-source AI coworkers that each get a computer of their own: a browser, files and tools, with every action decided before it happens and recorded after. Bring any AG-UI agent.

38.9 KB
Raw Blame
import Docker from "dockerode";
import { type ContainerHealth, healthOf } from "./health";
import {
  BOT_LABEL,
  type ComputerKind,
  type ComputerNames,
  DEFAULT_NAMESPACE,
  isComputerKind,
  KIND_LABEL,
  legacyNamesFor,
  NAMESPACE,
  NAMESPACE_LABEL,
  OWNER_LABEL,
  PERSON_LABEL,
} from "./names";

/**
 * The only Docker this service knows how to do.
 *
 * `dockerode` provides Engine API stream handling, error shapes and version negotiation.
 *
 * There is no passthrough. There is no generic "run this Docker call" here,
 * because the whole reason the API server does not hold the socket is that the socket is unrestricted
 * root on the host, a supervisor that forwarded arbitrary calls would hand that straight back
 * through a politer door. Four verbs, expressed in whose computer it is, and nothing else.
 *
 * Every write is scoped by ownership. Containers are created carrying `openbot.supervisor`, and
 * stop, reset and inspect refuse anything without it. A container whose name happens to match but
 * lacks the label is treated as absent.
 */

const docker = new Docker(
  process.env.DOCKER_SOCKET
    ? { socketPath: process.env.DOCKER_SOCKET }
    : undefined,
);

/** The port the computer listens on inside its own container. */
const COMPUTER_PORT = "4100/tcp";

/**
 * Readiness, stated at create time rather than read off the image.
 *
 * `agent-computer/Dockerfile` declares the same HEALTHCHECK, and under Docker that was enough: the
 * image carried it, the daemon ran it, and {@link waitUntilAnswering} could ask. Podman does not
 * report it. Its images are OCI-manifest, the OCI image config has no healthcheck field, and the
 * instruction is dropped, both when Podman builds the image and when it pulls one that has it. The
 * published `agent-computer` config does carry it; `podman inspect` of that same image reports
 * none.
 *
 * Silently, and into the one branch that cannot tell the difference: with no health to read,
 * `waitUntilAnswering` accepts `Running`, and a container that is running is not a Chromium that
 * is answering. Every cold start of a computer then raced the first request, which arrived at a
 * port nothing was listening on yet and was reported as a computer that is not running.
 *
 * Passed here, the engine is told what to run instead of asked what it inherited, which is also
 * true on Docker and one less thing that depends on how an image was built. Podman honours an
 * explicit healthcheck: it is how every service in `docker-compose.yml` reports healthy there.
 */
const COMPUTER_HEALTHCHECK = {
  Test: [
    "CMD-SHELL",
    `bun -e "const r = await fetch('http://localhost:${COMPUTER_PORT.split("/")[0]}/health'); process.exit(r.ok ? 0 : 1)"`,
  ],
  // Nanoseconds, which is what the API takes. The same numbers the Dockerfile states.
  Interval: 2_000_000_000,
  Timeout: 3_000_000_000,
  StartPeriod: 2_000_000_000,
  Retries: 30,
};

/**
 * How many times `ensure` will build a computer before giving up.
 *
 * One retry, because the only thing being retried is losing a race to another request for the same
 * Bot. A second loss means something else is removing the container, and retrying forever would hide
 * that behind a hung request.
 */
const ATTEMPTS = 2;

function statusOf(error: unknown): number | undefined {
  return (error as { statusCode?: number }).statusCode;
}

/** One poll interval, used both by the health wait and by the retry that follows a lost race. */
function pause(ms: number): Promise<void> {
  const { promise, resolve } = Promise.withResolvers<void>();
  setTimeout(resolve, ms);
  return promise;
}

export type ComputerState = {
  /** Whose computer: a person's, or an isolated Bot's. */
  kind: ComputerKind;
  id: string;
  container: string;
  status: string;
  /** When this computer started, so a surface can say how long it has been up. */
  startedAt?: string;
  /** Its published port, when it has one. Absent on a shared network, where nothing is published. */
  port?: number;
  /** Where to reach it, however it is arranged. */
  url?: string;
  /**
   * What the image's healthcheck says, when Docker reports one.
   *
   * `running` only means the process exists. A container still `starting` here is booting its
   * browser, and `ensure` would hold the caller until it is healthy, so a status read says so.
   */
  health?: ContainerHealth;
};

/**
 * The name this Bot's computer would have is held by a container this supervisor does not own.
 *
 * Separate from {@link DockerUnavailableError} because the daemon is fine and an operator sent
 * looking at it would find nothing. What has to happen is that somebody looks at the container
 * holding the name and decides whether it should be there.
 */
export class NameHeldError extends Error {
  constructor(container: string) {
    super(
      `${container} already exists and does not belong to this computer. Remove it or rename it; it will not be adopted.`,
    );
    this.name = "NameHeldError";
  }
}

/**
 * The container started and the computer inside it never answered.
 *
 * Its own class because the two failures need different actions. Docker being unreachable is the
 * supervisor's problem; this is the computer's, and the message has to say so or an operator reads
 * "could not reach Docker" about a daemon that is answering.
 */
export class ComputerNotAnsweringError extends Error {
  constructor(container: string, timeoutMs: number) {
    super(
      `The computer in ${container} started but did not answer within ${timeoutMs}ms. It is not ready, so it is not being handed out.`,
    );
    this.name = "ComputerNotAnsweringError";
  }
}

export class DockerUnavailableError extends Error {
  constructor(cause: string) {
    super(
      `The supervisor could not reach Docker (${cause}). A computer cannot be started without it.`,
    );
    this.name = "DockerUnavailableError";
  }
}

export async function reachable(): Promise<boolean> {
  try {
    await docker.ping();
    return true;
  } catch {
    return false;
  }
}

function portOf(ports?: Docker.Port[] | undefined): number | undefined {
  const published = ports?.find((p) => p.PrivatePort === 4100)?.PublicPort;
  return published ?? undefined;
}

/**
 * A published port from `inspect`, which Docker reports as a string.
 *
 * The daemon hands back `""` before a port is assigned and anything at all when it
 * misbehaves, so a bare `parseInt` turns `"abc"` into a `NaN` port and `"0"` into a
 * port nothing can dial. Only digits in range are a port; anything else is no port.
 */
export function parseHostPort(value: unknown): number | undefined {
  if (typeof value !== "string" || !/^\d+$/.test(value)) return undefined;
  const port = Number.parseInt(value, 10);
  if (!Number.isSafeInteger(port) || port < 1 || port > 65535) return undefined;
  return port;
}

/**
 * Whether a labelled thing belongs to this deployment.
 *
 * Containers created before the namespace label existed carry the default namespace, so an existing
 * deployment keeps its own computers across an upgrade.
 */
function ours(labels: Record<string, string> | undefined): boolean {
  if (labels?.[OWNER_LABEL] !== "true") return false;
  return (labels[NAMESPACE_LABEL] ?? DEFAULT_NAMESPACE) === NAMESPACE;
}

/** The labels every container and volume this supervisor creates carries. */
function labelsFor(names: ComputerNames): Record<string, string> {
  return {
    [OWNER_LABEL]: "true",
    [NAMESPACE_LABEL]: NAMESPACE,
    [KIND_LABEL]: names.kind,
    [names.kind === "bot" ? BOT_LABEL : PERSON_LABEL]: names.id,
  };
}

/** Whose computer a labelled thing says it is, when it is one of today's. */
function whoseLabels(
  labels: Record<string, string> | undefined,
): { kind: ComputerKind; id: string } | undefined {
  const kind = labels?.[KIND_LABEL];
  if (!isComputerKind(kind)) return undefined;
  const id = labels?.[kind === "bot" ? BOT_LABEL : PERSON_LABEL];
  return id ? { kind, id } : undefined;
}

/**
 * Whether a labelled thing is this computer, not merely something of ours with the same name.
 *
 * NAMES ALONE DO NOT SAY. A computer from before one per person was `<ns>-computer-<bot>`, so a Bot
 * whose id began `bot-` or `person-` had a name today's scheme can also derive for somebody else:
 * the old computer of a Bot called `bot-1` is named exactly what an isolated Bot called `1` would be.
 * Adopting it would hand one Bot's logins to another. The kind and id labels are what tell them
 * apart, and an old computer has no kind label at all.
 */
function isThisComputer(
  labels: Record<string, string> | undefined,
  names: ComputerNames,
): boolean {
  if (!ours(labels)) return false;
  const whose = whoseLabels(labels);
  return whose?.kind === names.kind && whose.id === names.id;
}

/** Every computer this supervisor owns, and only those. */
export async function listOwned(): Promise<ComputerState[]> {
  try {
    const containers = (
      await docker.listContainers({
        all: true,
        filters: { label: [`${OWNER_LABEL}=true`] },
      })
    ).filter(
      // Today's computers only. One from before one per person is reached by nothing any more and
      // is stopped when the supervisor starts (see `retireLegacyComputers`).
      (container) => ours(container.Labels) && whoseLabels(container.Labels),
    );
    return containers.map((container) => ({
      ...(whoseLabels(container.Labels) as { kind: ComputerKind; id: string }),
      container: (container.Names?.[0] ?? "").replace(/^\//, ""),
      status: container.State,
      ...(healthOf(container.Status)
        ? { health: healthOf(container.Status) }
        : {}),
      ...(container.Created
        ? { startedAt: new Date(container.Created * 1000).toISOString() }
        : {}),
      ...(portOf(container.Ports) ? { port: portOf(container.Ports) } : {}),
    }));
  } catch (error) {
    throw new DockerUnavailableError(String(error));
  }
}

/**
 * One computer, if this supervisor owns it.
 *
 * Ownership is checked here rather than at the call sites, so no verb can skip it by accident.
 */
async function inspectOwned(names: ComputerNames): Promise<{
  status: string;
  port?: number;
  image?: string;
  startedAt?: string;
  token?: string;
} | null> {
  try {
    const info = await docker.getContainer(names.container).inspect();
    if (!isThisComputer(info.Config?.Labels, names)) return null;
    const published =
      info.NetworkSettings?.Ports?.[COMPUTER_PORT]?.[0]?.HostPort;
    const port = parseHostPort(published);
    return {
      status: info.State?.Status ?? "unknown",
      ...(port !== undefined ? { port } : {}),
      // The resolved image, not the tag it was started from. A tag moves when the image is
      // rebuilt; this is what the container is actually running.
      ...(info.Image ? { image: info.Image } : {}),
      // The token this container was born holding, which is the one it will check callers against
      // for the rest of its life. See `holdsCurrentToken`.
      token: tokenIn(info.Config?.Env),
      /*
       * When this run of the container began, which is what tells two runs apart.
       *
       * `State.StartedAt` rather than `Created`, because a restart is a new run: the browser has
       * lost every page it had, so the refs the server handed out from the previous one describe
       * pages that no longer exist. `listOwned` reports `Created` for "how long has this been up",
       * which is a different question.
       */
      ...(info.State?.StartedAt ? { startedAt: info.State.StartedAt } : {}),
    };
  } catch (error) {
    if ((error as { statusCode?: number }).statusCode === 404) return null;
    throw new DockerUnavailableError(String(error));
  }
}

/**
 * Whether the computer that exists is running the image this deployment now ships.
 *
 * `ensure` reused any container with the right name, whatever it was built from, so once a Bot had a
 * computer, upgrading OpenBot never reached it. Rebuilding the image moves the tag; the container
 * goes on running the old one, indefinitely, and nothing says so. Found by rebuilding every image,
 * restarting the whole stack, and watching a Bot's computer answer with in-memory state from an hour
 * earlier: `docker compose down` does not touch these, because the supervisor makes them rather than
 * compose.
 *
 * That is worse than stale code. `agent-computer` is the browser, the workspace and the confinement,
 * so a fix to any of them silently would not apply to a Bot that already had a computer.
 *
 * Compared by resolved id rather than by tag, because both sides are the same tag and the whole
 * question is whether the tag has moved since.
 *
 * Unanswerable is not stale. If the image cannot be inspected — never pulled, a registry that cannot
 * be reached, a daemon that will not say — this reports true and the existing computer is kept.
 * Destroying a Bot's working browser over a failed inspect is a worse answer than running an image
 * that may be a version behind.
 */
async function runsCurrentImage(
  existingImage: string | undefined,
  image: string,
): Promise<boolean> {
  if (!existingImage) return true;
  try {
    const current = await docker.getImage(image).inspect();
    const id = current?.Id;
    return typeof id === "string" && id ? id === existingImage : true;
  } catch {
    return true;
  }
}

/** `COMPUTER_TOKEN=...` out of a list of `KEY=value`, which is how both sides carry an environment. */
function tokenIn(environment: string[] | undefined): string | undefined {
  const entry = environment?.find((line) => line.startsWith(`${TOKEN_NAME}=`));
  return entry?.slice(TOKEN_NAME.length + 1);
}

const TOKEN_NAME = "COMPUTER_TOKEN";

/**
 * Whether the computer that exists will accept the token this deployment now hands out.
 *
 * A computer is checked against the `COMPUTER_TOKEN` it was created with, and it holds that one for
 * as long as the container lives. Normally that is nothing to worry about, because the shell mints
 * the generated secrets once per deployment and deliberately does not rotate them: a computer
 * outliving a restart is the reason it does not.
 *
 * The token does change, though, on exactly the occasion nobody tests: a machine set up again from
 * nothing. Emptying the credential store, or installing over a deployment whose secrets are gone,
 * mints a new one. Compose then rebuilds everything it owns with it, the supervisor included, and
 * the computers are the one thing compose does not own. They survive, holding the old token, and
 * every call to them comes back 401.
 *
 * What that looks like to a person is the reason this is a defect rather than an inconvenience: the
 * gateway allows the action and the trail records it as carried out, the computer refuses it, and
 * the screen says "Not authorised" while naming nothing. Found on a first run of v0.0.9 against a
 * computer container created by the install before it, five days earlier.
 *
 * A deployment that sets no token is not a mismatch. That is a computer with no door on it, which is
 * a choice the environment makes, and replacing a working browser over it would be this function
 * inventing a policy of its own.
 */
function holdsCurrentToken(
  existingToken: string | undefined,
  environment: string[],
): boolean {
  const wanted = tokenIn(environment);
  if (wanted === undefined) return true;
  return existingToken === wanted;
}

/** Long enough for a cold start with a large image, short enough that a caller is not left hanging. */
const DEFAULT_READY_TIMEOUT_MS = 60_000;

/**
 * Wait until the computer actually answers.
 *
 * "Started" and "ready" are not the same thing, and the gap is seconds: Docker reports running as
 * soon as the process exists, while the computer inside is still starting. A caller told a computer
 * is ready and then refused by it cannot tell that from a broken one, so `ensure` waits.
 *
 * Asked of Docker, not over the network. Reaching the computer directly needs an address, and which
 * address works depends on how the deployment is wired: `127.0.0.1:<published port>` is right only
 * when the supervisor runs on the host, and the container name resolves only when both containers
 * share a network.
 *
 * The image carries a HEALTHCHECK, so Docker already knows the answer and can be asked from anywhere
 * the supervisor can reach the socket, which it must, or it could not have created the container.
 */
async function waitUntilAnswering(
  container: string,
  timeoutMs: number = DEFAULT_READY_TIMEOUT_MS,
): Promise<void> {
  const deadline = Date.now() + timeoutMs;
  while (Date.now() < deadline) {
    try {
      const info = await docker.getContainer(container).inspect();
      const health = info.State?.Health?.Status;
      // Nothing to read. Every computer this supervisor creates is given a healthcheck, so this is
      // an engine that does not report one rather than an image that does not carry one, and
      // waiting forever for an answer that will never come would be worse than going ahead.
      if (!health) {
        if (info.State?.Running) return;
      } else if (health === "healthy") {
        return;
      }
    } catch {
      // Mid-creation, or gone. The deadline is what ends this.
    }
    await pause(250);
  }

  /*
   * The deadline is a failure, not an answer.
   *
   * Returning here reported every computer that never came up as ready, which is the exact thing
   * this function exists to prevent: the caller is handed an address, sends the deployment's
   * computer token to it, and gets a transport error it cannot tell from a computer that is broken
   * in some other way. A wait that cannot fail is a sleep.
   */
  throw new ComputerNotAnsweringError(container, timeoutMs);
}

export type EnsureOptions = {
  image: string;
  /** Passed to the computer so a Bot's traffic still leaves by the route configured for it. */
  environment: string[];
  /** A network to join, when the supervisor runs alongside a compose stack. */
  network?: string;
  /**
   * The container runtime. Left unset this is Docker's default, which shares the host kernel.
   * Setting `runsc` runs each computer under gVisor, which intercepts syscalls in user space and is
   * the pragmatic middle ground for untrusted Bot code. A deployment that wants a kernel per Bot
   * points this at a microVM runtime instead.
   */
  runtime?: string;
  memoryBytes?: number;
  /**
   * How long a started computer is given to answer before the attempt is called a failure.
   *
   * Configurable because the wait now fails rather than returning, so the number decides when a slow
   * start becomes an error, and a deployment pulling a large image on a cold host is not the same as
   * a test that wants an answer in seconds.
   */
  readyTimeoutMs?: number;
  pidsLimit?: number;
  /**
   * The volume holding the SPIRE agent's Workload API socket, mounted read-only into each computer
   * so it can ask what it is. Unset means no identity, which is a deployment choice rather than a
   * failure.
   */
  spireSocketVolume?: string;
};

/**
 * Confinement applied to every computer.
 *
 * A container is a boundary only if it is configured as one. Each of these closes a specific route
 * off the host, and none of them costs a Bot anything it legitimately needs: a browser and a
 * filesystem under `/workspace`.
 */
function hostConfig(names: ComputerNames, options: EnsureOptions) {
  return {
    // The Bot's own storage. This is what turns the path confinement inside the computer from a
    // boundary between a Bot and the host into a boundary between one Bot and another.
    Binds: [
      `${names.profileVolume}:/profiles`,
      `${names.workspaceVolume}:/workspace`,
      // Read-only: a computer asks the agent what it is and has nothing to tell it.
      ...(options.spireSocketVolume
        ? [`${options.spireSocketVolume}:/tmp/spire-agent/public:ro`]
        : []),
    ],
    // On a shared network the computers are reached by name and nothing is published: a remote host
    // should not accumulate an open port per Bot. Off a network, a laptop, where the server runs
    // outside Docker, an ephemeral host port is the only way in, and it is reported back so nothing
    // has to guess and two computers never contend for the same number.
    ...(options.network
      ? {}
      : {
          // Loopback, not the world. An unqualified binding publishes on 0.0.0.0, which would put
          // every Bot's computer within reach of anything that can route to this machine. The token
          // the computer requires is the control; this keeps the surface off the network as well,
          // because both is the right number of locks on a browser holding somebody's logins.
          PortBindings: {
            [COMPUTER_PORT]: [{ HostIp: "127.0.0.1", HostPort: "" }],
          },
        }),
    RestartPolicy: { Name: "unless-stopped" },
    ...(options.network ? { NetworkMode: options.network } : {}),
    ...(options.runtime ? { Runtime: options.runtime } : {}),

    // No path from inside to more privilege than it started with, whatever it manages to run.
    SecurityOpt: ["no-new-privileges:true"],
    // Chromium needs none of these, and each is a documented container escape route.
    CapDrop: ["ALL"],
    // A runaway Bot is a resource problem for itself, not for every other Bot on the host.
    ...(options.memoryBytes ? { Memory: options.memoryBytes } : {}),
    PidsLimit: options.pidsLimit ?? 512,
    // Chromium's sandbox wants shared memory and will crash on the 64MB default.
    ShmSize: 1_073_741_824,
  };
}

/**
 * Make sure this Bot has a computer, and say where to reach it.
 *
 * Idempotent because the caller is a request handler that can run concurrently with itself: two
 * messages to one Bot at the same moment must not race into two containers. An existing owned
 * container is started if stopped, and otherwise left exactly as it is.
 */
export async function ensure(
  names: ComputerNames,
  options: EnsureOptions,
): Promise<ComputerState> {
  for (let attempt = ATTEMPTS; attempt > 0; attempt--) {
    let existing = await inspectOwned(names);

    /*
     * An upgrade reaches a computer that already exists, by replacing it.
     *
     * Safe to do: the profile and the workspace are named volumes and are not removed here, so the
     * Bot keeps its logins and its files and comes back on the new image. That is the difference
     * between this and `reset`, which is asked for deliberately and does take the profile.
     *
     * What is lost is whatever the old computer held in memory: an open page and an outstanding
     * request for a person to take the wheel. Both belong to a run that the upgrade has already
     * ended, and a Bot carrying an hour-old handover prompt into a new conversation is the symptom
     * that found this.
     */
    if (
      existing &&
      (!(await runsCurrentImage(existing.image, options.image)) ||
        !holdsCurrentToken(existing.token, options.environment))
    ) {
      try {
        await docker
          .getContainer(names.container)
          .remove({ force: true, v: false });
      } catch (error) {
        // Already gone is the outcome this wanted. Anything else and the computer stays as it is,
        // which is the same answer this function gave before it could replace one at all.
        if (statusOf(error) !== 404) {
          throw new DockerUnavailableError(String(error));
        }
      }
      existing = null;
    }

    if (!existing) {
      for (const volume of [names.profileVolume, names.workspaceVolume]) {
        try {
          await docker.createVolume({
            Name: volume,
            Labels: labelsFor(names),
          });
        } catch (error) {
          // Already exists is success for a restarted supervisor, when it is this computer's.
          if (statusOf(error) !== 409) {
            throw new DockerUnavailableError(String(error));
          }
          /*
           * But only then. A volume of the same name made for somebody else (an old computer whose
           * Bot id spells this name, see `isThisComputer`) would be mounted here, and its logins
           * would become this computer's. Refused, like a container that is not ours.
           */
          const held = await docker
            .getVolume(volume)
            .inspect()
            .catch(() => null);
          if (!held || !isThisComputer(held.Labels ?? undefined, names)) {
            throw new NameHeldError(volume);
          }
        }
      }

      try {
        await docker.createContainer({
          name: names.container,
          Image: options.image,
          // The Bot id is a label because that is what a SPIRE docker workload attestor selects on:
          // an identity per Bot then falls out of the same fact that names the container.
          Labels: labelsFor(names),
          Env: options.environment,
          ExposedPorts: { [COMPUTER_PORT]: {} },
          Healthcheck: COMPUTER_HEALTHCHECK,
          HostConfig: hostConfig(names, options),
        });
      } catch (error) {
        if (statusOf(error) !== 409) {
          throw new DockerUnavailableError(String(error));
        }
        /*
         * Something already holds the name, and 409 does not say what.
         *
         * Usually it is the other request creating the same computer, which is what idempotent means
         * here, and its container is the one this request goes on to start. The other case is a
         * container this supervisor does not own: left by a deployment that used a different
         * namespace, made by hand, or put there by somebody who guessed the name. Ownership is
         * checked everywhere else precisely so that one is treated as absent, and starting it here
         * on a 409 was the one path that adopted it instead: `start` names the container, not the
         * container this supervisor made, and the address goes back to a server that then sends the
         * deployment's computer token to whatever is listening inside it.
         */
        if (!(await inspectOwned(names))) {
          throw new NameHeldError(names.container);
        }
      }
    }

    if (existing?.status !== "running") {
      try {
        await docker.getContainer(names.container).start();
      } catch (error) {
        const status = statusOf(error);
        /*
         * Gone between creating it and starting it: a concurrent reset took the container away, the
         * create this request lost the race to was itself rolled back, or the daemon has not yet
         * published the name this request just created. Nothing about the Bot has changed, so the
         * answer is to build it again rather than to report Docker as unreachable.
         *
         * Paused first, because the retry is the whole budget. Going straight back round arrives
         * within a millisecond, sees the same not-yet-published name, and spends the second attempt
         * on the state that failed the first: the first browser action a Bot is ever asked for fails,
         * and the second one, seconds later, works. One poll interval is what the health wait uses
         * for the same question.
         */
        if (status === 404 && attempt > 1) {
          await pause(250);
          continue;
        }
        // 304 is "already running", which is success for an idempotent verb.
        if (status !== 304) {
          throw new DockerUnavailableError(String(error));
        }
      }
    }

    const settled = await inspectOwned(names);
    await waitUntilAnswering(names.container, options.readyTimeoutMs);

    return {
      kind: names.kind,
      id: names.id,
      container: names.container,
      status: settled?.status ?? "unknown",
      ...(settled?.startedAt ? { startedAt: settled.startedAt } : {}),
      ...(settled?.port ? { port: settled.port } : {}),
      // How to reach it. A name on a shared network, a host port otherwise, the caller does not have
      // to know which arrangement it is in.
      ...(options.network
        ? { url: `http://${names.container}:4100` }
        : settled?.port
          ? { url: `http://127.0.0.1:${settled.port}` }
          : {}),
    };
  }

  throw new DockerUnavailableError(
    `The computer for ${names.kind} ${names.id} was removed while it was being started.`,
  );
}

/** Stop this computer. Its storage is untouched, so its logins survive. */
export async function stop(names: ComputerNames): Promise<boolean> {
  if (!(await inspectOwned(names))) return false;
  try {
    // Long enough for Chromium to flush its profile, matching the compose grace period.
    await docker.getContainer(names.container).stop({ t: 30 });
  } catch (error) {
    const status = (error as { statusCode?: number }).statusCode;
    if (status !== 304 && status !== 404) {
      throw new DockerUnavailableError(String(error));
    }
  }
  return true;
}

/**
 * Throw this computer away so the next request builds a clean one.
 *
 * The profile goes with it, which on a person's computer is every Bot's logins there. The workspace
 * is left alone: files a Bot was asked to produce are work, not browser state.
 */
export async function reset(names: ComputerNames): Promise<boolean> {
  if (!(await inspectOwned(names))) return false;

  try {
    await docker
      .getContainer(names.container)
      .remove({ force: true, v: false });
  } catch (error) {
    if ((error as { statusCode?: number }).statusCode !== 404) {
      throw new DockerUnavailableError(String(error));
    }
  }

  try {
    await docker.getVolume(names.profileVolume).remove();
  } catch (error) {
    const status = (error as { statusCode?: number }).statusCode;
    // 409 is "still in use", which resolves itself once the container is gone.
    if (status !== 404 && status !== 409) {
      throw new DockerUnavailableError(String(error));
    }
  }
  return true;
}

/**
 * Stop every computer from before computers were one per person.
 *
 * Each Bot had its own container then, named for the Bot alone. Nothing reaches one any more: a
 * Bot runs on its person's computer, or on a computer of its own under today's names. Left running,
 * every one of them would go on holding a browser and its memory for ever, restarted with the
 * daemon by its restart policy, which is the opposite of what moving to one computer per person is
 * for. Stopped, not removed, and their volumes untouched: a private Bot's old profile is still read
 * from its volume to give it to its owner (see {@link carryOverProfile}), and nothing here is
 * destroyed that somebody might want back.
 */
export async function retireLegacyComputers(): Promise<string[]> {
  let running: Docker.ContainerInfo[];
  try {
    running = await docker.listContainers({
      filters: { label: [`${OWNER_LABEL}=true`], status: ["running"] },
    });
  } catch (error) {
    throw new DockerUnavailableError(String(error));
  }
  const stopped: string[] = [];
  for (const container of running) {
    if (!ours(container.Labels) || whoseLabels(container.Labels)) continue;
    try {
      await docker.getContainer(container.Id).stop({ t: 30 });
      stopped.push((container.Names?.[0] ?? container.Id).replace(/^\//, ""));
    } catch (error) {
      const status = statusOf(error);
      if (status !== 304 && status !== 404) {
        throw new DockerUnavailableError(String(error));
      }
    }
  }
  return stopped;
}

/**
 * What giving a private Bot's old profile to its owner did.
 *
 * `copied` it now; `kept` the owner's computer already had a profile for that Bot, which wins;
 * `done` an earlier run did it; `absent` there was no old profile of ours to give.
 */
export type CarryOverOutcome = "copied" | "kept" | "done" | "absent";

/**
 * The copy, run inside a throwaway container that mounts both volumes.
 *
 * That is Docker's own documented way to back up, restore or migrate a volume: a container with the
 * volume mounted, doing the copy itself, because a volume lives outside every image and `docker cp`
 * cannot reach it. Both volumes are mounted rather than tarred through a host directory, so nothing
 * of a person's logins or files ever lands on the host's disk.
 *
 * Once, recorded in the destination: a marker under `.openbot-carried-over/` says it was done, and
 * what the owner already has for that Bot is never overwritten. Staged under a dot-name and moved
 * into place, so the computer never sees half of it; a dot-name is not a seat it lists.
 *
 * `ROOT` is where the owner's volume is mounted (`/profiles`, `/workspace`) and `DEST` the Bot's
 * place in it: its profile is `<bot>`, its folder `computers/<bot>` (agent-computer's
 * `workspaceDirectoryFor`).
 *
 * The exit code is the outcome, so nothing depends on parsing a log stream.
 */
const CARRY_OVER_SCRIPT = `set -eu
marks="$ROOT/.openbot-carried-over"
dest="$ROOT/$DEST"
if [ -e "$marks/$BOT" ]; then exit 11; fi
mkdir -p "$marks"
if [ -e "$dest" ]; then touch "$marks/$BOT"; exit 10; fi
staging="$ROOT/.openbot-carrying-$BOT"
rm -rf "$staging"
mkdir "$staging"
cp -a /carry-from/. "$staging"/
mkdir -p "$(dirname "$dest")"
mv "$staging" "$dest"
touch "$marks/$BOT"
exit 0`;

/** Carry-overs started in this process, so two requests for one never copy at once. */
const carrying = new Map<string, Promise<CarryOverOutcome>>();

/**
 * Forget what this process remembers about carry-overs, as a restarted supervisor would.
 *
 * For a test of the marker the copy leaves in the owner's volume, which is the record that outlives
 * this process and the one that keeps the copy to once.
 */
export function forgetCarryOvers(): void {
  carrying.clear();
}

/** One old volume copied into the owner's, by {@link CARRY_OVER_SCRIPT}, in a container that can do nothing else. */
async function copyVolume(input: {
  botId: string;
  from: string;
  to: string;
  root: "/profiles" | "/workspace";
  dest: string;
  image: string;
  runtime?: string;
}): Promise<Exclude<CarryOverOutcome, "absent">> {
  let helper: Docker.Container;
  try {
    helper = await docker.createContainer({
      Image: input.image,
      Labels: {
        [OWNER_LABEL]: "true",
        [NAMESPACE_LABEL]: NAMESPACE,
        "openbot.carry-over": input.botId,
      },
      Entrypoint: ["sh", "-c"],
      Cmd: [CARRY_OVER_SCRIPT],
      // A plain id checked by the caller, and fixed paths, handed over as data, never spliced into the script.
      Env: [`BOT=${input.botId}`, `ROOT=${input.root}`, `DEST=${input.dest}`],
      NetworkDisabled: true,
      HostConfig: {
        Binds: [`${input.from}:/carry-from:ro`, `${input.to}:${input.root}`],
        NetworkMode: "none",
        SecurityOpt: ["no-new-privileges:true"],
        // Only what copying files with their owners and modes takes, and nothing else.
        CapDrop: ["ALL"],
        CapAdd: ["CHOWN", "DAC_OVERRIDE", "FOWNER"],
        ...(input.runtime ? { Runtime: input.runtime } : {}),
      },
    });
  } catch (error) {
    throw new DockerUnavailableError(String(error));
  }
  try {
    await helper.start();
    const { StatusCode } = (await helper.wait()) as { StatusCode: number };
    const outcome =
      StatusCode === 0
        ? "copied"
        : StatusCode === 10
          ? "kept"
          : StatusCode === 11
            ? "done"
            : undefined;
    if (!outcome) {
      throw new Error(
        `Copying ${input.from} into ${input.to} failed (exit ${StatusCode}).`,
      );
    }
    return outcome;
  } finally {
    await helper.remove({ force: true }).catch(() => undefined);
  }
}

/**
 * Give a private Bot's profile, and its files, from before one computer per person to its owner's computer.
 *
 * The server asks for it only for a private Bot, on its owner's computer: that profile and those files had exactly
 * one user. A public Bot's had no single owner, so its users sign in again once and nothing is asked.
 *
 * The old volumes are copied, not moved, and left exactly where they were; the old container is stopped
 * first, if it is still running, so Chromium has flushed what it holds and nothing writes to the
 * profile while it is read. The new computer is already running when this is asked, which is safe
 * because the Bot's folder in it does not exist until the copy lands: the server waits for this
 * before it sends that Bot anything.
 *
 * Remembered for the life of the process once it succeeds, so the common case costs nothing; a
 * failure is not remembered, so the next request tries again.
 */
export async function carryOverProfile(
  person: ComputerNames,
  botId: string,
  options: { image: string; runtime?: string },
): Promise<CarryOverOutcome> {
  if (person.kind !== "person") return "absent";
  const legacy = legacyNamesFor(botId);
  if (!legacy.ok) throw new Error(legacy.reason);
  const key = `${person.container}/${botId}`;
  const known = carrying.get(key);
  if (known) return known;

  const attempt = (async (): Promise<CarryOverOutcome> => {
    const isLegacyOf = (labels: Record<string, string> | undefined) =>
      ours(labels) && !whoseLabels(labels) && labels?.[BOT_LABEL] === botId;

    const volume = await docker
      .getVolume(legacy.names.profileVolume)
      .inspect()
      .catch((error: unknown) => {
        if (statusOf(error) === 404) return null;
        throw new DockerUnavailableError(String(error));
      });
    // Not there, or not ours to read: nothing is copied out of a volume this supervisor did not make.
    if (!volume || !isLegacyOf(volume.Labels ?? undefined)) return "absent";

    const old = await docker
      .getContainer(legacy.names.container)
      .inspect()
      .catch(() => null);
    if (old && isLegacyOf(old.Config?.Labels) && old.State?.Running) {
      await docker
        .getContainer(legacy.names.container)
        .stop({ t: 30 })
        .catch((error: unknown) => {
          const status = statusOf(error);
          if (status !== 304 && status !== 404) {
            throw new DockerUnavailableError(String(error));
          }
        });
    }

    const profile = await copyVolume({
      botId,
      from: legacy.names.profileVolume,
      to: person.profileVolume,
      root: "/profiles",
      dest: botId,
      ...options,
    });
    /*
     * Its files too, into its folder on the owner's computer, the same way and once. An old Bot with no workspace
     * volume of ours had nothing saved there, which is not a failure.
     */
    const files = await docker
      .getVolume(legacy.names.workspaceVolume)
      .inspect()
      .catch((error: unknown) => {
        if (statusOf(error) === 404) return null;
        throw new DockerUnavailableError(String(error));
      });
    const filesOutcome =
      files && isLegacyOf(files.Labels ?? undefined)
        ? await copyVolume({
            botId,
            from: legacy.names.workspaceVolume,
            to: person.workspaceVolume,
            root: "/workspace",
            dest: `computers/${botId}`,
            ...options,
          })
        : "absent";
    console.info(
      JSON.stringify({
        type: "computer-profile-carried-over",
        bot: botId,
        person: person.id,
        from: legacy.names.profileVolume,
        to: `${person.profileVolume}:/profiles/${botId}`,
        outcome: profile,
        files: {
          from: legacy.names.workspaceVolume,
          to: `${person.workspaceVolume}:/workspace/computers/${botId}`,
          outcome: filesOutcome,
        },
        note: "The old volumes are left in place.",
      }),
    );
    return profile;
  })();
  carrying.set(key, attempt);
  attempt.catch(() => {
    if (carrying.get(key) === attempt) carrying.delete(key);
  });
  return attempt;
}