Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

19.1 KB
Raw Blame
import { describe } from "vite-plus/test";
import { Workspace } from "@mastra/core/workspace";
import { it as effectIt } from "@effect/vitest";
import * as Clock from "effect/Clock";
import * as Effect from "effect/Effect";
import * as TestClock from "effect/testing/TestClock";
import { expect, it, vi } from "vite-plus/test";
import type { AkeruBotWorkspace } from "./botWorkspace.ts";
import { BotWorkspacePool } from "./botWorkspacePool.ts";
import { makebotWorkspacePoolTestSupport } from "./test-support/botWorkspacePool.ts";

const { localWorkspace, remoteWorkspace } = makebotWorkspacePoolTestSupport();

describe("BotWorkspacePool", () => {
  it("retries failed creation", async () => {
    const pool = new BotWorkspacePool();

    const create = vi
      .fn<() => Promise<Workspace>>()
      .mockRejectedValueOnce(new Error("unavailable"))
      .mockResolvedValueOnce(localWorkspace());

    await expect(pool.acquire("retry", create)).rejects.toThrow("unavailable");
    const lease = await pool.acquire("retry", create);
    await lease.release({ destroy: true });
    expect(create).toHaveBeenCalledTimes(2);
  });

  it("preserves a remote workspace after a failed initial wake and retries it", async () => {
    const pool = new BotWorkspacePool();
    const local = localWorkspace();

    const remote = {
      id: local.id,
      provider: "ascii" as const,
      workspace: local,
      inspect: async () => "running" as const,
      wake: () => local.init(),
      sleep: () => local.stop(),
      destroy: () => local.destroy(),
    };

    vi.spyOn(local, "init").mockRejectedValueOnce(new Error("wake failed"));
    const destroy = vi.spyOn(local, "destroy");
    const create = vi.fn(async () => remote);

    await expect(pool.acquire("remote-wake", create)).rejects.toThrow("wake failed");
    expect(destroy).not.toHaveBeenCalled();
    const lease = await pool.acquire("remote-wake", create);
    expect(create).toHaveBeenCalledTimes(2);
    await lease.release({ destroy: true });
    expect(destroy).toHaveBeenCalledOnce();
  });

  it("preserves and reattaches a remote workspace after idle sleep fails", async () => {
    const pool = new BotWorkspacePool();
    const local = localWorkspace();

    const remote = {
      id: local.id,
      provider: "ascii" as const,
      workspace: local,
      inspect: async () => "running" as const,
      wake: () => local.init(),
      sleep: () => local.stop(),
      destroy: () => local.destroy(),
    };

    vi.spyOn(local, "stop").mockRejectedValueOnce(new Error("sleep failed"));
    const destroy = vi.spyOn(local, "destroy");
    const create = vi.fn(async () => remote);
    const lease = await pool.acquire("remote-sleep", create);

    await expect(lease.release()).rejects.toThrow("sleep failed");
    expect(destroy).not.toHaveBeenCalled();
    const retry = await pool.acquire("remote-sleep", create);
    expect(create).toHaveBeenCalledTimes(2);
    await retry.release({ destroy: true });
    expect(destroy).toHaveBeenCalledOnce();
  });

  it("preserves a remote workspace when waking after idle fails", async () => {
    const pool = new BotWorkspacePool();
    const local = localWorkspace();
    const wake = vi.fn(async () => local.init());
    const destroy = vi.fn(async () => local.destroy());

    const create = vi.fn(async () => ({
      id: local.id,
      provider: "ascii" as const,
      workspace: local,
      inspect: async () => "running" as const,
      wake,
      sleep: () => local.stop(),
      destroy,
    }));

    const lease = await pool.acquire("remote-idle-wake", create);
    await lease.release();
    wake.mockRejectedValueOnce(new Error("resume timed out"));
    await expect(pool.acquire("remote-idle-wake", create)).rejects.toThrow("resume timed out");
    expect(destroy).not.toHaveBeenCalled();
    const retry = await pool.acquire("remote-idle-wake", create);
    expect(create).toHaveBeenCalledTimes(2);
    await retry.release({ destroy: true });
    expect(destroy).toHaveBeenCalledOnce();
  });

  it("pauses and reuses a remote workspace after initial wake failure", async () => {
    const pool = new BotWorkspacePool();

    const failed = remoteWorkspace({
      wake: vi.fn().mockRejectedValueOnce(new Error("wake failed")).mockResolvedValue(undefined),
    });

    const reattached = remoteWorkspace();
    const create = vi.fn().mockResolvedValueOnce(failed).mockResolvedValueOnce(reattached);

    await expect(pool.acquire("remote-initial", create)).rejects.toThrow("wake failed");
    expect(failed.destroy).not.toHaveBeenCalled();
    await pool.retryFailedSleeps();
    expect(failed.sleep).toHaveBeenCalledOnce();
    const lease = await pool.acquire("remote-initial", create);
    expect(create).toHaveBeenCalledOnce();
    expect(lease.workspace).toBe(failed);
    await lease.release({ destroy: true });
    expect(failed.destroy).toHaveBeenCalledOnce();
  });

  it("preserves a remote workspace after cached wake failure and retries cleanup", async () => {
    const pool = new BotWorkspacePool();

    const failed = remoteWorkspace({
      wake: vi
        .fn()
        .mockResolvedValueOnce(undefined)
        .mockRejectedValueOnce(new Error("wake failed"))
        .mockResolvedValue(undefined),
    });

    const reattached = remoteWorkspace();
    const create = vi.fn().mockResolvedValueOnce(failed).mockResolvedValueOnce(reattached);

    const initial = await pool.acquire("remote-cached", create);
    await initial.release();
    await expect(pool.acquire("remote-cached", create)).rejects.toThrow("wake failed");
    expect(failed.destroy).not.toHaveBeenCalled();
    await pool.retryFailedSleeps();
    expect(failed.sleep).toHaveBeenCalledTimes(2);
    const lease = await pool.acquire("remote-cached", create);
    expect(lease.workspace).toBe(failed);
    await lease.release({ destroy: true });
  });

  it("preserves and reuses a remote workspace after sleep failure", async () => {
    const pool = new BotWorkspacePool();

    const failed = remoteWorkspace({
      sleep: vi.fn().mockRejectedValueOnce(new Error("sleep failed")),
    });

    const reattached = remoteWorkspace();
    const create = vi.fn().mockResolvedValueOnce(failed).mockResolvedValueOnce(reattached);

    const initial = await pool.acquire("remote-sleep", create);
    await expect(initial.release()).rejects.toThrow("sleep failed");
    expect(failed.destroy).not.toHaveBeenCalled();
    const lease = await pool.acquire("remote-sleep", create);
    expect(lease.workspace).toBe(failed);
    expect(create).toHaveBeenCalledOnce();
    await lease.release({ destroy: true });
    expect(failed.destroy).toHaveBeenCalledOnce();
  });

  it("retries failed remote pauses without another acquisition and retains repeated failures", async () => {
    const pool = new BotWorkspacePool();

    const sleep = vi
      .fn()
      .mockRejectedValueOnce(new Error("pause failed"))
      .mockRejectedValueOnce(new Error("pause still unavailable"))
      .mockResolvedValue(undefined);

    const workspace = remoteWorkspace({ sleep });
    const lease = await pool.acquire("idle-retry", async () => workspace);
    await expect(lease.release()).rejects.toThrow("pause failed");
    await expect(pool.retryFailedSleeps()).rejects.toThrow("pause still unavailable");
    await pool.retryFailedSleeps();
    expect(sleep).toHaveBeenCalledTimes(3);
    expect(workspace.destroy).not.toHaveBeenCalled();
    await pool.retryFailedSleeps();
    expect(sleep).toHaveBeenCalledTimes(3);
    await pool.destroyAll();
    expect(workspace.destroy).toHaveBeenCalledOnce();
  });

  it("does not retry pause while a recovered workspace is leased", async () => {
    const pool = new BotWorkspacePool();

    const sleep = vi
      .fn()
      .mockRejectedValueOnce(new Error("pause failed"))
      .mockResolvedValue(undefined);

    const workspace = remoteWorkspace({ sleep });
    const create = vi.fn(async () => workspace);
    const lease = await pool.acquire("active-retry", create);
    await expect(lease.release()).rejects.toThrow("pause failed");
    const active = await pool.acquire("active-retry", create);
    await pool.retryFailedSleeps();
    expect(sleep).toHaveBeenCalledTimes(1);
    await active.release();
    expect(sleep).toHaveBeenCalledTimes(2);
  });

  it("waits for an in-flight idle retry before waking a new lease", async () => {
    const pool = new BotWorkspacePool();
    let finishPause!: () => void;
    let markStarted!: () => void;

    const started = new Promise<void>((resolve) => {
      markStarted = resolve;
    });

    const sleep = vi
      .fn()
      .mockRejectedValueOnce(new Error("pause failed"))
      .mockImplementationOnce(
        () =>
          new Promise<void>((resolve) => {
            finishPause = resolve;
            markStarted();
          }),
      );

    const workspace = remoteWorkspace({ sleep });
    const create = vi.fn(async () => workspace);
    const lease = await pool.acquire("concurrent-retry", create);
    await expect(lease.release()).rejects.toThrow("pause failed");
    const retry = pool.retryFailedSleeps();
    await started;
    const acquire = pool.acquire("concurrent-retry", create);
    expect(workspace.wake).toHaveBeenCalledOnce();
    finishPause();
    await retry;
    const active = await acquire;
    expect(workspace.wake).toHaveBeenCalledTimes(2);
    await active.release({ destroy: true });
  });

  effectIt.effect("waits for a destroyed workspace before opening its replacement", () =>
    Effect.gen(function* () {
      const clock = yield* Clock.Clock;
      const pool = new BotWorkspacePool({ clock });
      const events: Array<string> = [];
      const finishDestroy = Promise.withResolvers<void>();
      const destroyStarted = Promise.withResolvers<void>();

      const remote = (id: string): AkeruBotWorkspace => ({
        id,
        provider: "e2b",
        workspace: localWorkspace(),
        inspect: async () => "running",
        wake: async () => {},
        sleep: async () => {},
        destroy: async () => {
          events.push(`destroy ${id} started`);
          destroyStarted.resolve();
          await finishDestroy.promise;
          events.push(`destroy ${id} finished`);
        },
      });

      const first = yield* Effect.promise(() => pool.acquire("remote", async () => remote("old")));
      const releasing = first.release({ destroy: true });

      const replacement = pool.acquire("remote", async () => {
        events.push("create new");

        return remote("new");
      });

      // Let the replacement acquire run as far as it can while the destroy is still pending.
      yield* Effect.promise(() => destroyStarted.promise);
      yield* Effect.promise(() => new Promise<void>((resolve) => setImmediate(resolve)));
      finishDestroy.resolve();
      yield* Effect.promise(() => releasing);
      const second = yield* Effect.promise(() => replacement);
      expect(second.workspace.id).toBe("new");
      expect(events).toEqual(["destroy old started", "destroy old finished", "create new"]);
      yield* Effect.promise(() => second.release());
    }),
  );

  effectIt.effect("reuses and eventually destroys the same evicted remote wrapper", () =>
    Effect.gen(function* () {
      const clock = yield* Clock.Clock;
      const pool = new BotWorkspacePool({ clock });
      const workspace = localWorkspace();

      const remote: AkeruBotWorkspace = {
        id: "remote-1",
        provider: "e2b",
        workspace,
        inspect: async () => "running",
        wake: vi.fn(async () => {}),
        sleep: vi.fn(async () => {}),
        destroy: vi.fn(async () => {}),
      };

      const create = vi.fn(async () => remote);
      const first = yield* Effect.promise(() => pool.acquire("remote", create));
      yield* Effect.promise(() => first.release());
      const second = yield* Effect.promise(() => pool.acquire("remote", create));
      expect(second.workspace).toBe(first.workspace);
      expect(second.wokeFromSleep).toBe(true);
      expect(create).toHaveBeenCalledOnce();
      yield* Effect.promise(() => second.release());
      yield* Effect.promise(() => pool.destroyAll());
      expect(remote.destroy).toHaveBeenCalledOnce();
    }).pipe(Effect.provide(TestClock.layer())),
  );

  effectIt.effect("keeps destroy intent until the final shared release", () =>
    Effect.gen(function* () {
      const clock = yield* Clock.Clock;
      const pool = new BotWorkspacePool({ idleTimeToLive: "1 hour", clock });
      const workspace = localWorkspace();
      const destroy = vi.spyOn(workspace, "destroy");
      const stop = vi.spyOn(workspace, "stop");
      const first = yield* Effect.promise(() => pool.acquire("shared", async () => workspace));
      const second = yield* Effect.promise(() => pool.acquire("shared", async () => workspace));
      yield* Effect.promise(() => first.release({ destroy: true }));
      expect(destroy).not.toHaveBeenCalled();
      yield* Effect.promise(() => second.release());
      expect(destroy).toHaveBeenCalledOnce();
      expect(stop).not.toHaveBeenCalled();
    }).pipe(Effect.provide(TestClock.layer())),
  );

  effectIt.effect("invalidates on destroy release and reacquires a fresh workspace", () =>
    Effect.gen(function* () {
      const clock = yield* Clock.Clock;
      const pool = new BotWorkspacePool({ idleTimeToLive: "1 hour", clock });
      const firstWorkspace = localWorkspace();
      const secondWorkspace = localWorkspace();

      const create = vi
        .fn<() => Promise<Workspace>>()
        .mockResolvedValueOnce(firstWorkspace)
        .mockResolvedValueOnce(secondWorkspace);

      const first = yield* Effect.promise(() => pool.acquire("replace", create));
      yield* Effect.promise(() => first.release({ destroy: true }));
      const second = yield* Effect.promise(() => pool.acquire("replace", create));
      expect(second.workspace.workspace).toBe(secondWorkspace);
      expect(second.wokeFromSleep).toBe(false);
      expect(create).toHaveBeenCalledTimes(2);
      yield* Effect.promise(() => second.release({ destroy: true }));
    }).pipe(Effect.provide(TestClock.layer())),
  );

  effectIt.effect("destroyAll destroys leased and idle workspaces instead of sleeping them", () =>
    Effect.gen(function* () {
      const clock = yield* Clock.Clock;
      const pool = new BotWorkspacePool({ idleTimeToLive: "1 hour", clock });
      const leased = localWorkspace();
      const idle = localWorkspace();
      const leasedDestroy = vi.spyOn(leased, "destroy");
      const idleDestroy = vi.spyOn(idle, "destroy");
      const idleStop = vi.spyOn(idle, "stop");
      const lease = yield* Effect.promise(() => pool.acquire("leased", async () => leased));
      const idleLease = yield* Effect.promise(() => pool.acquire("idle", async () => idle));
      yield* Effect.promise(() => idleLease.release());
      yield* Effect.promise(() => pool.destroyAll());
      expect(leasedDestroy).toHaveBeenCalledOnce();
      expect(idleDestroy).toHaveBeenCalledOnce();
      expect(idleStop).not.toHaveBeenCalled();
      yield* Effect.promise(() => lease.release());
      expect(leasedDestroy).toHaveBeenCalledOnce();
    }).pipe(Effect.provide(TestClock.layer())),
  );

  effectIt.effect("rejects new leases after destroyAll without creating a workspace", () =>
    Effect.gen(function* () {
      const clock = yield* Clock.Clock;
      const pool = new BotWorkspacePool({ idleTimeToLive: "1 hour", clock });
      yield* Effect.promise(() => pool.destroyAll());
      const create = vi.fn(async () => localWorkspace());

      const error = yield* Effect.promise(() =>
        pool.acquire("late", create).then(
          () => undefined,
          (cause: unknown) => cause,
        ),
      );

      expect(error).toMatchObject({
        _tag: "BotWorkspacePoolError",
        message: "Bot workspaces are shutting down.",
      });
      expect(create).not.toHaveBeenCalled();
    }).pipe(Effect.provide(TestClock.layer())),
  );

  effectIt.effect("destroyAll reports a failed destroy", () =>
    Effect.gen(function* () {
      const clock = yield* Clock.Clock;
      const pool = new BotWorkspacePool({ idleTimeToLive: "1 hour", clock });
      const workspace = localWorkspace();
      vi.spyOn(workspace, "destroy").mockRejectedValueOnce(new Error("destroy failed"));
      yield* Effect.promise(() => pool.acquire("failing", async () => workspace));

      const error = yield* Effect.promise(() =>
        pool.destroyAll().then(
          () => undefined,
          (cause: unknown) => cause,
        ),
      );

      expect(error).toBeInstanceOf(Error);
      expect((error as Error).message).toBe("destroy failed");
    }).pipe(Effect.provide(TestClock.layer())),
  );

  effectIt.effect("destroys a workspace whose wake fails and retries creation", () =>
    Effect.gen(function* () {
      const clock = yield* Clock.Clock;
      const pool = new BotWorkspacePool({ idleTimeToLive: "1 hour", clock });
      const failed = localWorkspace();
      const healthy = localWorkspace();
      vi.spyOn(failed, "init").mockRejectedValueOnce(new Error("wake failed"));
      const failedDestroy = vi.spyOn(failed, "destroy");

      const create = vi
        .fn<() => Promise<Workspace>>()
        .mockRejectedValueOnce(new Error("unavailable"))
        .mockResolvedValueOnce(failed)
        .mockResolvedValueOnce(healthy);

      const rejection = (promise: Promise<unknown>) =>
        Effect.promise(() =>
          promise.then(
            () => undefined,
            (cause: unknown) => cause,
          ),
        );

      expect(yield* rejection(pool.acquire("retry", create))).toMatchObject({
        message: "unavailable",
      });
      expect(yield* rejection(pool.acquire("retry", create))).toMatchObject({
        message: "wake failed",
      });
      expect(failedDestroy).toHaveBeenCalledOnce();
      const lease = yield* Effect.promise(() => pool.acquire("retry", create));
      expect(lease.workspace.workspace).toBe(healthy);
      expect(create).toHaveBeenCalledTimes(3);
      yield* Effect.promise(() => lease.release({ destroy: true }));
    }).pipe(Effect.provide(TestClock.layer())),
  );

  effectIt.effect("destroys a workspace that fails to sleep and replaces it", () =>
    Effect.gen(function* () {
      const clock = yield* Clock.Clock;
      const pool = new BotWorkspacePool({ clock });
      const failed = localWorkspace();
      const replacement = localWorkspace();
      vi.spyOn(failed, "stop").mockRejectedValueOnce(new Error("sleep failed"));
      const failedDestroy = vi.spyOn(failed, "destroy");

      const create = vi
        .fn<() => Promise<Workspace>>()
        .mockResolvedValueOnce(failed)
        .mockResolvedValueOnce(replacement);

      const first = yield* Effect.promise(() => pool.acquire("sleep", create));

      const error = yield* Effect.promise(() =>
        first.release().then(
          () => undefined,
          (cause: unknown) => cause,
        ),
      );

      expect(error).toMatchObject({ message: "sleep failed" });
      const second = yield* Effect.promise(() => pool.acquire("sleep", create));
      expect(failedDestroy).toHaveBeenCalledOnce();
      expect(second.workspace.workspace).toBe(replacement);
      expect(second.wokeFromSleep).toBe(false);
      yield* Effect.promise(() => second.release({ destroy: true }));
    }).pipe(Effect.provide(TestClock.layer())),
  );
});