Skip to content

Open-source Grok Bot alternative for named AI teammates. Local-first desktop, web, and mobile clients with per-bot tools, instructions, and memory. Independent fork of T3 Code.

24.3 KB
Raw Blame
import type * as Schema from "effect/Schema";
import { it as effectIt } from "@effect/vitest";

import type { DesktopPreviewRecordingFrame } from "@akeru/contracts";

import * as Effect from "effect/Effect";

import * as Fiber from "effect/Fiber";

import { TestClock } from "effect/testing";

import { beforeEach, describe, expect, vi } from "vite-plus/test";

import * as PreviewManager from "./Manager.ts";

import {
  createPreviewManagerHarness,
  type TestCapturedPreviewImage,
} from "./test-support/PreviewManagerHarness.ts";

const {
  browserWindowConstructor,
  createFromBuffer,
  createFromPath,
  fromId,
  getFocusedWebContents,
  mkdir,
  showItemInFolder,
  webviewSend,
  writeFile,
  writeImage,
} = vi.hoisted(() => ({
  browserWindowConstructor: vi.fn(),
  createFromBuffer: vi.fn(),
  createFromPath: vi.fn((): Pick<Electron.NativeImage, "isEmpty"> => ({ isEmpty: () => false })),
  fromId: vi.fn((_id?: number): Electron.WebContents | null => null),
  getFocusedWebContents: vi.fn((): Electron.WebContents | null => null),
  mkdir: vi.fn((_path: string) => undefined),
  showItemInFolder: vi.fn(),
  webviewSend: vi.fn(),
  writeFile: vi.fn((_path: string, _data: Uint8Array) => undefined),
  writeImage: vi.fn(),
}));

vi.mock("electron", () => ({
  BrowserWindow: browserWindowConstructor,
  clipboard: {
    writeImage,
  },
  nativeImage: {
    createFromBuffer,
    createFromPath,
  },
  shell: {
    showItemInFolder,
  },
  session: {
    fromPartition: vi.fn(),
  },
  webContents: {
    fromId,
    getFocusedWebContents,
  },
}));

const { withManager, makeTestPreviewWebContents, makeTestPictureInPictureWindow } =
  createPreviewManagerHarness({
    browserWindowConstructor,
    createFromBuffer,
    createFromPath,
    fromId,
    getFocusedWebContents,
    mkdir,
    showItemInFolder,
    webviewSend,
    writeFile,
    writeImage,
  });

describe("PreviewManager", () => {
  beforeEach(() => {
    browserWindowConstructor.mockReset();
    fromId.mockClear();
    getFocusedWebContents.mockReset();
    getFocusedWebContents.mockReturnValue(null);
    mkdir.mockClear();
    writeFile.mockClear();
    showItemInFolder.mockClear();
    writeImage.mockClear();
    createFromPath.mockClear();
    createFromBuffer.mockReset();
    webviewSend.mockClear();
  });

  effectIt.effect("captures hidden preview recordings independently for concurrent tabs", () =>
    withManager((manager) =>
      Effect.gen(function* () {
        const firstJpeg = Buffer.from("first-recording-frame");
        const secondJpeg = Buffer.from("second-recording-frame");

        const firstCapturePage = vi.fn(async () => ({
          toJPEG: () => firstJpeg,
          getSize: () => ({ width: 800, height: 600 }),
        }));

        const secondCapturePage = vi.fn(async () => ({
          toJPEG: () => secondJpeg,
          getSize: () => ({ width: 390, height: 844 }),
        }));

        const firstSendCommand = vi.fn(async () => undefined);
        const secondSendCommand = vi.fn(async () => undefined);

        const makeWebContents = (
          id: number,
          capturePage: typeof firstCapturePage,
          sendCommand: typeof firstSendCommand,
        ) =>
          ({
            id,
            isDestroyed: () => false,
            getType: () => "webview",
            getURL: () => `https://example.com/${id}`,
            getTitle: () => `Example ${id}`,
            isLoading: () => false,
            getZoomFactor: () => 1,
            setZoomFactor: vi.fn(),
            setAudioMuted: vi.fn(),
            isCurrentlyAudible: () => false,
            on: vi.fn(),
            off: vi.fn(),
            ipc: { on: vi.fn(), off: vi.fn() },
            send: webviewSend,
            navigationHistory: { canGoBack: () => false, canGoForward: () => false },
            setIgnoreMenuShortcuts: vi.fn(),
            setWindowOpenHandler: vi.fn(),
            debugger: {
              isAttached: () => false,
              attach: vi.fn(),
              sendCommand,
              on: vi.fn(),
              off: vi.fn(),
            },
            capturePage,
          }) as never;

        const webContentsById = new Map([
          [41, makeWebContents(41, firstCapturePage, firstSendCommand)],
          [42, makeWebContents(42, secondCapturePage, secondSendCommand)],
        ]);

        fromId.mockImplementation((id) =>
          id === undefined ? null : (webContentsById.get(id) ?? null),
        );
        const frames: DesktopPreviewRecordingFrame[] = [];

        yield* manager.subscribeRecordingFrames((frame) =>
          Effect.sync(() => {
            frames.push(frame);
          }),
        );
        yield* manager.createTab("tab_1");
        yield* manager.createTab("tab_2");
        yield* manager.registerWebview("tab_1", 41);
        yield* manager.registerWebview("tab_2", 42);
        yield* Effect.all([manager.startRecording("tab_1"), manager.startRecording("tab_2")], {
          concurrency: 2,
          discard: true,
        });

        expect(firstCapturePage).toHaveBeenCalledOnce();
        expect(secondCapturePage).toHaveBeenCalledOnce();
        expect(frames).toHaveLength(2);
        expect(frames).toEqual(
          expect.arrayContaining([
            expect.objectContaining({
              tabId: "tab_1",
              data: firstJpeg.toString("base64"),
              width: 800,
              height: 600,
            }),
            expect.objectContaining({
              tabId: "tab_2",
              data: secondJpeg.toString("base64"),
              width: 390,
              height: 844,
            }),
          ]),
        );
        expect(firstSendCommand).not.toHaveBeenCalledWith(
          "Page.startScreencast",
          expect.anything(),
        );
        expect(secondSendCommand).not.toHaveBeenCalledWith(
          "Page.startScreencast",
          expect.anything(),
        );

        yield* Effect.all([manager.stopRecording("tab_1"), manager.stopRecording("tab_2")], {
          concurrency: 2,
          discard: true,
        });
      }),
    ),
  );

  effectIt.effect("drops a captured frame when the tab webview changes during capture", () =>
    withManager((manager) =>
      Effect.gen(function* () {
        const staleImage: TestCapturedPreviewImage = {
          toJPEG: vi.fn(() => Buffer.from("stale-recording-frame")),
          getSize: vi.fn(() => ({ width: 1280, height: 720 })),
        };

        let markCaptureStarted!: () => void;

        const captureStarted = new Promise<void>((resolve) => {
          markCaptureStarted = resolve;
        });

        let resolveCapture: ((image: TestCapturedPreviewImage) => void) | undefined;

        const staleCapturePage = vi.fn(() => {
          markCaptureStarted();

          return new Promise<TestCapturedPreviewImage>((resolve) => {
            resolveCapture = resolve;
          });
        });

        const replacementCapturePage = vi.fn(async () => ({
          toJPEG: () => Buffer.from("replacement-recording-frame"),
          getSize: () => ({ width: 1280, height: 720 }),
        }));

        const initialWebContents = makeTestPreviewWebContents(staleCapturePage, 42);
        const replacementWebContents = makeTestPreviewWebContents(replacementCapturePage, 43);
        fromId.mockImplementation((webContentsId?: number) => {
          if (webContentsId === 42) return initialWebContents;

          if (webContentsId === 43) return replacementWebContents;

          return null;
        });
        const frames: DesktopPreviewRecordingFrame[] = [];

        yield* manager.subscribeRecordingFrames((frame) =>
          Effect.sync(() => {
            frames.push(frame);
          }),
        );
        yield* manager.createTab("tab_capture_replaced");
        yield* manager.registerWebview("tab_capture_replaced", 42);

        const recordingFiber = yield* manager
          .startRecording("tab_capture_replaced")
          .pipe(Effect.forkChild({ startImmediately: true }));

        yield* Effect.promise(() => captureStarted);

        yield* manager.registerWebview("tab_capture_replaced", 43);
        resolveCapture?.(staleImage);
        yield* Fiber.join(recordingFiber);

        expect(staleImage.getSize).not.toHaveBeenCalled();
        expect(staleImage.toJPEG).not.toHaveBeenCalled();
        expect(frames).toHaveLength(0);
        expect(replacementCapturePage).not.toHaveBeenCalled();

        yield* manager.stopRecording("tab_capture_replaced");
      }),
    ),
  );

  effectIt.effect("keeps an in-flight frame when a capture consumer is added", () =>
    withManager((manager) =>
      Effect.gen(function* () {
        const image: TestCapturedPreviewImage = {
          toJPEG: vi.fn(() => Buffer.from("shared-in-flight-frame")),
          getSize: vi.fn(() => ({ width: 1280, height: 720 })),
        };

        let markCaptureStarted!: () => void;

        const captureStarted = new Promise<void>((resolve) => {
          markCaptureStarted = resolve;
        });

        let resolveCapture: ((captured: TestCapturedPreviewImage) => void) | undefined;

        const capturePage = vi.fn(() => {
          markCaptureStarted();

          return new Promise<TestCapturedPreviewImage>((resolve) => {
            resolveCapture = resolve;
          });
        });

        fromId.mockReturnValue(makeTestPreviewWebContents(capturePage));
        const { pictureInPictureWindow, send } = makeTestPictureInPictureWindow();
        browserWindowConstructor.mockImplementation(function () {
          return pictureInPictureWindow;
        });
        const recordingFrames: DesktopPreviewRecordingFrame[] = [];
        yield* manager.subscribeRecordingFrames((frame) =>
          Effect.sync(() => {
            recordingFrames.push(frame);
          }),
        );

        yield* manager.createTab("tab_capture_consumer_added");
        yield* manager.registerWebview("tab_capture_consumer_added", 42);

        const recordingFiber = yield* manager
          .startRecording("tab_capture_consumer_added")
          .pipe(Effect.forkChild({ startImmediately: true }));

        yield* Effect.promise(() => captureStarted);

        yield* manager.openPictureInPicture("tab_capture_consumer_added");
        resolveCapture?.(image);
        yield* Fiber.join(recordingFiber);

        expect(recordingFrames).toHaveLength(1);
        expect(send).toHaveBeenCalledWith(
          "desktop:preview-pip-frame",
          expect.objectContaining({
            tabId: "tab_capture_consumer_added",
            data: Buffer.from("shared-in-flight-frame").toString("base64"),
          }),
        );

        yield* manager.stopRecording("tab_capture_consumer_added");
        yield* manager.closePictureInPicture("tab_capture_consumer_added");
      }),
    ),
  );

  effectIt.effect("emits debugger screencast frames only while recording is active", () =>
    withManager((manager) =>
      Effect.gen(function* () {
        let debuggerMessage:
          | ((
              event: Electron.Event,
              method: string,
              params: Record<string, Schema.Json | undefined>,
            ) => void)
          | undefined;

        const capturePage = vi.fn(async () => ({
          toJPEG: () => Buffer.from("scheduled-recording-frame"),
          getSize: () => ({ width: 1280, height: 720 }),
        }));

        const sendCommand = vi.fn(async (method: string) =>
          method === "Runtime.evaluate" ? { result: { value: null } } : undefined,
        );

        fromId.mockReturnValue({
          id: 42,
          isDestroyed: () => false,
          getType: () => "webview",
          getURL: () => "https://example.com",
          getTitle: () => "Example",
          isLoading: () => false,
          isDevToolsOpened: () => false,
          getZoomFactor: () => 1,
          setZoomFactor: vi.fn(),
          setAudioMuted: vi.fn(),
          isCurrentlyAudible: () => false,
          on: vi.fn(),
          off: vi.fn(),
          ipc: { on: vi.fn(), off: vi.fn() },
          send: webviewSend,
          navigationHistory: { canGoBack: () => false, canGoForward: () => false },
          setIgnoreMenuShortcuts: vi.fn(),
          setWindowOpenHandler: vi.fn(),
          debugger: {
            isAttached: () => false,
            attach: vi.fn(),
            sendCommand,
            on: vi.fn(
              (
                event: string,
                listener: (
                  event: Electron.Event,
                  method: string,
                  params: Record<string, Schema.Json | undefined>,
                ) => void,
              ) => {
                if (event === "message") debuggerMessage = listener;
              },
            ),
            off: vi.fn(),
          },
          capturePage,
        } as never);
        const recordingFrames: DesktopPreviewRecordingFrame[] = [];

        yield* manager.subscribeRecordingFrames((frame) =>
          Effect.sync(() => {
            recordingFrames.push(frame);
          }),
        );
        yield* manager.createTab("tab_screencast_guard");
        yield* manager.registerWebview("tab_screencast_guard", 42);
        yield* manager.automationEvaluate("tab_screencast_guard", { expression: "null" });

        debuggerMessage?.(
          { preventDefault: () => {}, defaultPrevented: false },
          "Page.screencastFrame",
          {
            sessionId: 1,
            data: "inactive-frame",
            metadata: { deviceWidth: 1280, deviceHeight: 720 },
          },
        );
        yield* Effect.yieldNow;
        expect(recordingFrames).toHaveLength(0);

        yield* manager.startRecording("tab_screencast_guard");
        recordingFrames.length = 0;
        debuggerMessage?.(
          { preventDefault: () => {}, defaultPrevented: false },
          "Page.screencastFrame",
          {
            sessionId: 2,
            data: "active-frame",
            metadata: { deviceWidth: 1280, deviceHeight: 720 },
          },
        );
        yield* Effect.yieldNow;

        expect(recordingFrames).toEqual([
          expect.objectContaining({
            tabId: "tab_screencast_guard",
            data: "active-frame",
            width: 1280,
            height: 720,
          }),
        ]);
        yield* manager.stopRecording("tab_screencast_guard");
      }),
    ),
  );

  effectIt.effect("shares background frame capture between recording and picture-in-picture", () =>
    withManager((manager) =>
      Effect.gen(function* () {
        const setBackgroundThrottling = vi.fn();
        const mainWindowWebContents = { setBackgroundThrottling };
        const jpeg = Buffer.from("shared-preview-frame");

        const capturePage = vi.fn(async () => ({
          toJPEG: () => jpeg,
          getSize: () => ({ width: 1280, height: 720 }),
        }));

        fromId.mockReturnValue({
          id: 42,
          hostWebContents: mainWindowWebContents,
          isDestroyed: () => false,
          getType: () => "webview",
          getURL: () => "https://example.com",
          getTitle: () => "Example",
          isLoading: () => false,
          getZoomFactor: () => 1,
          setZoomFactor: vi.fn(),
          setAudioMuted: vi.fn(),
          isCurrentlyAudible: () => false,
          on: vi.fn(),
          off: vi.fn(),
          ipc: { on: vi.fn(), off: vi.fn() },
          send: webviewSend,
          navigationHistory: { canGoBack: () => false, canGoForward: () => false },
          setIgnoreMenuShortcuts: vi.fn(),
          setWindowOpenHandler: vi.fn(),
          debugger: {
            isAttached: () => false,
            attach: vi.fn(),
            sendCommand: vi.fn(async () => undefined),
            on: vi.fn(),
            off: vi.fn(),
          },
          capturePage,
        } as never);

        const pictureInPictureListeners = new Map<string, () => void>();
        const pictureInPictureSend = vi.fn();

        const pictureInPictureWindow = {
          isDestroyed: vi.fn(() => false),
          once: vi.fn((event: string, listener: () => void) => {
            pictureInPictureListeners.set(event, listener);
          }),
          setAlwaysOnTop: vi.fn(),
          setVisibleOnAllWorkspaces: vi.fn(),
          setAspectRatio: vi.fn(),
          getContentSize: vi.fn(() => [480, 320] as [number, number]),
          setContentSize: vi.fn(),
          loadURL: vi.fn(async () => undefined),
          showInactive: vi.fn(),
          close: vi.fn(() => {
            pictureInPictureListeners.get("closed")?.();
          }),
          webContents: {
            on: vi.fn(),
            off: vi.fn(),
            send: pictureInPictureSend,
          },
        };

        browserWindowConstructor.mockImplementation(function () {
          return pictureInPictureWindow;
        });
        const states: PreviewManager.PreviewTabState[] = [];
        const recordingFrames: DesktopPreviewRecordingFrame[] = [];

        yield* manager.setMainWindow({
          isDestroyed: () => false,
          once: vi.fn(),
          webContents: mainWindowWebContents,
        } as never);

        yield* manager.subscribeStateChanges((_tabId, state) =>
          Effect.sync(() => {
            states.push(state);
          }),
        );
        yield* manager.subscribeRecordingFrames((frame) =>
          Effect.sync(() => {
            recordingFrames.push(frame);
          }),
        );
        yield* manager.createTab("tab_pip");
        yield* manager.registerWebview("tab_pip", 42);
        yield* manager.openPictureInPicture("tab_pip");

        expect(setBackgroundThrottling.mock.calls).toEqual([[false]]);
        expect(browserWindowConstructor).toHaveBeenCalledWith(
          expect.objectContaining({
            alwaysOnTop: true,
            show: false,
            skipTaskbar: true,
            webPreferences: expect.objectContaining({
              preload: "/tmp/t3/desktop/preview-pip-preload.cjs",
              backgroundThrottling: false,
            }),
          }),
        );
        expect(pictureInPictureWindow.showInactive).toHaveBeenCalledOnce();
        expect(pictureInPictureWindow.setVisibleOnAllWorkspaces).toHaveBeenCalledWith(true, {
          visibleOnFullScreen: true,
          skipTransformProcessType: true,
        });
        expect(pictureInPictureWindow.setAspectRatio.mock.calls).toEqual([[0], [1280 / 720]]);
        expect(pictureInPictureWindow.setContentSize).toHaveBeenCalledWith(523, 294, false);
        expect(pictureInPictureWindow.setAspectRatio.mock.invocationCallOrder[0]).toBeLessThan(
          pictureInPictureWindow.setContentSize.mock.invocationCallOrder[0] ?? 0,
        );
        expect(pictureInPictureWindow.setContentSize.mock.invocationCallOrder[0]).toBeLessThan(
          pictureInPictureWindow.setAspectRatio.mock.invocationCallOrder[1] ?? 0,
        );
        expect(pictureInPictureSend).toHaveBeenCalledWith(
          "desktop:preview-pip-frame",
          expect.objectContaining({
            tabId: "tab_pip",
            data: jpeg.toString("base64"),
            width: 1280,
            height: 720,
          }),
        );
        expect(states.at(-1)?.pictureInPicture).toBe(true);
        expect(capturePage).toHaveBeenCalledOnce();
        const pictureInPictureFramesBeforeRecording = pictureInPictureSend.mock.calls.length;

        yield* manager.startRecording("tab_pip");
        expect(capturePage).toHaveBeenCalledOnce();
        expect(recordingFrames).toHaveLength(0);

        yield* TestClock.adjust(100);
        expect(capturePage).toHaveBeenCalledTimes(2);
        expect(pictureInPictureSend).toHaveBeenCalledTimes(pictureInPictureFramesBeforeRecording);
        expect(recordingFrames).toHaveLength(1);

        yield* manager.stopRecording("tab_pip");
        expect(setBackgroundThrottling.mock.calls).toEqual([[false]]);
        const framesBeforePictureInPictureOnlyTick = pictureInPictureSend.mock.calls.length;
        yield* TestClock.adjust(100);
        expect(capturePage).toHaveBeenCalledTimes(3);
        expect(pictureInPictureSend.mock.calls.length).toBe(framesBeforePictureInPictureOnlyTick);
        expect(recordingFrames).toHaveLength(1);

        setBackgroundThrottling.mockImplementationOnce(() => {
          throw new Error("picture-in-picture throttling restore failed");
        });
        yield* manager.closePictureInPicture("tab_pip");
        expect(setBackgroundThrottling.mock.calls).toEqual([[false], [true], [true]]);
        expect(pictureInPictureWindow.close).toHaveBeenCalledOnce();
        expect(states.at(-1)?.pictureInPicture).toBe(false);
        const capturesAfterClose = capturePage.mock.calls.length;
        yield* TestClock.adjust(200);
        expect(capturePage).toHaveBeenCalledTimes(capturesAfterClose);
      }),
    ),
  );

  effectIt.effect("keeps recording cadence when pixels remain unchanged", () =>
    withManager((manager) =>
      Effect.gen(function* () {
        const unchanged = Buffer.from("unchanged-preview-frame");

        const capturePage = vi.fn(async () => ({
          toJPEG: () => unchanged,
          getSize: () => ({ width: 1280, height: 720 }),
        }));

        fromId.mockReturnValue(makeTestPreviewWebContents(capturePage));
        const frames: DesktopPreviewRecordingFrame[] = [];

        yield* manager.subscribeRecordingFrames((frame) =>
          Effect.sync(() => {
            frames.push(frame);
          }),
        );
        yield* manager.createTab("tab_unchanged_frame");
        yield* manager.registerWebview("tab_unchanged_frame", 42);
        yield* manager.startRecording("tab_unchanged_frame");

        expect(frames.map((frame) => frame.data)).toEqual([unchanged.toString("base64")]);

        yield* TestClock.adjust(100);

        expect(capturePage).toHaveBeenCalledTimes(2);
        expect(frames.map((frame) => frame.data)).toEqual([
          unchanged.toString("base64"),
          unchanged.toString("base64"),
        ]);

        yield* TestClock.adjust(100);

        expect(capturePage).toHaveBeenCalledTimes(3);
        expect(frames.map((frame) => frame.data)).toEqual([
          unchanged.toString("base64"),
          unchanged.toString("base64"),
          unchanged.toString("base64"),
        ]);
        yield* manager.stopRecording("tab_unchanged_frame");
      }),
    ),
  );

  effectIt.effect("retries recording delivery when pixels remain unchanged", () =>
    withManager((manager) =>
      Effect.gen(function* () {
        const jpeg = Buffer.from("retry-recording-frame");

        const capturePage = vi.fn(async () => ({
          toJPEG: () => jpeg,
          getSize: () => ({ width: 1280, height: 720 }),
        }));

        fromId.mockReturnValue(makeTestPreviewWebContents(capturePage));
        let deliveries = 0;

        yield* manager.subscribeRecordingFrames(() =>
          Effect.sync(() => {
            deliveries += 1;

            if (deliveries === 1) throw new Error("recording delivery failed");
          }),
        );
        yield* manager.createTab("tab_recording_delivery_retry");
        yield* manager.registerWebview("tab_recording_delivery_retry", 42);
        yield* manager.startRecording("tab_recording_delivery_retry");
        expect(deliveries).toBe(1);

        yield* TestClock.adjust(100);

        expect(deliveries).toBe(2);
        yield* manager.stopRecording("tab_recording_delivery_retry");
      }),
    ),
  );

  effectIt.effect("retries a cold hidden-tab capture without dropping recording", () =>
    withManager((manager) =>
      Effect.gen(function* () {
        const jpeg = Buffer.from("recovered-preview-frame");

        const capturePage = vi.fn(async () => ({
          toJPEG: () => jpeg,
          getSize: () => ({ width: 1280, height: 720 }),
        }));

        capturePage.mockRejectedValueOnce(new Error("UnknownVizError"));
        fromId.mockReturnValue(makeTestPreviewWebContents(capturePage));
        const frames: DesktopPreviewRecordingFrame[] = [];

        yield* manager.subscribeRecordingFrames((frame) =>
          Effect.sync(() => {
            frames.push(frame);
          }),
        );
        yield* manager.createTab("tab_cold_capture");
        yield* manager.registerWebview("tab_cold_capture", 42);

        yield* manager.startRecording("tab_cold_capture");

        expect(capturePage).toHaveBeenCalledOnce();
        expect(frames).toHaveLength(0);

        yield* TestClock.adjust(100);

        expect(capturePage).toHaveBeenCalledTimes(2);
        expect(frames).toEqual([
          expect.objectContaining({
            tabId: "tab_cold_capture",
            data: jpeg.toString("base64"),
            width: 1280,
            height: 720,
          }),
        ]);

        yield* manager.stopRecording("tab_cold_capture");
      }),
    ),
  );
});