Skip to content

Self-improving AI Employees for everyone

4.4 KB
Raw Blame
/** Versioned browser stream envelope. The bytes after the UTF-8 JSON header are JPEG. */
export interface BrowserFrameHeader {
  seq: number;
  target_id?: string;
  device_width: number;
  device_height: number;
  page_scale_factor?: number;
  offset_top?: number;
}

export function decodeBrowserFrame(buffer: ArrayBuffer): { header: BrowserFrameHeader; jpeg: Blob } {
  if (buffer.byteLength < 5) throw new Error('Incomplete browser frame');
  const view = new DataView(buffer);
  if (view.getUint8(0) !== 1 || view.getUint8(1) !== 1) throw new Error('Unsupported browser stream version');
  const end = 4 + view.getUint16(2, false);
  if (end >= buffer.byteLength) throw new Error('Incomplete browser frame payload');
  const header = JSON.parse(new TextDecoder().decode(new Uint8Array(buffer, 4, end - 4))) as BrowserFrameHeader;
  if (!Number.isFinite(header.seq) || !Number.isFinite(header.device_width) || !Number.isFinite(header.device_height) || !(header.device_width > 0) || !(header.device_height > 0)) {
    throw new Error('Invalid browser frame metadata');
  }
  return { header, jpeg: new Blob([new Uint8Array(buffer, end)], { type: 'image/jpeg' }) };
}

/** Map the aspect-fitted image into CDP viewport CSS coordinates; letterbox clicks are ignored. */
export function browserPoint(
  x: number, y: number, width: number, height: number,
  imageWidth: number, imageHeight: number, header: BrowserFrameHeader,
): { x: number; y: number } | null {
  if (![width, height, imageWidth, imageHeight].every((n) => n > 0)) return null;
  const scale = Math.min(width / imageWidth, height / imageHeight);
  const drawnWidth = imageWidth * scale;
  const drawnHeight = imageHeight * scale;
  const px = x - (width - drawnWidth) / 2;
  const py = y - (height - drawnHeight) / 2;
  if (px < 0 || py < 0 || px > drawnWidth || py > drawnHeight) return null;
  const pageScale = header.page_scale_factor && header.page_scale_factor > 0 ? header.page_scale_factor : 1;
  return {
    x: (px / drawnWidth * header.device_width) / pageScale,
    y: Math.max(0, (py / drawnHeight * header.device_height - (header.offset_top ?? 0)) / pageScale),
  };
}

/** Where a viewport CSS point shows on the picture, as fractions of its width
 *  and height (0 to 1 inside it): the inverse of `browserPoint`. */
export function framePosition(x: number, y: number, header: BrowserFrameHeader): { x: number; y: number } {
  const pageScale = header.page_scale_factor && header.page_scale_factor > 0 ? header.page_scale_factor : 1;
  return { x: (x * pageScale) / header.device_width, y: (y * pageScale + (header.offset_top ?? 0)) / header.device_height };
}

/** Where the agent acted (`agent_action`), in the viewport CSS pixels
 *  `browserPoint` gives: `point` is where its cursor goes, `box` the field it
 *  typed into or chose from. `id` counts actions, so the same point twice
 *  still shows twice. */
export interface BrowserAgentAction {
  id: number;
  action: 'click' | 'hover' | 'type' | 'select';
  targetId?: string;
  point: { x: number; y: number };
  box?: { left: number; top: number; width: number; height: number };
}

const AGENT_ACTIONS: ReadonlySet<string> = new Set(['click', 'hover', 'type', 'select']);
const finite = (value: unknown): value is number => typeof value === 'number' && Number.isFinite(value);

/** An `agent_action` message, or null. The cursor goes into a field where
 *  the design aims at one: 55% across (at most 60px in) and 55% down. */
export function parseAgentAction(message: Record<string, unknown>, id: number): BrowserAgentAction | null {
  const action = message.action;
  if (typeof action !== 'string' || !AGENT_ACTIONS.has(action)) return null;
  const base = { id, action: action as BrowserAgentAction['action'], targetId: typeof message.target_id === 'string' ? message.target_id : undefined };
  const box = message.box;
  if (Array.isArray(box) && box.length === 4 && box.every(finite) && box[2] >= 0 && box[3] >= 0) {
    const [left, top, width, height] = box;
    return { ...base, box: { left, top, width, height }, point: { x: left + Math.min(width * 0.55, 60), y: top + height * 0.55 } };
  }
  return finite(message.x) && finite(message.y) ? { ...base, point: { x: message.x, y: message.y } } : null;
}

export function browserModifiers(event: { altKey: boolean; ctrlKey: boolean; metaKey: boolean; shiftKey: boolean }): number {
  return (event.altKey ? 1 : 0) | (event.ctrlKey ? 2 : 0) | (event.metaKey ? 4 : 0) | (event.shiftKey ? 8 : 0);
}