Skip to content

armgit / E2B

Public Imported from GitHub
0

Open-source, secure environment with real-world tools for enterprise-grade agents.

19.6 KB
Raw Blame
import {
  Sandbox as SandboxBase,
  SandboxOpts as SandboxOptsBase,
  CommandHandle,
  CommandResult,
  CommandExitError,
  TimeoutError,
} from 'e2b'

import { generateRandomString } from './utils'

interface CursorPosition {
  x: number
  y: number
}

interface ScreenSize {
  width: number
  height: number
}

const MOUSE_BUTTONS = {
  left: 1,
  right: 3,
  middle: 2,
}

const KEYS = {
  alt: 'Alt_L',
  alt_left: 'Alt_L',
  alt_right: 'Alt_R',
  backspace: 'BackSpace',
  break: 'Pause',
  caps_lock: 'Caps_Lock',
  cmd: 'Super_L',
  command: 'Super_L',
  control: 'Control_L',
  control_left: 'Control_L',
  control_right: 'Control_R',
  ctrl: 'Control_L',
  del: 'Delete',
  delete: 'Delete',
  down: 'Down',
  end: 'End',
  enter: 'Return',
  esc: 'Escape',
  escape: 'Escape',
  f1: 'F1',
  f2: 'F2',
  f3: 'F3',
  f4: 'F4',
  f5: 'F5',
  f6: 'F6',
  f7: 'F7',
  f8: 'F8',
  f9: 'F9',
  f10: 'F10',
  f11: 'F11',
  f12: 'F12',
  home: 'Home',
  insert: 'Insert',
  left: 'Left',
  menu: 'Menu',
  meta: 'Meta_L',
  num_lock: 'Num_Lock',
  page_down: 'Page_Down',
  page_up: 'Page_Up',
  pause: 'Pause',
  print: 'Print',
  right: 'Right',
  scroll_lock: 'Scroll_Lock',
  shift: 'Shift_L',
  shift_left: 'Shift_L',
  shift_right: 'Shift_R',
  space: 'space',
  super: 'Super_L',
  super_left: 'Super_L',
  super_right: 'Super_R',
  tab: 'Tab',
  up: 'Up',
  win: 'Super_L',
  windows: 'Super_L',
}

function mapKey(key: string): string {
  const lowerKey = key.toLowerCase()
  if (lowerKey in KEYS) {
    return KEYS[lowerKey as keyof typeof KEYS]
  }
  return lowerKey
}

/**
 * Configuration options for the Sandbox environment.
 * @interface SandboxOpts
 * @extends {SandboxOptsBase}
 */
export interface SandboxOpts extends SandboxOptsBase {
  /**
   * The screen resolution in pixels, specified as [width, height].
   * @type {[number, number]}
   */
  resolution?: [number, number]

  /**
   * Dots per inch (DPI) setting for the display.
   * @type {number}
   */
  dpi?: number

  /**
   * Display identifier.
   * @type {string}
   */
  display?: string
}

export class Sandbox extends SandboxBase {
  protected static override readonly defaultTemplate: string = 'desktop'
  public display: string = ':0'
  public stream: VNCServer = new VNCServer(this)
  private lastXfce4Pid: number | null = null

  /**
   * Use {@link Sandbox.create} to create a new Sandbox instead.
   *
   * @hidden
   * @hide
   * @internal
   * @access protected
   */
  constructor(
    opts: Omit<SandboxOpts, 'timeoutMs' | 'metadata'> & {
      sandboxId: string
      envdVersion: string
    }
  ) {
    super(opts)
  }

  /**
   * Create a new sandbox from the default `desktop` sandbox template.
   *
   * @param opts connection options.
   *
   * @returns sandbox instance for the new sandbox.
   *
   * @example
   * ```ts
   * const sandbox = await Sandbox.create()
   * ```
   * @constructs Sandbox
   */
  static async create<S extends typeof Sandbox>(
    this: S,
    opts?: SandboxOpts
  ): Promise<InstanceType<S>>
  /**
   * Create a new sandbox from the specified sandbox template.
   *
   * @param template sandbox template name or ID.
   * @param opts connection options.
   *
   * @returns sandbox instance for the new sandbox.
   *
   * @example
   * ```ts
   * const sandbox = await Sandbox.create('<template-name-or-id>')
   * ```
   * @constructs Sandbox
   */
  static async create<S extends typeof Sandbox>(
    this: S,
    template: string,
    opts?: SandboxOpts
  ): Promise<InstanceType<S>>
  static async create<S extends typeof Sandbox>(
    this: S,
    templateOrOpts?: SandboxOpts | string,
    opts?: SandboxOpts
  ): Promise<InstanceType<S>> {
    const { template, sandboxOpts } =
      typeof templateOrOpts === 'string'
        ? { template: templateOrOpts, sandboxOpts: opts }
        : { template: this.defaultTemplate, sandboxOpts: templateOrOpts }

    // Add DISPLAY environment variable if not already set
    const display = opts?.display || ':0'
    const sandboxOptsWithDisplay = {
      ...sandboxOpts,
      envs: {
        ...sandboxOpts?.envs,
        DISPLAY: display,
      },
    }

    const sbx = (await super.create(
      template,
      sandboxOptsWithDisplay
    )) as InstanceType<S>
    try {
      await sbx._start(display, sandboxOptsWithDisplay)
    } catch (error) {
      await sbx.kill().catch(() => {})
      throw error
    }

    return sbx
  }

  /**
   * Wait for a command to return a specific result.
   * @param cmd - The command to run.
   * @param onResult - The function to check the result of the command.
   * @param timeout - The maximum time to wait for the command to return the result.
   * @param interval - The interval to wait between checks.
   * @returns `true` if the command returned the result within the timeout, otherwise `false`.
   */
  async waitAndVerify(
    cmd: string,
    onResult: (result: CommandResult) => boolean,
    timeout: number = 10,
    interval: number = 0.5
  ): Promise<boolean> {
    let elapsed = 0

    while (elapsed < timeout) {
      try {
        if (onResult(await this.commands.run(cmd))) {
          return true
        }
      } catch (e) {
        if (!(e instanceof CommandExitError)) {
          throw e
        }
      }

      await new Promise((resolve) => setTimeout(resolve, interval * 1000))
      elapsed += interval
    }

    return false
  }

  /**
   * Take a screenshot and save it to the given name.
   * @param format - The format of the screenshot.
   * @returns A Uint8Array bytes representation of the screenshot.
   */
  async screenshot(): Promise<Uint8Array>
  /**
   * Take a screenshot and save it to the given name.
   * @param format - The format of the screenshot.
   * @returns A Uint8Array bytes representation of the screenshot.
   */
  async screenshot(format: 'bytes'): Promise<Uint8Array>
  /**
   * Take a screenshot and save it to the given name.
   * @returns A Blob representation of the screenshot.
   */
  async screenshot(format: 'blob'): Promise<Blob>
  /**
   * Take a screenshot and save it to the given name.
   * @returns A ReadableStream of bytes representation of the screenshot.
   */
  async screenshot(format: 'stream'): Promise<ReadableStream<Uint8Array>>
  async screenshot(format: 'bytes' | 'blob' | 'stream' = 'bytes') {
    const path = `/tmp/screenshot-${generateRandomString()}.png`
    await this.commands.run(`scrot --pointer ${path}`)

    // @ts-expect-error
    const file = await this.files.read(path, { format })
    // Cleanup of the temporary file is best-effort: it must not block the
    // returned file (which may be an unconsumed stream) and must not reject
    // if the sandbox is already gone.
    void this.files.remove(path).catch(() => {})
    return file
  }

  /**
   * Left click on the mouse position.
   */
  async leftClick(x?: number, y?: number): Promise<void> {
    if (x && y) {
      await this.moveMouse(x, y)
    }

    await this.commands.run('xdotool click 1')
  }

  /**
   * Double left click on the mouse position.
   */
  async doubleClick(x?: number, y?: number): Promise<void> {
    if (x && y) {
      await this.moveMouse(x, y)
    }

    await this.commands.run('xdotool click --repeat 2 1')
  }

  /**
   * Right click on the mouse position.
   */
  async rightClick(x?: number, y?: number): Promise<void> {
    if (x && y) {
      await this.moveMouse(x, y)
    }

    await this.commands.run('xdotool click 3')
  }

  /**
   * Middle click on the mouse position.
   */
  async middleClick(x?: number, y?: number): Promise<void> {
    if (x && y) {
      await this.moveMouse(x, y)
    }

    await this.commands.run('xdotool click 2')
  }

  /**
   * Scroll the mouse wheel by the given amount.
   * @param direction - The direction to scroll. Can be "up" or "down".
   * @param amount - The amount to scroll.
   */
  async scroll(
    direction: 'up' | 'down' = 'down',
    amount: number = 1
  ): Promise<void> {
    const button = direction === 'up' ? '4' : '5'
    await this.commands.run(`xdotool click --repeat ${amount} ${button}`)
  }

  /**
   * Move the mouse to the given coordinates.
   * @param x - The x coordinate.
   * @param y - The y coordinate.
   */
  async moveMouse(x: number, y: number): Promise<void> {
    await this.commands.run(`xdotool mousemove --sync ${x} ${y}`)
  }

  /**
   * Press the mouse button.
   */
  async mousePress(
    button: 'left' | 'right' | 'middle' = 'left'
  ): Promise<void> {
    await this.commands.run(`xdotool mousedown ${MOUSE_BUTTONS[button]}`)
  }

  /**
   * Release the mouse button.
   */
  async mouseRelease(
    button: 'left' | 'right' | 'middle' = 'left'
  ): Promise<void> {
    await this.commands.run(`xdotool mouseup ${MOUSE_BUTTONS[button]}`)
  }

  /**
   * Get the current cursor position.
   * @returns A object with the x and y coordinates
   * @throws Error if cursor position cannot be determined
   */
  async getCursorPosition(): Promise<CursorPosition> {
    const result = await this.commands.run('xdotool getmouselocation')

    const match = result.stdout.match(/x:(\d+)\s+y:(\d+)/)
    if (!match) {
      throw new Error(
        `Failed to parse cursor position from output: ${result.stdout}`
      )
    }

    const [, x, y] = match
    if (!x || !y) {
      throw new Error(`Invalid cursor position values: x=${x}, y=${y}`)
    }

    return { x: parseInt(x), y: parseInt(y) }
  }

  /**
   * Get the current screen size.
   * @returns An {@link ScreenSize} object
   * @throws Error if screen size cannot be determined
   */
  async getScreenSize(): Promise<ScreenSize> {
    const result = await this.commands.run('xrandr')

    const match = result.stdout.match(/(\d+x\d+)/)
    if (!match) {
      throw new Error(
        `Failed to parse screen size from output: ${result.stdout}`
      )
    }

    try {
      const [width, height] = match[1].split('x').map((val) => parseInt(val))
      return { width, height }
    } catch (error) {
      throw new Error(`Invalid screen size format: ${match[1]}`)
    }
  }

  /**
   * Write the given text at the current cursor position.
   * @param text - The text to write.
   * @param options - An object containing the chunk size and delay between each chunk of text.
   * @param options.chunkSize - The size of each chunk of text to write. Default is 25 characters.
   * @param options.delayInMs - The delay between each chunk of text. Default is 75 ms.
   */
  async write(
    text: string,
    options: { chunkSize: number; delayInMs: number } = {
      chunkSize: 25,
      delayInMs: 75,
    }
  ): Promise<void> {
    const chunks = this.breakIntoChunks(text, options.chunkSize)

    for (const chunk of chunks) {
      await this.commands.run(
        `xdotool type --delay ${options.delayInMs} -- ${this.quoteString(
          chunk
        )}`
      )
    }
  }

  /**
   * Press a key.
   * @param key - The key to press (e.g. "enter", "space", "backspace", etc.). Can be a single key or an array of keys.
   */
  async press(key: string | string[]): Promise<void> {
    if (Array.isArray(key)) {
      key = key.map(mapKey).join('+')
    } else {
      key = mapKey(key)
    }

    await this.commands.run(`xdotool key ${key}`)
  }

  /**
   * Drag the mouse from the given position to the given position.
   * @param from - The starting position as `[x, y]`.
   * @param to - The ending position as `[x, y]`.
   */
  async drag(from: [number, number], to: [number, number]): Promise<void> {
    await this.moveMouse(...from)
    await this.mousePress()
    await this.moveMouse(...to)
    await this.mouseRelease()
  }

  /**
   * Wait for the given amount of time.
   * @param ms - The amount of time to wait in milliseconds.
   */
  async wait(ms: number): Promise<void> {
    await this.commands.run(`sleep ${ms / 1000}`)
  }

  /**
   * Open a file or a URL in the default application.
   * @param fileOrUrl - The file or URL to open.
   */
  async open(fileOrUrl: string): Promise<void> {
    const handle = await this.commands.run(`xdg-open ${fileOrUrl}`, {
      background: true,
    })
    await handle.disconnect()
  }

  /**
   * Get the current window ID.
   * @returns The ID of the current window.
   */
  async getCurrentWindowId(): Promise<string> {
    const result = await this.commands.run('xdotool getwindowfocus')
    return result.stdout.trim()
  }

  /**
   * Get the window ID of the window with the given title.
   * @param title - The title of the window.
   * @returns The ID of the window.
   */
  async getApplicationWindows(application: string): Promise<string[]> {
    const result = await this.commands.run(
      `xdotool search --onlyvisible --class ${application}`
    )

    return result.stdout.trim().split('\n')
  }

  /**
   * Get the title of the window with the given ID.
   * @param windowId - The ID of the window.
   * @returns The title of the window.
   */
  async getWindowTitle(windowId: string): Promise<string> {
    const result = await this.commands.run(`xdotool getwindowname ${windowId}`)

    return result.stdout.trim()
  }

  /**
   * Launch an application.
   * @param application - The application to launch.
   * @param uri - The URI to open in the application.
   */
  async launch(application: string, uri?: string): Promise<void> {
    const handle = await this.commands.run(
      `gtk-launch ${application} ${uri ?? ''}`,
      {
        background: true,
        timeoutMs: 0,
      }
    )
    await handle.disconnect()
  }

  protected async _start(display: string, opts?: SandboxOpts): Promise<void> {
    this.display = display
    this.lastXfce4Pid = null
    this.stream = new VNCServer(this)

    const [width, height] = opts?.resolution ?? [1024, 768]
    const xvfbHandle = await this.commands.run(
      `Xvfb ${display} -ac -screen 0 ${width}x${height}x24 ` +
        `-retro -dpi ${opts?.dpi ?? 96} -nolisten tcp -nolisten unix`,
      { background: true, timeoutMs: 0 }
    )
    await xvfbHandle.disconnect()

    const hasStarted = await this.waitAndVerify(
      `xdpyinfo -display ${display}`,
      (r: CommandResult) => r.exitCode === 0
    )
    if (!hasStarted) {
      throw new TimeoutError('Could not start Xvfb')
    }

    await this.startXfce4()
  }

  /**
   * Start xfce4 session if logged out or not running.
   */
  private async startXfce4(): Promise<void> {
    if (
      this.lastXfce4Pid === null ||
      (
        await this.commands.run(
          `ps aux | grep ${this.lastXfce4Pid} | grep -v grep | head -n 1`
        )
      ).stdout
        .trim()
        .includes('[xfce4-session] <defunct>')
    ) {
      const result = await this.commands.run('startxfce4', {
        background: true,
        timeoutMs: 0,
      })
      this.lastXfce4Pid = result.pid
      await result.disconnect()
    }

    const hasStarted = await this.waitAndVerify(
      'pgrep -x xfce4-session >/dev/null && pgrep -x xfwm4 >/dev/null && pgrep -x xfdesktop >/dev/null',
      (r: CommandResult) => r.exitCode === 0,
      60
    )
    if (!hasStarted) {
      throw new TimeoutError('Could not start XFCE')
    }
  }

  private *breakIntoChunks(text: string, n: number): Generator<string> {
    for (let i = 0; i < text.length; i += n) {
      yield text.slice(i, i + n)
    }
  }

  private quoteString(s: string): string {
    if (!s) {
      return "''"
    }

    if (!/[^\w@%+=:,./-]/.test(s)) {
      return s
    }

    // use single quotes, and put single quotes into double quotes
    // the string $'b is then quoted as '$'"'"'b'
    return "'" + s.replace(/'/g, "'\"'\"'") + "'"
  }
}

interface VNCServerOptions {
  vncPort?: number
  port?: number
  requireAuth?: boolean
  windowId?: string
}

interface UrlOptions {
  autoConnect?: boolean
  viewOnly?: boolean
  resize?: 'off' | 'scale' | 'remote'
  authKey?: string
}

// Modified VNCServer class
class VNCServer {
  private vncPort: number = 5900
  private port: number = 6080
  private novncAuthEnabled: boolean = false
  private url: URL | null = null
  private novncHandle: CommandHandle | null = null
  private password: string | undefined
  private readonly novncCommand: string
  private readonly desktop: Sandbox

  constructor(desktop: Sandbox) {
    this.desktop = desktop
    this.novncCommand =
      `cd /opt/noVNC/utils && ./novnc_proxy --vnc localhost:${this.vncPort} ` +
      `--listen ${this.port} --web /opt/noVNC > /tmp/novnc.log 2>&1`
  }

  public getAuthKey(): string {
    if (!this.password) {
      throw new Error(
        'Unable to retrieve stream auth key, check if requireAuth is enabled'
      )
    }

    return this.password
  }

  /**
   * Get the URL to a web page with a stream of the desktop sandbox.
   * @param autoConnect - Whether to automatically connect to the server after opening the URL.
   * @param viewOnly - Whether to prevent user interaction through the client.
   * @param resize - Whether to resize the view when the window resizes.
   * @param authKey - The password to use to connect to the server.
   * @returns The URL to connect to the VNC server.
   */
  public getUrl({
    autoConnect = true,
    viewOnly = false,
    resize = 'scale',
    authKey,
  }: UrlOptions = {}): string {
    if (this.url === null) {
      throw new Error('Server is not running')
    }

    const url = new URL(this.url)
    if (autoConnect) {
      url.searchParams.set('autoconnect', 'true')
    }
    if (viewOnly) {
      url.searchParams.set('view_only', 'true')
    }
    if (resize) {
      url.searchParams.set('resize', resize)
    }
    if (authKey) {
      url.searchParams.set('password', authKey)
    }
    return url.toString()
  }

  /**
   * Start the VNC server.
   */
  public async start(opts: VNCServerOptions = {}): Promise<void> {
    // If stream is already running, throw an error.
    if (await this.checkVNCRunning()) {
      throw new Error('Stream is already running')
    }

    this.vncPort = opts.vncPort ?? this.vncPort
    this.port = opts.port ?? this.port
    this.novncAuthEnabled = opts.requireAuth ?? this.novncAuthEnabled
    this.password = this.novncAuthEnabled ? generateRandomString() : undefined
    this.url = new URL(`https://${this.desktop.getHost(this.port)}/vnc.html`)

    const vncCommand = await this.getVNCCommand(opts.windowId)
    await this.desktop.commands.run(vncCommand)

    this.novncHandle = await this.desktop.commands.run(this.novncCommand, {
      background: true,
      timeoutMs: 0,
    })
    if (!(await this.waitForPort(this.port))) {
      throw new Error('Could not start noVNC server')
    }
  }

  /**
   * Stop the VNC server.
   */
  public async stop(): Promise<void> {
    if (await this.checkVNCRunning()) {
      await this.desktop.commands.run('pkill x11vnc')
    }

    if (this.novncHandle) {
      await this.novncHandle.kill()
      this.novncHandle = null
    }
  }

  /**
   * Set the VNC command to start the VNC server.
   */
  private async getVNCCommand(windowId?: string): Promise<string> {
    let pwdFlag = '-nopw'
    if (this.novncAuthEnabled) {
      // Create .vnc directory if it doesn't exist
      await this.desktop.commands.run('mkdir -p ~/.vnc')
      await this.desktop.commands.run(
        `x11vnc -storepasswd ${this.password} ~/.vnc/passwd`
      )
      pwdFlag = '-usepw'
    }

    return (
      `x11vnc -bg -display ${this.desktop.display} -forever -wait 50 -shared ` +
      `-rfbport ${this.vncPort} ${pwdFlag} 2>/tmp/x11vnc_stderr.log` +
      (windowId ? ` -id ${windowId}` : '')
    )
  }

  private async waitForPort(port: number): Promise<boolean> {
    return await this.desktop.waitAndVerify(
      `netstat -tuln | grep ":${port} "`,
      (r: CommandResult) => r.stdout.trim() !== ''
    )
  }

  /**
   * Check if the VNC server is running.
   * @returns Whether the VNC server is running.
   */
  private async checkVNCRunning(): Promise<boolean> {
    try {
      const result = await this.desktop.commands.run('pgrep -x x11vnc')
      return result.stdout.trim() !== ''
    } catch (error) {
      return false
    }
  }
}