Skip to content

armgit / cua

Public Imported from GitHub
0

Scale computer-use 2.0 with open-source drivers, cross-OS sandboxes, and benchmarks for training, evaluation, and data generation.

19.5 KB
Raw Blame
<!doctype html>
<html lang="en">
<head>
  <meta charset="utf-8">
  <meta name="viewport" content="width=device-width, initial-scale=1">
  <title>CUA Media Viewer</title>
  <style>
    :root { color-scheme: dark; font: 14px/1.4 ui-sans-serif, system-ui, sans-serif; }
    body { margin: 0; background: #111318; color: #e8eaf0; }
    main { display: grid; grid-template-columns: minmax(250px, 340px) 1fr; min-height: 100vh; }
    aside { padding: 18px; border-right: 1px solid #30343d; background: #181b21; }
    section { display: grid; place-items: center; padding: 20px; overflow: auto; }
    label { display: block; margin: 10px 0 4px; color: #aeb4c0; }
    input, button, textarea { box-sizing: border-box; width: 100%; border: 1px solid #3a404b; border-radius: 7px; background: #232730; color: inherit; padding: 8px 10px; }
    button { margin-top: 8px; cursor: pointer; background: #315dcb; border-color: #4776e8; }
    button.secondary { background: #292e38; border-color: #3a404b; }
    canvas { max-width: 100%; max-height: calc(100vh - 40px); background: #050608; box-shadow: 0 10px 40px #0009; cursor: crosshair; outline: none; }
    textarea { min-height: 150px; resize: vertical; font: 12px/1.35 ui-monospace, monospace; }
    #status, #stats { margin: 12px 0; color: #8fb0ff; overflow-wrap: anywhere; }
    #stats { color: #aeb4c0; font: 12px/1.35 ui-monospace, monospace; white-space: pre; }
    .row { display: grid; grid-template-columns: 1fr 1fr; gap: 8px; }
    @media (max-width: 760px) { main { grid-template-columns: 1fr; } aside { border-right: 0; border-bottom: 1px solid #30343d; } }
  </style>
</head>
<body>
<main>
  <aside>
    <label for="url">Media URL (from StreamService.OpenMedia: ws(s)://host:3211 + ws_path)</label>
    <input id="url" placeholder="ws://127.0.0.1:3211/media?ticket=…">
    <button id="connect">Connect</button>
    <div id="status">Disconnected</div>
    <div class="row">
      <button id="keyframe" class="secondary">Request keyframe</button>
      <button id="mute" class="secondary">Audio: off</button>
    </div>
    <button id="state" class="secondary">Get window state</button>
    <label for="text">Type text (interactive input)</label>
    <input id="text" placeholder="Text to type">
    <button id="type" class="secondary">Send text</button>
    <div id="stats"></div>
    <label for="events">Events</label>
    <textarea id="events" readonly></textarea>
  </aside>
  <section><canvas id="video" width="960" height="540" tabindex="0"></canvas></section>
</main>
<script>
// rcdp wire v2 viewer (libs/cua/proto/MEDIA.md).
//
// The server speaks first after the ticketed upgrade: hello, session_opened,
// then one audio_config per audio track. Control frames are JSON text
// {"type", "payload"}; video is a length-prefixed binary packet (JSON
// descriptor + Annex B / BGRA / PNG payload); audio is a binary RAU2 packet.
// H.264 decodes with WebCodecs; BGRA and PNG are fallbacks. Opus decodes with
// WebCodecs AudioDecoder; PCM plays directly. Both go through a small jitter
// buffer on the AudioContext clock.
(() => {
  const $ = id => document.getElementById(id);
  const canvas = $('video');
  const context = canvas.getContext('2d', { alpha: false });
  const text = new TextDecoder();
  let socket;
  let session = null;               // session_opened payload
  let geometry = { width_px: 1, height_px: 1, scale_factor: 1 };
  let geometryEpoch = 0;
  let codecEpoch = -1;
  let lastSequence = 0;
  let videoDecoder = null;
  let awaitingKeyframe = true;
  let lastKeyframeRequest = 0;
  const sequenceByTimestamp = new Map();
  let inputSequence = Math.floor(Math.random() * 1e9) + 1;
  const counters = { frames: 0, bytes: 0, audioPackets: 0, audioLost: 0, decodeErrors: 0 };

  const params = new URLSearchParams(location.search);
  $('url').value = params.get('media') || '';

  const log = value => {
    const line = typeof value === 'string' ? value : JSON.stringify(value);
    $('events').value = `${line}\n${$('events').value}`.slice(0, 16000);
  };
  const status = value => { $('status').textContent = value; };
  const send = (type, payload) => {
    if (!socket || socket.readyState !== WebSocket.OPEN) return;
    socket.send(JSON.stringify(payload === undefined ? { type } : { type, payload }));
  };
  const sessionId = () => (session ? session.session_id : '');

  function requestKeyframe() {
    const now = performance.now();
    if (now - lastKeyframeRequest < 1000) return;   // at most one per second
    lastKeyframeRequest = now;
    send('request_keyframe', { session_id: sessionId() });
  }

  // ---------------------------------------------------------------- video
  function codecString(bytes) {
    for (let offset = 0; offset + 5 < bytes.length; offset++) {
      let header = -1;
      if (bytes[offset] === 0 && bytes[offset + 1] === 0 && bytes[offset + 2] === 1) header = offset + 3;
      else if (bytes[offset] === 0 && bytes[offset + 1] === 0 && bytes[offset + 2] === 0 && bytes[offset + 3] === 1) header = offset + 4;
      if (header >= 0 && (bytes[header] & 0x1f) === 7 && header + 3 < bytes.length) {
        return `avc1.${[bytes[header + 1], bytes[header + 2], bytes[header + 3]]
          .map(value => value.toString(16).padStart(2, '0')).join('').toUpperCase()}`;
      }
    }
    throw new Error('H.264 keyframe without SPS');
  }

  function configureDecoder(epoch, keyframe) {
    if (!('VideoDecoder' in window)) throw new Error('This browser has no WebCodecs VideoDecoder; open the session with BGRA or PNG');
    if (videoDecoder && videoDecoder.state !== 'closed') videoDecoder.close();
    videoDecoder = new VideoDecoder({
      output(frame) {
        if (canvas.width !== frame.displayWidth || canvas.height !== frame.displayHeight) {
          canvas.width = frame.displayWidth;
          canvas.height = frame.displayHeight;
        }
        context.drawImage(frame, 0, 0);
        const sequence = sequenceByTimestamp.get(frame.timestamp);
        sequenceByTimestamp.delete(frame.timestamp);
        frame.close();
        if (sequence !== undefined) {
          send('frame_ack', { session_id: sessionId(), sequence, decode_queue: videoDecoder.decodeQueueSize });
        }
      },
      error(error) {
        counters.decodeErrors++;
        status(`Decoder error: ${error.message}`);
        codecEpoch = -1;            // reconfigure on the next keyframe
        awaitingKeyframe = true;
        requestKeyframe();
      }
    });
    videoDecoder.configure({ codec: codecString(keyframe), optimizeForLatency: true, hardwareAcceleration: 'no-preference' });
    codecEpoch = epoch;
    awaitingKeyframe = true;
  }

  async function handleVideo(descriptor, bytes) {
    if (descriptor.session_id !== sessionId()) return;
    counters.frames++;
    counters.bytes += bytes.byteLength;
    if (descriptor.codec === 'png') {
      const bitmap = await createImageBitmap(new Blob([bytes], { type: 'image/png' }));
      canvas.width = descriptor.width_px; canvas.height = descriptor.height_px;
      context.drawImage(bitmap, 0, 0); bitmap.close();
      lastSequence = descriptor.sequence;
      return;
    }
    if (descriptor.codec === 'bgra') {
      const rgba = new Uint8ClampedArray(bytes.byteLength);
      for (let offset = 0; offset < bytes.byteLength; offset += 4) {
        rgba[offset] = bytes[offset + 2]; rgba[offset + 1] = bytes[offset + 1];
        rgba[offset + 2] = bytes[offset]; rgba[offset + 3] = 255;
      }
      canvas.width = descriptor.width_px; canvas.height = descriptor.height_px;
      context.putImageData(new ImageData(rgba, descriptor.width_px, descriptor.height_px), 0, 0);
      lastSequence = descriptor.sequence;
      return;
    }
    if (descriptor.codec !== 'h264') return;
    if (descriptor.codec_epoch !== codecEpoch) {
      if (!descriptor.keyframe) { requestKeyframe(); return; }
      configureDecoder(descriptor.codec_epoch, bytes);
    }
    // Sequences start anywhere; a gap means frames were replaced upstream.
    if (lastSequence && descriptor.sequence > lastSequence + 1 && !descriptor.keyframe) awaitingKeyframe = true;
    if (awaitingKeyframe && !descriptor.keyframe) { requestKeyframe(); return; }
    if (videoDecoder.decodeQueueSize > 4 && !descriptor.keyframe) {   // decoder overload
      awaitingKeyframe = true; requestKeyframe(); return;
    }
    if (descriptor.keyframe) awaitingKeyframe = false;
    lastSequence = descriptor.sequence;
    sequenceByTimestamp.set(descriptor.capture_timestamp_us, descriptor.sequence);
    if (sequenceByTimestamp.size > 256) sequenceByTimestamp.delete(sequenceByTimestamp.keys().next().value);
    videoDecoder.decode(new EncodedVideoChunk({
      type: descriptor.keyframe ? 'key' : 'delta',
      timestamp: descriptor.capture_timestamp_us,
      data: bytes
    }));
  }

  // ---------------------------------------------------------------- audio
  let audioContext = null;
  let audioEnabled = false;
  const tracks = new Map();   // track_id -> { config, decoder, playAt, lastSequence }
  const JITTER_TARGET = 0.04; // seconds (MEDIA.md: 20-60 ms)

  function ensureAudioContext() {
    if (!audioContext) audioContext = new AudioContext({ sampleRate: 48000, latencyHint: 'interactive' });
    if (audioContext.state === 'suspended') audioContext.resume();
  }

  function schedule(track, channels, sampleRate) {
    if (!audioEnabled || !audioContext) return;
    const frames = channels[0].length;
    const buffer = audioContext.createBuffer(channels.length, frames, sampleRate);
    channels.forEach((samples, channel) => buffer.copyToChannel(samples, channel));
    const source = audioContext.createBufferSource();
    source.buffer = buffer;
    source.connect(audioContext.destination);
    const now = audioContext.currentTime;
    // Late or first packet: re-anchor at now + target; never play late audio.
    if (track.playAt < now || track.playAt > now + 0.3) track.playAt = now + JITTER_TARGET;
    source.start(track.playAt);
    track.playAt += frames / sampleRate;
  }

  function audioConfig(config) {
    const previous = tracks.get(config.track_id);
    if (previous && previous.decoder && previous.decoder.state !== 'closed') previous.decoder.close();
    const track = { config, decoder: null, playAt: 0, lastSequence: null };
    if (config.codec === 'opus' && 'AudioDecoder' in window) {
      track.decoder = new AudioDecoder({
        output(data) {
          const channels = [];
          for (let channel = 0; channel < data.numberOfChannels; channel++) {
            const samples = new Float32Array(data.numberOfFrames);
            data.copyTo(samples, { planeIndex: channel, format: 'f32-planar' });
            channels.push(samples);
          }
          schedule(track, channels, data.sampleRate);
          data.close();
        },
        error(error) { log(`audio decoder: ${error.message}`); }
      });
      track.decoder.configure({ codec: 'opus', sampleRate: config.sample_rate_hz, numberOfChannels: config.channels });
    }
    tracks.set(config.track_id, track);
  }

  function handleAudio(buffer) {
    const view = new DataView(buffer);
    if (view.byteLength < 24 || view.getUint8(4) !== 2) return;
    const trackId = view.getUint16(6, false);
    const sequence = view.getUint32(8, false);
    const pts = Number(view.getBigUint64(12, false));
    const epoch = view.getUint8(22);
    const track = tracks.get(trackId);
    if (!track || track.config.config_epoch !== epoch) return;
    counters.audioPackets++;
    if (track.lastSequence !== null && ((sequence - track.lastSequence) >>> 0) > 1) {
      counters.audioLost += ((sequence - track.lastSequence) >>> 0) - 1;
    }
    track.lastSequence = sequence;
    const payload = new Uint8Array(buffer, 24);
    if (track.config.codec === 'pcm_s16le') {
      const count = track.config.channels;
      const frames = payload.byteLength / 2 / count;
      const samples = new DataView(buffer, 24);
      const channels = Array.from({ length: count }, () => new Float32Array(frames));
      for (let frame = 0; frame < frames; frame++) {
        for (let channel = 0; channel < count; channel++) {
          channels[channel][frame] = samples.getInt16((frame * count + channel) * 2, true) / 32768;
        }
      }
      schedule(track, channels, track.config.sample_rate_hz);
    } else if (track.decoder) {
      track.decoder.decode(new EncodedAudioChunk({ type: 'key', timestamp: pts, data: payload }));
    }
  }

  // ---------------------------------------------------------------- control
  function handleControl(message) {
    const { type, payload } = message;
    if (type === 'hello') {
      log(`hello: wire v${payload.selected_version} [${payload.capabilities.join(', ')}]`);
    } else if (type === 'session_opened') {
      session = payload;
      geometry = payload.geometry;
      geometryEpoch = payload.geometry_epoch;
      codecEpoch = -1; lastSequence = 0; awaitingKeyframe = true;
      const target = payload.target.kind === 'display' ? `display ${payload.target.display_id}` : `window ${payload.target.handle}`;
      status(`Streaming ${target} (${payload.codec}, policy ${payload.policy})`);
    } else if (type === 'audio_config') {
      audioConfig(payload);
      log(`audio track ${payload.track_id}: ${payload.codec} ${payload.sample_rate_hz} Hz x${payload.channels}`);
    } else if (type === 'lifecycle') {
      if (payload.event.kind === 'geometry_changed') {
        geometry = payload.event.geometry;
        geometryEpoch = payload.event.geometry_epoch;
      }
      if (payload.event.kind === 'closed') status('Target closed');
      log(message);
    } else if (type === 'error') {
      if (payload.code === 'input_sequence_gap' && payload.expected_sequence) inputSequence = payload.expected_sequence;
      log(message);
    } else if (type === 'stats') {
      $('stats').textContent =
        `frames since attach ${payload.frames_since_attach}  keyframes ${payload.keyframes_sent}\n` +
        `dropped ${payload.frames_dropped}  idle ${payload.target_idle}  age ${payload.last_frame_age_ms ?? '-'} ms\n` +
        `encoder ${payload.encoder ?? '-'} @ ${payload.encoder_bitrate_kbps ?? '-'} kbps, ${payload.effective_max_fps} fps\n` +
        `ack rtt ${payload.ack_rtt_ms ? payload.ack_rtt_ms.toFixed(1) : '-'} ms  audio ${counters.audioPackets} pkts, ${counters.audioLost} lost`;
    } else if (type !== 'interactive_input_acknowledgement' && type !== 'pong') {
      log(message);
    }
  }

  function handleBinary(buffer) {
    const bytes = new Uint8Array(buffer);
    if (bytes[0] === 0x52 && bytes[1] === 0x41 && bytes[2] === 0x55 && bytes[3] === 0x32) { handleAudio(buffer); return; }
    const view = new DataView(buffer);
    const headerLength = view.getUint32(0, false);
    const payloadLength = view.getUint32(4, false);
    if (8 + headerLength + payloadLength !== view.byteLength) throw new Error('invalid binary lengths');
    const header = JSON.parse(text.decode(new Uint8Array(buffer, 8, headerLength)));
    if (header.direction !== 'video') return;
    handleVideo(header.message, new Uint8Array(buffer, 8 + headerLength, payloadLength))
      .catch(error => { status(error.message); log(error.stack || error.message); });
  }

  // ---------------------------------------------------------------- input
  let pending = [];
  function queueInput(event) {
    pending.push(event);
    if (pending.length === 1) queueMicrotask(flushInput);
  }
  function flushInput() {
    if (!pending.length || !session) { pending = []; return; }
    const events = pending.splice(0, 256);
    send('interactive_input', { session_id: sessionId(), first_sequence: inputSequence, events });
    inputSequence += events.length;
  }
  const normalized = event => {
    const rect = canvas.getBoundingClientRect();
    return {
      x_normalized: Math.min(1, Math.max(0, (event.clientX - rect.left) / rect.width)),
      y_normalized: Math.min(1, Math.max(0, (event.clientY - rect.top) / rect.height))
    };
  };
  const modifiers = event => [
    event.shiftKey && 'shift', event.ctrlKey && 'control', event.altKey && 'option', event.metaKey && 'command'
  ].filter(Boolean);
  const buttons = ['left', 'middle', 'right'];
  let lastMove = 0;
  canvas.addEventListener('pointermove', event => {
    const now = performance.now();
    if (now - lastMove < 16) return;
    lastMove = now;
    queueInput({ kind: 'pointer', phase: 'move', button: null, ...normalized(event), modifiers: modifiers(event) });
  });
  canvas.addEventListener('pointerdown', event => {
    canvas.focus();
    queueInput({ kind: 'pointer', phase: 'down', button: buttons[event.button] || 'left', ...normalized(event), modifiers: modifiers(event) });
  });
  canvas.addEventListener('pointerup', event => {
    queueInput({ kind: 'pointer', phase: 'up', button: buttons[event.button] || 'left', ...normalized(event), modifiers: modifiers(event) });
  });
  canvas.addEventListener('contextmenu', event => event.preventDefault());
  canvas.addEventListener('wheel', event => {
    event.preventDefault();
    queueInput({
      kind: 'scroll', ...normalized(event), delta_x: event.deltaX, delta_y: event.deltaY,
      phase: 'none', momentum_phase: 'none', precise: event.deltaMode === 0
    });
  }, { passive: false });
  const keyName = event => (event.key.length === 1 ? event.key.toLowerCase() : event.key.toLowerCase());
  canvas.addEventListener('keydown', event => {
    event.preventDefault();
    if (event.key.length === 1 && !event.ctrlKey && !event.metaKey && !event.altKey) {
      queueInput({ kind: 'text_commit', text: event.key });
    } else {
      queueInput({ kind: 'key', key: keyName(event), state: 'down', modifiers: modifiers(event), repeat: event.repeat });
    }
  });
  canvas.addEventListener('keyup', event => {
    event.preventDefault();
    if (event.key.length === 1 && !event.ctrlKey && !event.metaKey && !event.altKey) return;
    queueInput({ kind: 'key', key: keyName(event), state: 'up', modifiers: modifiers(event), repeat: false });
  });

  // ---------------------------------------------------------------- wiring
  let statsTimer = null;
  $('connect').onclick = () => {
    if (socket) socket.close();
    session = null;
    status('Connecting…');
    // The ticket authenticates the upgrade; offering rcdp.v2 lets the
    // server confirm the wire version.
    socket = new WebSocket($('url').value, ['rcdp.v2']);
    socket.binaryType = 'arraybuffer';
    socket.onopen = () => status('Attached; waiting for hello…');
    socket.onmessage = event => {
      try {
        if (typeof event.data === 'string') handleControl(JSON.parse(event.data));
        else handleBinary(event.data);
      } catch (error) {
        status(error.message);
        log(error.stack || error.message);
      }
    };
    socket.onclose = event => {
      clearInterval(statsTimer);
      const reasons = { 4401: 'ticket invalid or token rotated', 4404: 'session closed', 4408: 'idle timeout', 4410: 'target gone', 4429: 'too many sockets' };
      status(`Disconnected (${event.code}${reasons[event.code] ? `: ${reasons[event.code]}` : ''})`);
    };
    socket.onerror = () => status('WebSocket error (401: bad or expired ticket, 410: session closed)');
    clearInterval(statsTimer);
    statsTimer = setInterval(() => send('get_stats', { session_id: sessionId() }), 2000);
  };
  $('keyframe').onclick = () => { lastKeyframeRequest = 0; requestKeyframe(); };
  $('mute').onclick = () => {
    audioEnabled = !audioEnabled;
    if (audioEnabled) ensureAudioContext();   // needs this user gesture
    $('mute').textContent = `Audio: ${audioEnabled ? 'on' : 'off'}`;
  };
  $('state').onclick = () => send('get_window_state', { session_id: sessionId() });
  $('type').onclick = () => {
    if ($('text').value) queueInput({ kind: 'text_commit', text: $('text').value });
  };
  if (params.get('media')) $('connect').click();
})();
</script>
</body>
</html>