Skip to content

Open-source AI browser agent for Chrome and Firefox (monorepo) 🧠

15.0 KB
Raw Blame
// Long-prompt check for in-browser WebGPU models through the production worker.
//
// The short prompts in check-bonsai-webgpu.mjs cannot reveal prefill problems
// (a CPU-fallback recurrence or full-sequence logits only show up with
// thousands of prompt tokens). This runs real text of several lengths (and,
// for vision presets, a long prompt that includes an image) through
// src/chrome/src/offscreen/inference-worker.js and reports wall time per
// prompt and seconds per 1k prompt tokens. Software (fallback) WebGPU adapters
// are rejected. See docs/webgpu-models.md.
//
//   node scripts/check-webgpu-long-prompts.mjs --model webbrain-one/Ternary-Bonsai-2-2B-ONNX-WebGPU-Vision \
//     --profile .build/bonsai/chunked-2b-profile --offline --lengths 1000,4000,8000 --slices 0,512
//
// It fails when:
// - a text prompt of at least 4k tokens is slower than --max-seconds-per-1k
//   (default 10; a CPU fallback costs 100 or more);
// - for vision presets, the answer with a sliced prefill differs from the first
//   --slices value within its first --min-common-words words (default 8). A
//   broken cache handoff loses the prompt and diverges at once, while FP16
//   rounding differences between slice sizes can change a near-tied token later
//   in a long greedy answer; the common prefix is reported either way;
// - the run aborts (busy port, failed launch or probe, worker error, timeout).
//
// The model's production path and dtype come from
// src/chrome/src/providers/webgpu.js. Shipped `onnx-vl` presets run multimodal
// chat, which the worker accepts only for them; prefill slicing applies to the
// Bonsai 2 Vision presets. Other ONNX presets and custom repositories run text
// chat with their preset dtype (q4f16 by default) once per prompt, for timing
// only: the text path does not slice and some text presets sample. --vision is
// accepted only for onnx-vl presets, --dtype (a name or a JSON object)
// overrides the dtype, and the `bitgpu` runtime is rejected because it does
// not use this worker.
//
// Options mirror check-bonsai-webgpu.mjs: --local-repo owner/repo=dir serves
// unpublished files and clears the tested model's cached files first, so a
// rebuilt export is not masked by an earlier copy; --clear-model-cache does
// the same for a published model; --offline requires a cached model and
// blocks Hugging Face; --adapter-luid high,low (decimal) selects a Windows
// GPU; and --timeout-ms sets the deadline for worker start-up and each
// operation (default 30 minutes).
import assert from 'node:assert/strict';
import fs from 'node:fs/promises';
import { createReadStream } from 'node:fs';
import http from 'node:http';
import path from 'node:path';
import { fileURLToPath } from 'node:url';
import { chromium } from 'playwright';
import {
  WEBGPU_RUNTIME_BITGPU, WEBGPU_RUNTIME_ONNX_VL, webgpuModelDtype, webgpuModelRuntime,
} from '../src/chrome/src/providers/webgpu.js';

const root = path.resolve(path.dirname(fileURLToPath(import.meta.url)), '..');
const args = process.argv.slice(2);
const option = (name, fallback) => { const i = args.indexOf(name); return i < 0 ? fallback : args[i + 1]; };
const model = option('--model', 'webbrain-one/Ternary-Bonsai-2-2B-ONNX-WebGPU-Vision');
const runtime = webgpuModelRuntime(model);
assert(runtime !== WEBGPU_RUNTIME_BITGPU, `${model} uses the ${runtime} runtime, which this ONNX worker check does not cover`);
// As in production, onnx-vl presets always use the multimodal path. The
// worker's multimodal chat accepts only those shipped presets, so --vision
// cannot select it for another repository.
assert(!args.includes('--vision') || runtime === WEBGPU_RUNTIME_ONNX_VL,
  `${model} is not a shipped onnx-vl preset; the worker's multimodal chat only runs those`);
const vision = runtime === WEBGPU_RUNTIME_ONNX_VL;
const offline = args.includes('--offline');
// A rebuilt local export reuses the model ID, and the worker keeps an already
// cached copy, so serving the model from --local-repo clears its cache first.
const clearModelCacheRequested = args.includes('--clear-model-cache');
assert(!(clearModelCacheRequested && offline), '--clear-model-cache cannot be used with --offline');
const parseDtype = value => { try { return JSON.parse(value); } catch { return value; } };
const dtype = option('--dtype', null) ? parseDtype(option('--dtype')) : webgpuModelDtype(model);
const lengths = option('--lengths', '1000,4000,8000').split(',').map(Number);
const slices = option('--slices', '0,512').split(',').map(Number);
const maxTokens = Number(option('--max-tokens', 48));
const maxSecondsPer1k = Number(option('--max-seconds-per-1k', 10));
const minCommonWords = Number(option('--min-common-words', 8));
const timeout = Number(option('--timeout-ms', 30 * 60_000));
const profile = path.resolve(option('--profile', '.build/webgpu-long-prompts/profile'));
const output = path.resolve(option('--output', '.build/webgpu-long-prompts/result.json'));
const localRepos = {};
for (let i = 0; i < args.length; i++) {
  if (args[i] !== '--local-repo') continue;
  const entry = args[++i] || '';
  const split = entry.indexOf('=');
  assert(split > 0, '--local-repo requires owner/repo=directory');
  localRepos[entry.slice(0, split)] = path.resolve(entry.slice(split + 1));
}
const mime = file => ({ '.mjs': 'text/javascript', '.js': 'text/javascript', '.json': 'application/json', '.wasm': 'application/wasm' }[path.extname(file)] || 'application/octet-stream');
const workerBootstrap = `
const nativeFetch = globalThis.fetch.bind(globalThis);
const mapped = ${JSON.stringify(Object.keys(localRepos))};
globalThis.fetch = (input, init) => {
  const url = new URL(typeof input === 'string' ? input : input.url || String(input), self.location.href);
  if (url.hostname === 'huggingface.co') {
    if (${offline}) throw new Error('Offline check blocked a Hugging Face fetch: ' + url.pathname);
    const match = url.pathname.match(/^\\/([^/]+\\/[^/]+)\\/resolve\\/[^/]+\\/(.+)$/);
    if (match && mapped.includes(match[1])) return nativeFetch('/models/' + match[1] + '/' + match[2], init);
  }
  return nativeFetch(input, init);
};
await import('/extension/src/offscreen/inference-worker.js');
self.postMessage({ type: 'harness-ready' });`;
const server = http.createServer(async (request, response) => {
  try {
    const pathname = decodeURIComponent(new URL(request.url, 'http://localhost').pathname);
    if (pathname === '/') { response.setHeader('Content-Type', 'text/html'); response.end('<!doctype html><title>WebGPU long prompts</title>'); return; }
    if (pathname === '/worker.mjs') { response.setHeader('Content-Type', 'text/javascript'); response.end(workerBootstrap); return; }
    let base, relative;
    if (pathname.startsWith('/extension/')) { base = path.join(root, 'src/chrome'); relative = pathname.slice('/extension/'.length); }
    else if (pathname.startsWith('/models/')) {
      const match = pathname.match(/^\/models\/([^/]+\/[^/]+)\/(.+)$/);
      base = match && localRepos[match[1]]; relative = match?.[2];
    }
    if (!base || !relative) { response.writeHead(404); response.end(); return; }
    const file = path.resolve(base, relative);
    if (!file.startsWith(path.resolve(base) + path.sep)) { response.writeHead(403); response.end(); return; }
    const stat = await fs.stat(file);
    response.setHeader('Content-Type', mime(file)); response.setHeader('Content-Length', stat.size);
    createReadStream(file).on('error', () => response.destroy()).pipe(response);
  } catch (error) {
    response.writeHead(error.code === 'ENOENT' ? 404 : 500); response.end();
  }
});
await fs.mkdir(path.dirname(output), { recursive: true });
const words = text => String(text || '').trim().split(/\s+/).filter(Boolean);
const commonPrefixWords = (a, b) => {
  const left = words(a), right = words(b);
  let count = 0;
  while (count < left.length && count < right.length && left[count] === right[count]) count++;
  return count;
};
const result = {
  model, vision, dtype, maxSecondsPer1k, minCommonWords, startedAt: new Date().toISOString(),
  runs: [], divergences: [], failures: [],
};
let context;
try {
  // Inside the guarded block so a busy port is recorded as a failed run too.
  await new Promise((resolve, reject) => { server.once('error', reject); server.listen(Number(option('--port', 8769)), '127.0.0.1', resolve); });
  context = await chromium.launchPersistentContext(profile, {
    headless: true,
    channel: 'chromium',
    args: ['--enable-unsafe-webgpu', '--force-high-performance-gpu',
      ...(option('--adapter-luid', '') ? [`--use-adapter-luid=${option('--adapter-luid')}`] : [])],
  });
  const page = await context.newPage();
  page.on('console', message => { if (message.type() === 'error') console.log(`[browser:error] ${message.text().slice(0, 300)}`); });
  await page.goto(`http://127.0.0.1:${server.address().port}/`);
  await page.evaluate(timeout => new Promise((ready, failed) => {
    const pending = new Map(); let id = 0;
    const startTimer = setTimeout(() => failed(new Error('Timeout: worker start')), timeout);
    const worker = new Worker('/worker.mjs', { type: 'module' });
    worker.onmessage = ({ data }) => {
      if (data.type === 'harness-ready') { clearTimeout(startTimer); ready(); }
      else if (data.id && pending.has(data.id)) {
        const { resolve, reject, timer } = pending.get(data.id);
        clearTimeout(timer); pending.delete(data.id);
        if (data.ok === false) reject(new Error(data.error)); else resolve(data);
      }
    };
    worker.onerror = event => {
      clearTimeout(startTimer);
      failed(new Error(event.message));
      for (const { reject, timer } of pending.values()) { clearTimeout(timer); reject(new Error(event.message)); }
      pending.clear();
    };
    window.workerCall = (type, payload, timeout) => new Promise((resolve, reject) => {
      const requestId = ++id;
      const timer = setTimeout(() => { pending.delete(requestId); reject(new Error(`Timeout: ${type}`)); }, timeout);
      pending.set(requestId, { resolve, reject, timer });
      worker.postMessage({ id: requestId, type, payload });
    });
  }), timeout);
  const origin = new URL(page.url()).origin;
  const call = (type, payload = {}) => page.evaluate(({ type, payload, timeout }) => window.workerCall(type, payload, timeout), { type, payload, timeout });
  await call('init', {
    transformersUrl: `${origin}/extension/vendor/transformers/transformers.web.js`,
    wasmMjsUrl: `${origin}/extension/vendor/transformers/ort-wasm-simd-threaded.asyncify.mjs`,
    wasmUrl: `${origin}/extension/vendor/transformers/ort-wasm-simd-threaded.asyncify.wasm`,
  });
  result.probe = await call('probe');
  assert(result.probe.hasWebGPU, 'Hardware WebGPU required');
  assert(!result.probe.isFallbackAdapter, 'Software WebGPU cannot validate prefill speed');
  result.adapter = await page.evaluate(async () => {
    const adapter = await navigator.gpu.requestAdapter({ powerPreference: 'high-performance' });
    return { vendor: adapter?.info.vendor, architecture: adapter?.info.architecture };
  });
  result.clearedModelCache = !offline && (clearModelCacheRequested || Object.hasOwn(localRepos, model));
  if (result.clearedModelCache) await call('stop-text-download', { modelId: model, dtype });
  const download = await call(offline ? 'text-download-status' : 'download-text', { modelId: model, dtype, device: 'webgpu' });
  assert(download.ready, `${model} is not ready: ${JSON.stringify(download)}`);
  const corpus = await fs.readFile(path.join(root, 'CHANGELOG.md'), 'utf8');
  // About 3.3 characters per token for this English changelog.
  const textFor = tokens => corpus.slice(0, Math.round(tokens * 3.3));
  const prompts = lengths.map(tokens => ({
    name: `text-${tokens}`, tokens,
    messages: [{ role: 'user', content: `${textFor(tokens)}\n\nIn one sentence, what is the most recent change above?` }],
  }));
  if (vision) {
    const image = await page.evaluate(() => {
      const canvas = document.createElement('canvas'); canvas.width = 448; canvas.height = 448;
      const ctx = canvas.getContext('2d'); ctx.fillStyle = 'white'; ctx.fillRect(0, 0, 448, 448);
      ctx.fillStyle = 'red'; ctx.fillRect(64, 40, 320, 180);
      return canvas.toDataURL('image/png');
    });
    prompts.push({ name: `image-text-${lengths[0]}`, tokens: lengths[0], messages: [{ role: 'user', content: [
      { type: 'text', text: textFor(lengths[0]) },
      { type: 'image_url', image_url: { url: image } },
      { type: 'text', text: 'Ignore the release notes. What color is the large shape in the image, and what is its shape?' },
    ] }] });
  }
  const type = vision ? 'multimodal-text-chat' : 'text-chat';
  await call(type, { modelId: model, dtype, device: 'webgpu', messages: [{ role: 'user', content: 'Hi' }], options: { maxTokens: 4 } });
  // Only the vision path slices its prefill and decodes greedily.
  const runSlices = vision ? slices : [null];
  for (const prompt of prompts) {
    const answers = new Map();
    for (const slice of runSlices) {
      const started = Date.now();
      const options = { maxTokens, ...(slice == null ? {} : { prefillSliceTokens: slice }) };
      const reply = await call(type, { modelId: model, dtype, device: 'webgpu', messages: prompt.messages, options });
      const seconds = (Date.now() - started) / 1000;
      const run = { prompt: prompt.name, slice, seconds, secondsPer1kPromptTokens: +(seconds / (prompt.tokens / 1000)).toFixed(3), content: reply.content };
      result.runs.push(run);
      answers.set(slice, reply.content);
      console.log(JSON.stringify(run).slice(0, 300));
      if (prompt.name.startsWith('text-') && prompt.tokens >= 4000 && run.secondsPer1kPromptTokens > maxSecondsPer1k) {
        result.failures.push({ prompt: prompt.name, slice, check: 'prefill-speed', secondsPer1kPromptTokens: run.secondsPer1kPromptTokens });
      }
    }
    const reference = answers.get(runSlices[0]);
    for (const [slice, content] of answers) {
      if (content === reference) continue;
      const common = commonPrefixWords(reference, content);
      const divergence = { prompt: prompt.name, slice, reference: runSlices[0], commonPrefixWords: common, referenceWords: words(reference).length };
      result.divergences.push(divergence);
      if (common < Math.min(minCommonWords, words(reference).length)) result.failures.push({ ...divergence, check: 'early-divergence' });
    }
  }
  result.passed = result.failures.length === 0;
} catch (error) {
  // A timeout, worker error or failed probe aborts the run; record it as a failure.
  result.passed = false;
  result.error = String(error?.stack || error);
  result.failures.push({ check: 'run-error', error: String(error?.message || error) });
  console.error(error);
} finally {
  result.finishedAt = new Date().toISOString();
  await fs.writeFile(output, JSON.stringify(result, null, 2));
  await context?.close().catch(() => {});
  if (server.listening) server.close();
}
for (const divergence of result.divergences) console.log(`divergence: ${JSON.stringify(divergence)}`);
for (const failure of result.failures || []) console.log(`failure: ${JSON.stringify(failure)}`);
console.log(`${result.passed ? 'PASSED' : 'FAILED'}: ${output}`);
if (!result.passed) process.exitCode = 1;