Open-source AI browser agent for Chrome and Firefox (monorepo) ðŸ§
JavaScript ai-agentai-agentsai-toolsbrowser-aibrowser-automationbrowser-extensionchrome-aichrome-devtoolschrome-extensionchrome-extensionsfirefox-addonfirefox-addonsfirefox-extensionfirefox-extensionsgemmagemma4minimaxoffline-aioffline-llmqwen
{
"generatedAt": "2026-10-29T10:14:00Z",
"environment": {
"hardware": "Dual NVIDIA GeForce RTX 5060 Ti",
"runtime": "llama.cpp b0-unknown (llama-server)",
"concurrency": 2,
"benchmark": "WebBrain 100-case browser viewport vision suite",
"promptMode": "production"
},
"compactModelComparison": [
{
"id": "qwen3.5-4b",
"label": "Qwen 3.5 4B (MLX)",
"parameters": "4B",
"format": "MLX 4-bit",
"strictPasses": 55,
"meanRubric": 0.834,
"latencyMs": 25917,
"errors": 0,
"notes": "Strongest general-purpose model in the compact table"
},
{
"id": "webbrain-vl-2-450m",
"label": "WebBrain VL 2 450M (ONNX)",
"parameters": "453M",
"format": "ONNX / WebGPU",
"strictPasses": 44,
"meanRubric": 0.762,
"latencyMs": 1998,
"errors": 0,
"notes": "Fine-tuned specialist; WebGPU browser extension default"
},
{
"id": "gemma-4-e4b-it",
"label": "Gemma 4 E4B IT (MLX)",
"parameters": "E4B",
"format": "MLX 4-bit",
"strictPasses": 41,
"meanRubric": 0.813,
"latencyMs": 5956,
"errors": 0,
"notes": "Strong partial coverage, high mean rubric"
},
{
"id": "qwen3.5-2b",
"label": "Qwen 3.5 2B (MLX)",
"parameters": "2B",
"format": "MLX 4-bit",
"strictPasses": 39,
"meanRubric": 0.791,
"latencyMs": 10010,
"errors": 0,
"notes": "Solid 2B general-purpose baseline"
},
{
"id": "webbrain-vl-3-450m",
"label": "WebBrain VL 3 450M (Winner 3750)",
"parameters": "453M",
"format": "BF16",
"strictPasses": 26,
"meanRubric": 0.693,
"latencyMs": 3577,
"errors": 0,
"notes": "Sub-1B browser specialist checkpoint"
},
{
"id": "qwen3.5-0.8b",
"label": "Qwen 3.5 0.8B",
"parameters": "0.8B",
"format": "GGUF",
"strictPasses": 20,
"meanRubric": 0.416,
"latencyMs": 7350,
"errors": 0,
"notes": "Capable off-the-shelf sub-1B baseline"
},
{
"id": "lfm2.5-vl-1.6b",
"label": "LFM2.5-VL-1.6B",
"parameters": "1.6B",
"format": "MLX",
"strictPasses": 12,
"meanRubric": 0.444,
"latencyMs": 809,
"errors": 0,
"notes": "Off-the-shelf Liquid foundation model"
},
{
"id": "lfm3-vld-2b",
"label": "LFM3-VLD-2B (New Q4 GGUF)",
"parameters": "2B",
"format": "Q4_K_M GGUF (1.25 GB)",
"strictPasses": 4,
"meanRubric": 0.447,
"latencyMs": 2200,
"errors": 0,
"hardware": "NVIDIA RTX 5060 Ti",
"notes": "Perfect 6-section adherence; tool routing decision specialist"
},
{
"id": "lfm2.5-vl-450m",
"label": "LFM2.5-VL-450M",
"parameters": "450M",
"format": "MLX",
"strictPasses": 1,
"meanRubric": 0.216,
"latencyMs": 436,
"errors": 0,
"notes": "Base starting point before fine-tuning"
},
{
"id": "lfm2.5-vl-3b",
"label": "LFM2.5-VL-3B",
"parameters": "3B",
"format": "MLX",
"strictPasses": 0,
"meanRubric": 0.078,
"latencyMs": 1930,
"errors": 0,
"notes": "Formatting collapse; failed numbered section contract"
},
{
"id": "minicpm-v-4.7-1b",
"label": "MiniCPM-V 4.7 1B (New Q4 GGUF)",
"parameters": "1B",
"format": "Q4_K_M GGUF (641 MB)",
"strictPasses": 0,
"meanRubric": 0.094,
"latencyMs": 1299,
"errors": 0,
"hardware": "NVIDIA RTX 5060 Ti",
"notes": "Strict contract fail: emitted unnumbered headers; scores 37 strict (68.2%) with normalized headers"
}
],
"lfm3_vld_2b_summary": {
"schemaVersion": 1,
"createdAt": "2026-10-29T10:12:16.657Z",
"model": "webbrain-one/lfm3-vld-2b",
"endpoint": "http://127.0.0.1:18080/v1/chat/completions",
"promptMode": "production",
"productionUserText": "Describe this screenshot of the current browser viewport for a web-automation agent. Follow the format in the system prompt.",
"promptSha256": "8465a0e60af8f3e4dcc5ff188a7682c22b1d52974db54ccb42ff8a1a5346a207",
"overall": {
"cases": 100,
"scored": 100,
"errors": 0,
"successes": 4,
"successRate": 0.04,
"meanScore": 0.4470201002701004,
"meanLatencyMs": 2199.8
},
"byDifficulty": {
"easy": {
"cases": 20,
"scored": 20,
"errors": 0,
"successes": 2,
"successRate": 0.1,
"meanScore": 0.5120418470418471,
"meanLatencyMs": 1700.5
},
"basic": {
"cases": 20,
"scored": 20,
"errors": 0,
"successes": 2,
"successRate": 0.1,
"meanScore": 0.5063274225774226,
"meanLatencyMs": 2054.05
},
"intermediate": {
"cases": 20,
"scored": 20,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.4455548155548156,
"meanLatencyMs": 2043.95
},
"advanced": {
"cases": 20,
"scored": 20,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.3229453416953417,
"meanLatencyMs": 3097.5
},
"challenging": {
"cases": 20,
"scored": 20,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.44823107448107447,
"meanLatencyMs": 2103
}
},
"byCategory": {
"authentication": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.3855710955710956,
"meanLatencyMs": 1272.6
},
"search-results": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.45757575757575764,
"meanLatencyMs": 1064
},
"checkout": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.2843822843822844,
"meanLatencyMs": 1999.6
},
"form-validation": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.25999999999999995,
"meanLatencyMs": 1785.6
},
"modal-overlay": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.6181818181818182,
"meanLatencyMs": 1209
},
"toast-notification": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.41798941798941797,
"meanLatencyMs": 3571.4
},
"loading-state": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.570959595959596,
"meanLatencyMs": 1523.8
},
"consent-banner": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 1,
"successRate": 0.2,
"meanScore": 0.34,
"meanLatencyMs": 2755
},
"dashboard": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.40499999999999997,
"meanLatencyMs": 2672
},
"chart-reading": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.475,
"meanLatencyMs": 768.4
},
"data-table": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 1,
"successRate": 0.2,
"meanScore": 0.5333333333333333,
"meanLatencyMs": 3031.6
},
"email-compose": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.4416666666666666,
"meanLatencyMs": 1908
},
"kanban-board": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.3904761904761905,
"meanLatencyMs": 4522
},
"calendar": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.4095238095238095,
"meanLatencyMs": 678.4
},
"map-and-travel": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 1,
"successRate": 0.2,
"meanScore": 0.5816666666666667,
"meanLatencyMs": 2899.2
},
"photo-understanding": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 1,
"successRate": 0.2,
"meanScore": 0.6222222222222222,
"meanLatencyMs": 943.4
},
"multilingual-ocr": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.32727272727272727,
"meanLatencyMs": 5366.2
},
"occlusion-and-contrast": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.40666666666666673,
"meanLatencyMs": 1636.4
},
"security-challenge": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.4195804195804196,
"meanLatencyMs": 2222
},
"uncertainty-calibration": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.5933333333333334,
"meanLatencyMs": 2167.4
}
}
},
"minicpm_v_47_1b_summary": {
"schemaVersion": 1,
"createdAt": "2026-10-29T10:11:40.073Z",
"model": "openbmb/MiniCPM-V-4.7-1B",
"endpoint": "http://127.0.0.1:18081/v1/chat/completions",
"promptMode": "production",
"productionUserText": "Describe this screenshot of the current browser viewport for a web-automation agent. Follow the format in the system prompt.",
"promptSha256": "8465a0e60af8f3e4dcc5ff188a7682c22b1d52974db54ccb42ff8a1a5346a207",
"overall": {
"cases": 100,
"scored": 100,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.09426587301587301,
"meanLatencyMs": 1298.99
},
"byDifficulty": {
"easy": {
"cases": 20,
"scored": 20,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.11914682539682539,
"meanLatencyMs": 1249.25
},
"basic": {
"cases": 20,
"scored": 20,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.10271825396825399,
"meanLatencyMs": 1283.65
},
"intermediate": {
"cases": 20,
"scored": 20,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.09771825396825398,
"meanLatencyMs": 1284.35
},
"advanced": {
"cases": 20,
"scored": 20,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.04503968253968254,
"meanLatencyMs": 1354.3
},
"challenging": {
"cases": 20,
"scored": 20,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.10670634920634921,
"meanLatencyMs": 1323.4
}
},
"byCategory": {
"authentication": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0,
"meanLatencyMs": 1326.4
},
"search-results": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.016666666666666666,
"meanLatencyMs": 1420.2
},
"checkout": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0,
"meanLatencyMs": 1420.8
},
"form-validation": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0,
"meanLatencyMs": 1510.4
},
"modal-overlay": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0,
"meanLatencyMs": 1047.8
},
"toast-notification": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0,
"meanLatencyMs": 1437.4
},
"loading-state": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0,
"meanLatencyMs": 1023.2
},
"consent-banner": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0,
"meanLatencyMs": 1541.4
},
"dashboard": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0,
"meanLatencyMs": 1188
},
"chart-reading": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.45,
"meanLatencyMs": 1243.8
},
"data-table": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.34285714285714286,
"meanLatencyMs": 1402.8
},
"email-compose": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0,
"meanLatencyMs": 1385
},
"kanban-board": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.2571428571428571,
"meanLatencyMs": 1152.8
},
"calendar": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.23809523809523808,
"meanLatencyMs": 1404
},
"map-and-travel": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.3383333333333333,
"meanLatencyMs": 1269.6
},
"photo-understanding": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.22222222222222224,
"meanLatencyMs": 1170.2
},
"multilingual-ocr": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0,
"meanLatencyMs": 1340.4
},
"occlusion-and-contrast": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0,
"meanLatencyMs": 1255.8
},
"security-challenge": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0,
"meanLatencyMs": 1306.8
},
"uncertainty-calibration": {
"cases": 5,
"scored": 5,
"errors": 0,
"successes": 0,
"successRate": 0,
"meanScore": 0.02,
"meanLatencyMs": 1133
}
}
},
"minicpm_v_47_normalized": {
"strictPasses": 37,
"meanRubric": 0.682,
"explanation": "When evaluating with header regex recognizing unnumbered Section names (Page purpose:, Visible text:, Inputs:, State signals:, Blockers:, Unknowns:)"
}
}