The headless browser for AI agents and web scraping
#!/usr/bin/env python3
"""Interleaved latency, CPU, peak-RSS, and result-correctness comparison.
Linux wait4 supplies precise CPU and peak RSS for the timeout-guarded process tree.
"""
from __future__ import annotations
import argparse
import json
import math
import os
import signal
import statistics
import subprocess
import tempfile
import time
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
from pathlib import Path
from threading import Thread
ROUNDS = 5
PROCESS_TIMEOUT_SECONDS = 45
LATENCY_FAILURE_RATIO = 1.10
CPU_FAILURE_RATIO = 1.10
CPU_FAILURE_DELTA_SECONDS = 0.02
LATENCY_FAILURE_DELTA_MS = 10.0
RSS_FAILURE_RATIO = 1.15
RSS_FAILURE_DELTA_KIB = 8 * 1024
HTML = b"<!doctype html><html><body><main id='root'></main></body></html>"
SCENARIOS = {
"dom-build": (
"(function(){var r=document.getElementById('root');for(var i=0;i<5000;i++){var e=document.createElement('div');e.className='row';e.textContent='item-'+i;r.appendChild(e);}return r.children.length;})()",
5000,
),
"storage": (
"(function(){for(var i=0;i<2000;i++)localStorage.setItem('k'+i,'value-'+i);var n=0;for(var j=0;j<2000;j++)n+=localStorage.getItem('k'+j).length;return n;})()",
18890,
),
"storage-method-read-hot": (
"(() => {const s=localStorage;s.setItem('item','value');let n=0;"
"for(let i=0;i<200000;i++)n+=s.getItem('item').length;return n})()",
1000000,
),
"storage-named-read-hot": (
"(() => {const s=localStorage;s.item='value';let n=0;"
"for(let i=0;i<200000;i++)n+=s.item.length;return n})()",
1000000,
),
"storage-members-hot": (
"(() => {const s=sessionStorage;s.item='value';let n=0;"
"for(let i=0;i<200000;i++)n+=s.length+('item' in s)+('getItem' in s);return n})()",
600000,
),
"storage-named-write-delete-hot": (
"(() => {const s=sessionStorage;let n=0;for(let i=0;i<50000;i++){"
"s.item='value';n+=delete s.item;}return n+s.length})()",
50000,
),
"storage-enumeration-hot": (
"(() => {const s=localStorage;s.alpha='one';s.beta='two';let n=0;"
"for(let i=0;i<20000;i++)n+=Object.keys(s).length+"
"(Object.getOwnPropertyDescriptor(s,'beta').value==='two');return n})()",
60000,
),
"dom-lookup": (
"(() => {let h=document.createElement('div');h.id='probe';document.body.append(h);let n=0;for(let i=0;i<2000000;i++)if(document.getElementById('probe')===h)n++;return n;})()",
2000000,
),
"shadow-build": (
"(() => {let n=0;for(let i=0;i<1500;i++){let h=document.createElement('div');document.body.append(h);let r=h.attachShadow({mode:'open'});let t=document.createElement('span');t.textContent='row';r.append(t);if(h.shadowRoot===r && r.textContent==='row')n++;}return n;})()",
1500,
),
"shadow-lookup": (
"(() => {let h=document.createElement('div');document.body.append(h);let r=h.attachShadow({mode:'open'});let n=0;for(let i=0;i<200000;i++)if(h.shadowRoot===r)n++;return n;})()",
200000,
),
"table-create": (
"(() => {for(let i=0;i<5000;i++)document.body.append(document.createElement('table'));return document.querySelectorAll('table').length})()",
5000,
),
"table-parse-wrap": (
"(() => {document.body.innerHTML='<table><tr><td>row</td></tr></table>'.repeat(5000);return document.querySelectorAll('table').length})()",
5000,
),
"table-rows-hot": (
"(() => {document.body.innerHTML='<table><thead><tr><th>head</th></tr></thead><tbody>'+'<tr><td>body</td></tr>'.repeat(100)+'</tbody></table>';const t=document.querySelector('table'),first=t.rows[0];let n=0;for(let i=0;i<1000000;i++){const rows=t.rows;if(rows[0]===first)n+=rows.length;}return n})()",
101000000,
),
"table-cells-hot": (
"(() => {document.body.innerHTML='<table><tbody>'+'<tr><td>A</td><td>B</td><td>C</td></tr>'.repeat(100)+'</tbody></table>';const row=document.querySelector('tr');let n=0;for(let i=0;i<100000;i++){const cells=row.cells||row.children;n+=cells.length;if(cells[0].textContent==='A')n++;}return n})()",
400000,
),
"select-create-wrap": (
"(() => {document.body.innerHTML='<select><option>A</option><option>B</option></select>'.repeat(5000);return document.querySelectorAll('select').length})()",
5000,
),
"select-create-wrap-50k": (
"(() => {document.body.innerHTML='<select><option>A</option><option>B</option></select>'.repeat(50000);return document.querySelectorAll('select').length})()",
50000,
),
"select-item-or-index-hot": (
"(() => {document.body.innerHTML='<select><option>A</option><option>B</option></select>';const s=document.querySelector('select'),first=s.item(0),indexed='0' in s;let n=0;for(let i=0;i<1000000;i++)if((indexed?s[0]:s.item(0))===first)n++;return n})()",
1000000,
),
"select-existing-getters-hot": (
"(() => {document.body.innerHTML='<select><option>A</option><option>B</option></select>';const s=document.querySelector('select');let n=0;for(let i=0;i<1000000;i++)n+=s.length;return n})()",
2000000,
),
"form-elements-owned": (
"(() => {document.body.innerHTML='<form id=f>'+'<input name=inside>'.repeat(1000)+'</form>';const f=document.getElementById('f');let n=0;for(let i=0;i<200;i++)n+=f.elements.length;return n})()",
200000,
),
"form-elements-external-mutation": (
"(() => {document.body.innerHTML='<form id=f><input name=inside></form>'+'<input form=f name=outside>'.repeat(1000);const f=document.getElementById('f');let n=0;for(let i=0;i<200;i++){document.body.className='state'+i;if(f.elements[0].name==='inside')n++;}return n})()",
200,
),
"form-owner-ancestor-scaled": (
"(() => {document.body.innerHTML='<form id=f><div><div><div><input id=input></div></div></div></form>';const f=document.getElementById('f'),input=document.getElementById('input');let n=0;for(let i=0;i<10000000;i++)n+=input.form===f;return n})()",
10000000,
),
"form-owner-explicit-scaled": (
# Older baselines return null; ownership correctness has separate tests.
"(() => {document.body.innerHTML='<form id=f></form><input id=input form=f>';const input=document.getElementById('input');let n=0;for(let i=0;i<10000000;i++)n+=typeof input.form==='object';return n})()",
10000000,
),
"form-owner-explicit-mutation": (
"(() => {document.body.innerHTML='<form id=f></form><input id=input form=f>'+'<div id=extra></div>'.repeat(1000);const input=document.getElementById('input');let n=0;for(let i=0;i<10000;i++){input.className='state'+i;n+=typeof input.form==='object';}return n})()",
10000,
),
"frame-content-window-hot": (
"(() => {const e=document.createElement('iframe');document.body.appendChild(e);const w=e.contentWindow;let n=0;for(let i=0;i<50000;i++)n+=e.contentWindow===w;return n})()",
50000,
),
"frame-indexed-root-hot": (
"(() => {document.body.innerHTML='<iframe></iframe>'.repeat(3);const w=frames[0];let n=0;for(let i=0;i<10000;i++)n+=frames[0]===w;return n})()",
10000,
),
"frame-live-owner-dom-build": (
"(() => {document.body.innerHTML='<iframe></iframe>'.repeat(16);for(let i=0;i<5000;i++)document.body.appendChild(document.createElement('div'));return document.querySelectorAll('div').length+frames.length})()",
5016,
),
}
SCENARIOS.update({
f"html-parse-depth-{depth}": (
"(function(){const html='<div>'.repeat(" + str(depth) + ")+"
"'text'+'</div>'.repeat(" + str(depth) + ");"
"for(let i=0;i<3;i++)document.body.innerHTML=html.repeat(60);"
"return document.querySelectorAll('div').length})()",
depth * 60,
)
for depth in (4, 32, 128, 300)
})
SCENARIOS.update({
f"frame-length-{count}-dom-{padding}": (
"(() => {document.body.innerHTML='<div></div>'.repeat(" + str(padding)
+ ")+'<iframe></iframe>'.repeat(" + str(count)
+ ");let n=0;for(let i=0;i<" + str(iterations)
+ ";i++)n+=frames.length;return n})()",
count * iterations,
)
for count, iterations, padding in ((0, 100000, 0), (3, 10000, 0),
(32, 10000, 0), (3, 1000, 2500))
})
class Handler(BaseHTTPRequestHandler):
def do_GET(self) -> None:
self.send_response(200)
self.send_header("Content-Type", "text/html; charset=utf-8")
self.send_header("Content-Length", str(len(HTML)))
self.end_headers()
self.wfile.write(HTML)
def log_message(self, _format: str, *_args: object) -> None:
return
def run_once(binary: Path, url: str, expression: str, expected: int) -> tuple[float, int, float]:
with tempfile.NamedTemporaryFile(prefix="obscura-stdout-", delete=False) as stdout_file:
stdout_path = Path(stdout_file.name)
with tempfile.NamedTemporaryFile(prefix="obscura-stderr-", delete=False) as stderr_file:
stderr_path = Path(stderr_file.name)
command = [
"timeout",
"--signal=KILL",
str(PROCESS_TIMEOUT_SECONDS),
"prlimit",
"--core=0",
"--fsize=2097152",
"--nofile=256",
"--",
str(binary),
"fetch",
url,
"--allow-private-network",
"--quiet",
"--timeout",
"20",
"--wait",
"0",
"--eval",
expression,
]
started = time.perf_counter()
try:
with stdout_path.open("wb") as stdout_handle, stderr_path.open("wb") as stderr_handle:
process = subprocess.Popen(
command,
stdout=stdout_handle,
stderr=stderr_handle,
start_new_session=True,
)
try:
# Linux wait4 keeps precise CPU counters for the guarded process tree.
_, status, usage = os.wait4(process.pid, 0)
process.returncode = os.waitstatus_to_exitcode(status)
finally:
# Remove background descendants left by an untrusted candidate binary.
try:
os.killpg(process.pid, signal.SIGKILL)
except ProcessLookupError:
pass
elapsed_ms = (time.perf_counter() - started) * 1000
if process.returncode != 0:
raise RuntimeError(f"{binary} exited with status {process.returncode}")
cpu_seconds = usage.ru_utime + usage.ru_stime
rss_kib = usage.ru_maxrss
if not math.isfinite(cpu_seconds) or usage.ru_utime < 0 or usage.ru_stime < 0 or rss_kib <= 0:
raise RuntimeError(f"{binary} produced an invalid CPU/RSS measurement")
try:
result = json.loads(stdout_path.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError) as error:
raise RuntimeError(f"{binary} produced an invalid evaluation result") from error
if result != expected:
raise RuntimeError(
f"{binary} returned {result!r}; expected {expected!r}"
)
return elapsed_ms, rss_kib, cpu_seconds
finally:
stdout_path.unlink(missing_ok=True)
stderr_path.unlink(missing_ok=True)
def median(values: list[float | int]) -> float:
return float(statistics.median(values))
def add_summary(markdown: str) -> None:
summary = os.environ.get("GITHUB_STEP_SUMMARY")
if summary:
with open(summary, "a", encoding="utf-8") as output:
output.write(markdown)
else:
print(markdown)
def main() -> None:
parser = argparse.ArgumentParser()
parser.add_argument("--base", type=Path, required=True)
parser.add_argument("--candidate", type=Path, required=True)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()
for binary in (args.base, args.candidate):
if not binary.is_file():
raise SystemExit(f"missing binary: {binary}")
server = ThreadingHTTPServer(("127.0.0.1", 0), Handler)
thread = Thread(target=server.serve_forever, daemon=True)
thread.start()
url = f"http://127.0.0.1:{server.server_port}/"
samples: dict[str, dict[str, dict[str, list[float | int]]]] = {}
try:
for scenario, (expression, expected) in SCENARIOS.items():
samples[scenario] = {
"base": {"latency_ms": [], "rss_kib": [], "cpu_seconds": []},
"candidate": {"latency_ms": [], "rss_kib": [], "cpu_seconds": []},
}
for round_number in range(ROUNDS):
order = (
(("base", args.base), ("candidate", args.candidate))
if round_number % 2 == 0
else (("candidate", args.candidate), ("base", args.base))
)
for label, binary in order:
latency_ms, rss_kib, cpu_seconds = run_once(binary, url, expression, expected)
samples[scenario][label]["latency_ms"].append(latency_ms)
samples[scenario][label]["rss_kib"].append(rss_kib)
samples[scenario][label]["cpu_seconds"].append(cpu_seconds)
finally:
server.shutdown()
server.server_close()
report: dict[str, object] = {"rounds": ROUNDS, "scenarios": {}}
lines = [
"## Performance smoke comparison\n\n",
"Fails above 10% and 10 ms for latency, 10% and 20 ms for CPU, or 15% and 8 MiB for RSS.\n\n",
"| Scenario | Base latency | PR latency | Change | Base CPU | PR CPU | Base RSS | PR RSS | Change |\n",
"| --- | ---: | ---: | ---: | ---: | ---: | ---: | ---: | ---: |\n",
]
failures = []
for scenario in SCENARIOS:
base_latency = median(samples[scenario]["base"]["latency_ms"])
pr_latency = median(samples[scenario]["candidate"]["latency_ms"])
base_rss = median(samples[scenario]["base"]["rss_kib"])
pr_rss = median(samples[scenario]["candidate"]["rss_kib"])
latency_ratio = pr_latency / base_latency
rss_ratio = pr_rss / base_rss
base_cpu = median(samples[scenario]["base"]["cpu_seconds"])
pr_cpu = median(samples[scenario]["candidate"]["cpu_seconds"])
report["scenarios"][scenario] = {
"base_cpu_seconds": base_cpu,
"candidate_cpu_seconds": pr_cpu,
"base_latency_ms": base_latency,
"candidate_latency_ms": pr_latency,
"latency_ratio": latency_ratio,
"base_rss_kib": base_rss,
"candidate_rss_kib": pr_rss,
"rss_ratio": rss_ratio,
"samples": samples[scenario],
}
lines.append(
f"| {scenario} | {base_latency:.1f} ms | {pr_latency:.1f} ms | "
f"{(latency_ratio - 1) * 100:+.1f}% | {base_cpu:.4f} s | {pr_cpu:.4f} s | {base_rss / 1024:.1f} MiB | "
f"{pr_rss / 1024:.1f} MiB | {(rss_ratio - 1) * 100:+.1f}% |\n"
)
latency_delta = pr_latency - base_latency
rss_delta = pr_rss - base_rss
if (
latency_ratio > LATENCY_FAILURE_RATIO
and latency_delta > LATENCY_FAILURE_DELTA_MS
):
failures.append(f"{scenario} latency is {latency_ratio:.2f}x the base")
if pr_cpu > base_cpu * CPU_FAILURE_RATIO and pr_cpu - base_cpu > CPU_FAILURE_DELTA_SECONDS + 1e-9:
failures.append(f"{scenario} CPU increased from {base_cpu:.4f} s to {pr_cpu:.4f} s")
if rss_ratio > RSS_FAILURE_RATIO and rss_delta > RSS_FAILURE_DELTA_KIB:
failures.append(f"{scenario} RSS is {rss_ratio:.2f}x the base")
args.output.parent.mkdir(parents=True, exist_ok=True)
args.output.write_text(json.dumps(report, indent=2) + "\n", encoding="utf-8")
add_summary("".join(lines))
for failure in failures:
print(f"::error::{failure}")
if failures:
raise SystemExit("candidate exceeds the performance regression threshold")
if __name__ == "__main__":
main()