Lightpanda: the headless browser designed for AI and automation
test: add agent regression suite
Adrià Arrufatf191410773ee
diff --git a/.github/workflows/agent-regression.yml b/.github/workflows/agent-regression.ymlnew file mode 100644index 000000000..b506731eb--- /dev/null+++ b/.github/workflows/agent-regression.yml@@ -0,0 +1,66 @@+name: agent-regression++on:+ schedule:+ - cron: "17 3 * * *"+ workflow_dispatch:+ inputs:+ model:+ description: "Gemini model id for the live layer (empty = suite default, see test/agent/run.sh)"+ required: false++jobs:+ # Keep identical to e2e-test.yml's zig-build-release so the nightly suite+ # exercises the same build variant that gates PRs.+ zig-build-release:+ name: zig build release+ runs-on: ubuntu-latest+ timeout-minutes: 20++ steps:+ - uses: actions/checkout@v6+ with:+ fetch-depth: 0++ - uses: ./.github/actions/install+ - uses: ./.github/actions/v8-snapshot++ - name: zig build release+ run: zig build -Dsnapshot_path=../../snapshot.bin -Dprebuilt_v8_path=v8/libc_v8.a -Doptimize=ReleaseFast -Dcpu=x86_64++ - name: upload artifact+ uses: actions/upload-artifact@v7+ with:+ name: lightpanda-build-release+ path: |+ zig-out/bin/lightpanda+ retention-days: 1++ agent-regression:+ name: agent-regression+ needs: zig-build-release+ runs-on: ubuntu-latest+ # The live HN save task drives the LLM through many turns (~7 min); give+ # the whole suite generous headroom.+ timeout-minutes: 25++ steps:+ - uses: actions/checkout@v6++ - name: download lightpanda release+ uses: actions/download-artifact@v8+ with:+ name: lightpanda-build-release+ path: bin++ - run: chmod a+x bin/lightpanda++ - name: run agent regression suite (deterministic + live)+ env:+ LPD: ${{ github.workspace }}/bin/lightpanda+ GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}+ LP_MODEL: ${{ github.event.inputs.model }}+ # Optional: news.ycombinator.com often blocks datacenter IPs. Reuse+ # the residential proxy secret if present; empty is fine locally.+ LP_HTTP_PROXY: ${{ secrets.MASSIVE_PROXY_RESIDENTIAL_US }}+ run: ./test/agent/run.sh alldiff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.ymlindex 9ac05be88..bb645759a 100644--- a/.github/workflows/e2e-test.yml+++ b/.github/workflows/e2e-test.yml@@ -13,6 +13,7 @@ on:paths:- ".github/**"- "src/**"+ - "test/agent/**"- "build.zig"- "build.zig.zon"@@ -28,6 +29,7 @@ on:paths:- ".github/**"- "src/**"+ - "test/agent/**"- "build.zig"- "build.zig.zon"@@ -402,6 +404,33 @@ jobs:- run: ./lightpanda fetch https://demo-browser.lightpanda.io/campfire-commerce/+ agent-deterministic:+ name: agent deterministic replay+ needs: zig-build-release++ runs-on: ubuntu-latest+ timeout-minutes: 5++ steps:+ # Checkout THIS repo — agent fixtures, golden files and run.sh live here.+ - uses: actions/checkout@v6++ - name: download artifact+ uses: actions/download-artifact@v8+ with:+ name: lightpanda-build-release+ path: bin++ - run: chmod a+x bin/lightpanda++ # No API key: replays a golden PandaScript against frozen local fixtures+ # and diffs the returned JSON against golden/hn-front.json. The keyed+ # live layer runs separately in agent-regression.yml.+ - name: deterministic agent replay+ env:+ LPD: ${{ github.workspace }}/bin/lightpanda+ run: ./test/agent/run.sh deterministic+mcp-smoke:name: mcp smokeneeds: zig-build-releasediff --git a/Makefile b/Makefileindex 22a828795..dcecf8ffa 100644--- a/Makefile+++ b/Makefile@@ -128,6 +128,13 @@ end2end:@test -d ../democd ../demo && go run runner/main.go+## Run the agent regression suite (LAYER=deterministic|live|all, default all).+## The live layer needs GOOGLE_API_KEY or GEMINI_API_KEY; without one only+## the deterministic layer runs. See test/agent/README.md.+test-agent:+ @test -x zig-out/bin/lightpanda || $(MAKE) build+ @./test/agent/run.sh $(LAYER)+## Remove build artifacts (keeps .lp-cache/ and zig-pkg/ — slow to re-fetch)clean:rm -rf zig-out .zig-cache src/snapshot.bindiff --git a/test/agent/README.md b/test/agent/README.mdnew file mode 100644index 000000000..a887e9888--- /dev/null+++ b/test/agent/README.md@@ -0,0 +1,84 @@+# Agent regression suite++End-to-end regression tests for `lightpanda agent`. Two layers, so the cheap+deterministic checks can gate every PR while the expensive keyed checks run+nightly / on demand.++```+test/agent/+ run.sh orchestrator (bash + jq)+ fixtures/ static HTML served locally over HTTP+ facts/company.html closed-form Q&A page+ hn/front.html frozen HN front page+ hn/item-*.html frozen HN comment threads+ scripts/hn-front.js golden PandaScript (replays against the fixtures)+ golden/hn-front.json exact expected replay output+ cases/static-qa.tsv <task>\t<expected-substring> per line+ cases/hn-live.task task prompt for the live HN case (pins the contract)+ schemas/hn-live.jq shape invariant for the live HN output+```++## Layers++### Deterministic (no API key, runs on every PR)++Replays `scripts/hn-front.js` — a top-5-stories / top-3-comments HN scrape+pointed at the local fixture server — and diffs the returned JSON **exactly**+against `golden/hn-front.json`. This guards the replay engine and the+`goto` / `extract` primitives. No LLM, no network.++### Live (needs `GOOGLE_API_KEY` or `GEMINI_API_KEY`, runs nightly / on demand)++Drives the real Gemini-backed agent:++- **Static Q&A** — for each row in `cases/static-qa.tsv`, run+ `lightpanda agent --task "<question about a local fixture>"` and assert the+ expected substring appears in the answer. Closed-form answers make the+ substring match robust to LLM phrasing.+- **HN save + replay** — ask the agent (task in `cases/hn-live.task`) to+ scrape live Hacker News and `/save` a reproducible script, then replay that+ script **token-free** and validate the output against `schemas/hn-live.jq`+ (a *shape* invariant: exactly 5 stories, non-empty titles, ≤3 comments each+ with non-empty `user`/`text`). We can't assert exact values because the live+ site and the LLM both vary — so the task prompt pins the output contract and+ the schema verifies it; edit those two files together.++The stable `$usage total=N` line lightpanda prints to stderr is captured per+task; a loose `MAX_TOKENS` ceiling flags a runaway agent loop.++## Running locally++```bash+make build # produces zig-out/bin/lightpanda+make test-agent # all layers (live skipped if no key)+make test-agent LAYER=deterministic++# or invoke run.sh directly:+./test/agent/run.sh deterministic+GEMINI_API_KEY=... ./test/agent/run.sh live+```++Environment knobs (see the header of `run.sh`): `LPD`, `LP_MODEL`,+`LP_HTTP_PROXY`, `MAX_TOKENS`. The fixture-server port (8081) is part of the+fixture contract — the golden script and TSV embed it — and is not+configurable.++## CI++- `.github/workflows/e2e-test.yml` runs the **deterministic** layer on every PR+ (job `agent-deterministic`).+- `.github/workflows/agent-regression.yml` runs **all** layers nightly and via+ *Run workflow* (`workflow_dispatch`). It uses the `GEMINI_API_KEY` repo secret;+ it reuses the `MASSIVE_PROXY_RESIDENTIAL_US` secret as `LP_HTTP_PROXY` for the+ live HN call (datacenter IPs are often blocked by news.ycombinator.com).++## Maintenance++- **Add a Q&A case:** append a `<task>\t<expected>` row to+ `cases/static-qa.tsv` (point the task at a `fixtures/` page).+- **Change the deterministic scrape:** edit `scripts/hn-front.js` and/or the+ `fixtures/hn/*` pages, then regenerate the golden:+ ```bash+ ./test/agent/run.sh update-golden+ ```+ Review the diff before committing — the golden is the contract.diff --git a/test/agent/cases/hn-live.task b/test/agent/cases/hn-live.tasknew file mode 100644index 000000000..30f46a4c9--- /dev/null+++ b/test/agent/cases/hn-live.task@@ -0,0 +1 @@+Go to https://news.ycombinator.com, collect the top 5 stories and the top 3 comments of each story. Return a JSON array of exactly 5 objects, each with keys "title" (the story title string) and "comments" (an array of at most 3 objects, each with keys "user" and "text"; leave it empty if the story has no comments yet).diff --git a/test/agent/cases/static-qa.tsv b/test/agent/cases/static-qa.tsvnew file mode 100644index 000000000..5291c9ee4--- /dev/null+++ b/test/agent/cases/static-qa.tsv@@ -0,0 +1,11 @@+# Static Q&A regression cases for the live LLM layer.+# One case per line, TAB-separated: <task>\t<expected-substring>+# The task tells the agent to visit a local fixture page and answer a+# closed-form question; the expected substring must appear (case-insensitive)+# in the agent's final answer on stdout. Keep answers unambiguous so LLM+# phrasing does not matter — match on the load-bearing token only.+# Lines starting with # and blank lines are ignored.+Go to http://127.0.0.1:8081/facts/company.html and tell me only the year Acme Robotics was founded. 1987+Go to http://127.0.0.1:8081/facts/company.html and tell me the city where Acme Robotics is headquartered. Reykjavik+Go to http://127.0.0.1:8081/facts/company.html and tell me the full name of the CEO of Acme Robotics. Mira Voss+Go to http://127.0.0.1:8081/facts/company.html and tell me the name of Acme Robotics' flagship product. Puffin 9diff --git a/test/agent/fixtures/facts/company.html b/test/agent/fixtures/facts/company.htmlnew file mode 100644index 000000000..de21304a8--- /dev/null+++ b/test/agent/fixtures/facts/company.html@@ -0,0 +1,28 @@+<!DOCTYPE html>+<html lang="en">+<head>+ <meta charset="utf-8">+ <title>Acme Robotics — Company Facts</title>+</head>+<body>+ <main>+ <h1>Acme Robotics</h1>+ <p>Acme Robotics is a fictional company used only for automated testing.</p>++ <table id="facts">+ <tbody>+ <tr><th>Founded</th><td id="founded">1987</td></tr>+ <tr><th>Headquarters</th><td id="hq">Reykjavik, Iceland</td></tr>+ <tr><th>CEO</th><td id="ceo">Dr. Mira Voss</td></tr>+ <tr><th>Employees</th><td id="employees">4,213</td></tr>+ <tr><th>Flagship product</th><td id="product">The Puffin 9 autonomous drone</td></tr>+ </tbody>+ </table>++ <section id="motto">+ <h2>Company motto</h2>+ <p>"We build calm machines for a loud world."</p>+ </section>+ </main>+</body>+</html>diff --git a/test/agent/fixtures/hn/front.html b/test/agent/fixtures/hn/front.htmlnew file mode 100644index 000000000..4d00120b1--- /dev/null+++ b/test/agent/fixtures/hn/front.html@@ -0,0 +1,94 @@+<!DOCTYPE html>+<html lang="en" op="news">+<head>+ <meta charset="utf-8">+ <title>Hacker News (fixture)</title>+</head>+<body>+ <center>+ <table id="hnmain" border="0" cellpadding="0" cellspacing="0" width="85%">+ <tr>+ <td>+ <table border="0" cellpadding="0" cellspacing="0">+ <tr class="athing" id="101">+ <td align="right" valign="top" class="title"><span class="rank">1.</span></td>+ <td valign="top" class="votelinks"><a href="vote?id=101"><div class="votearrow" title="upvote"></div></a></td>+ <td class="title"><span class="titleline"><a href="https://example.com/1">Story One: Local-first software</a><span class="sitebit comhead"> (<a href="from?site=example.com"><span class="sitestr">example.com</span></a>)</span></span></td>+ </tr>+ <tr>+ <td colspan="2"></td>+ <td class="subtext"><span class="subline"><span class="score" id="score_101">210 points</span> by <a href="user?id=alice" class="hnuser">alice</a></span></td>+ </tr>+ <tr class="spacer" style="height:5px"></tr>++ <tr class="athing" id="102">+ <td align="right" valign="top" class="title"><span class="rank">2.</span></td>+ <td valign="top" class="votelinks"><a href="vote?id=102"><div class="votearrow" title="upvote"></div></a></td>+ <td class="title"><span class="titleline"><a href="https://example.com/2">Story Two: A tiny headless browser</a><span class="sitebit comhead"> (<a href="from?site=example.com"><span class="sitestr">example.com</span></a>)</span></span></td>+ </tr>+ <tr>+ <td colspan="2"></td>+ <td class="subtext"><span class="subline"><span class="score" id="score_102">184 points</span> by <a href="user?id=bob" class="hnuser">bob</a></span></td>+ </tr>+ <tr class="spacer" style="height:5px"></tr>++ <tr class="athing" id="103">+ <td align="right" valign="top" class="title"><span class="rank">3.</span></td>+ <td valign="top" class="votelinks"><a href="vote?id=103"><div class="votearrow" title="upvote"></div></a></td>+ <td class="title"><span class="titleline"><a href="https://example.com/3">Story Three: Zig comptime tricks</a><span class="sitebit comhead"> (<a href="from?site=example.com"><span class="sitestr">example.com</span></a>)</span></span></td>+ </tr>+ <tr>+ <td colspan="2"></td>+ <td class="subtext"><span class="subline"><span class="score" id="score_103">153 points</span> by <a href="user?id=carol" class="hnuser">carol</a></span></td>+ </tr>+ <tr class="spacer" style="height:5px"></tr>++ <tr class="athing" id="104">+ <td align="right" valign="top" class="title"><span class="rank">4.</span></td>+ <td valign="top" class="votelinks"><a href="vote?id=104"><div class="votearrow" title="upvote"></div></a></td>+ <td class="title"><span class="titleline"><a href="https://example.com/4">Story Four: Deterministic replay for agents</a><span class="sitebit comhead"> (<a href="from?site=example.com"><span class="sitestr">example.com</span></a>)</span></span></td>+ </tr>+ <tr>+ <td colspan="2"></td>+ <td class="subtext"><span class="subline"><span class="score" id="score_104">121 points</span> by <a href="user?id=dave" class="hnuser">dave</a></span></td>+ </tr>+ <tr class="spacer" style="height:5px"></tr>++ <tr class="athing" id="105">+ <td align="right" valign="top" class="title"><span class="rank">5.</span></td>+ <td valign="top" class="votelinks"><a href="vote?id=105"><div class="votearrow" title="upvote"></div></a></td>+ <td class="title"><span class="titleline"><a href="https://example.com/5">Story Five: The end of CDP overhead</a><span class="sitebit comhead"> (<a href="from?site=example.com"><span class="sitestr">example.com</span></a>)</span></span></td>+ </tr>+ <tr>+ <td colspan="2"></td>+ <td class="subtext"><span class="subline"><span class="score" id="score_105">98 points</span> by <a href="user?id=erin" class="hnuser">erin</a></span></td>+ </tr>+ <tr class="spacer" style="height:5px"></tr>++ <tr class="athing" id="106">+ <td align="right" valign="top" class="title"><span class="rank">6.</span></td>+ <td valign="top" class="votelinks"><a href="vote?id=106"><div class="votearrow" title="upvote"></div></a></td>+ <td class="title"><span class="titleline"><a href="https://example.com/6">Story Six: Should not be scraped</a></span></td>+ </tr>+ <tr>+ <td colspan="2"></td>+ <td class="subtext"><span class="subline"><span class="score" id="score_106">80 points</span> by <a href="user?id=frank" class="hnuser">frank</a></span></td>+ </tr>+ <tr class="spacer" style="height:5px"></tr>++ <tr class="athing" id="107">+ <td align="right" valign="top" class="title"><span class="rank">7.</span></td>+ <td valign="top" class="votelinks"><a href="vote?id=107"><div class="votearrow" title="upvote"></div></a></td>+ <td class="title"><span class="titleline"><a href="https://example.com/7">Story Seven: Also should not be scraped</a></span></td>+ </tr>+ <tr>+ <td colspan="2"></td>+ <td class="subtext"><span class="subline"><span class="score" id="score_107">61 points</span> by <a href="user?id=grace" class="hnuser">grace</a></span></td>+ </tr>+ </table>+ </td>+ </tr>+ </table>+ </center>+</body>+</html>diff --git a/test/agent/fixtures/hn/item-101.html b/test/agent/fixtures/hn/item-101.htmlnew file mode 100644index 000000000..3ad817be6--- /dev/null+++ b/test/agent/fixtures/hn/item-101.html@@ -0,0 +1,105 @@+<!DOCTYPE html>+<html lang="en" op="item">+<head>+ <meta charset="utf-8">+ <title>Story One: Local-first software | Hacker News (fixture)</title>+</head>+<body>+ <center>+ <table id="hnmain" border="0" cellpadding="0" cellspacing="0" width="85%">+ <tr>+ <td>+ <table class="fatitem" border="0">+ <tr class="athing" id="101">+ <td class="title"><span class="titleline"><a href="https://example.com/1">Story One: Local-first software</a></span></td>+ </tr>+ </table>+ <br>+ <table border="0" class="comment-tree">+ <tr class="athing comtr" id="c1011">+ <td>+ <table border="0">+ <tr>+ <td class="default">+ <div class="comhead">+ <a href="user?id=user_101_1" class="hnuser">user_101_1</a>+ <span class="age" title="fixture"><a href="item?id=101">1 hours ago</a></span>+ </div>+ <br>+ <div class="comment"><div class="commtext c00">Comment 1 on story 101. This is deterministic fixture text.</div></div>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ <tr class="athing comtr" id="c1012">+ <td>+ <table border="0">+ <tr>+ <td class="default">+ <div class="comhead">+ <a href="user?id=user_101_2" class="hnuser">user_101_2</a>+ <span class="age" title="fixture"><a href="item?id=101">2 hours ago</a></span>+ </div>+ <br>+ <div class="comment"><div class="commtext c00">Comment 2 on story 101. This is deterministic fixture text.</div></div>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ <tr class="athing comtr" id="c1013">+ <td>+ <table border="0">+ <tr>+ <td class="default">+ <div class="comhead">+ <a href="user?id=user_101_3" class="hnuser">user_101_3</a>+ <span class="age" title="fixture"><a href="item?id=101">3 hours ago</a></span>+ </div>+ <br>+ <div class="comment"><div class="commtext c00">Comment 3 on story 101. This is deterministic fixture text.</div></div>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ <tr class="athing comtr" id="c1014">+ <td>+ <table border="0">+ <tr>+ <td class="default">+ <div class="comhead">+ <a href="user?id=user_101_4" class="hnuser">user_101_4</a>+ <span class="age" title="fixture"><a href="item?id=101">4 hours ago</a></span>+ </div>+ <br>+ <div class="comment"><div class="commtext c00">Comment 4 on story 101. This is deterministic fixture text.</div></div>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ <tr class="athing comtr" id="c1015">+ <td>+ <table border="0">+ <tr>+ <td class="default">+ <div class="comhead">+ <a href="user?id=user_101_5" class="hnuser">user_101_5</a>+ <span class="age" title="fixture"><a href="item?id=101">5 hours ago</a></span>+ </div>+ <br>+ <div class="comment"><div class="commtext c00">Comment 5 on story 101. This is deterministic fixture text.</div></div>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ </table>+ </center>+</body>+</html>diff --git a/test/agent/fixtures/hn/item-102.html b/test/agent/fixtures/hn/item-102.htmlnew file mode 100644index 000000000..a4650eb63--- /dev/null+++ b/test/agent/fixtures/hn/item-102.html@@ -0,0 +1,105 @@+<!DOCTYPE html>+<html lang="en" op="item">+<head>+ <meta charset="utf-8">+ <title>Story Two: A tiny headless browser | Hacker News (fixture)</title>+</head>+<body>+ <center>+ <table id="hnmain" border="0" cellpadding="0" cellspacing="0" width="85%">+ <tr>+ <td>+ <table class="fatitem" border="0">+ <tr class="athing" id="102">+ <td class="title"><span class="titleline"><a href="https://example.com/2">Story Two: A tiny headless browser</a></span></td>+ </tr>+ </table>+ <br>+ <table border="0" class="comment-tree">+ <tr class="athing comtr" id="c1021">+ <td>+ <table border="0">+ <tr>+ <td class="default">+ <div class="comhead">+ <a href="user?id=user_102_1" class="hnuser">user_102_1</a>+ <span class="age" title="fixture"><a href="item?id=102">1 hours ago</a></span>+ </div>+ <br>+ <div class="comment"><div class="commtext c00">Comment 1 on story 102. This is deterministic fixture text.</div></div>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ <tr class="athing comtr" id="c1022">+ <td>+ <table border="0">+ <tr>+ <td class="default">+ <div class="comhead">+ <a href="user?id=user_102_2" class="hnuser">user_102_2</a>+ <span class="age" title="fixture"><a href="item?id=102">2 hours ago</a></span>+ </div>+ <br>+ <div class="comment"><div class="commtext c00">Comment 2 on story 102. This is deterministic fixture text.</div></div>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ <tr class="athing comtr" id="c1023">+ <td>+ <table border="0">+ <tr>+ <td class="default">+ <div class="comhead">+ <a href="user?id=user_102_3" class="hnuser">user_102_3</a>+ <span class="age" title="fixture"><a href="item?id=102">3 hours ago</a></span>+ </div>+ <br>+ <div class="comment"><div class="commtext c00">Comment 3 on story 102. This is deterministic fixture text.</div></div>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ <tr class="athing comtr" id="c1024">+ <td>+ <table border="0">+ <tr>+ <td class="default">+ <div class="comhead">+ <a href="user?id=user_102_4" class="hnuser">user_102_4</a>+ <span class="age" title="fixture"><a href="item?id=102">4 hours ago</a></span>+ </div>+ <br>+ <div class="comment"><div class="commtext c00">Comment 4 on story 102. This is deterministic fixture text.</div></div>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ <tr class="athing comtr" id="c1025">+ <td>+ <table border="0">+ <tr>+ <td class="default">+ <div class="comhead">+ <a href="user?id=user_102_5" class="hnuser">user_102_5</a>+ <span class="age" title="fixture"><a href="item?id=102">5 hours ago</a></span>+ </div>+ <br>+ <div class="comment"><div class="commtext c00">Comment 5 on story 102. This is deterministic fixture text.</div></div>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ </table>+ </center>+</body>+</html>diff --git a/test/agent/fixtures/hn/item-103.html b/test/agent/fixtures/hn/item-103.htmlnew file mode 100644index 000000000..7ae4af487--- /dev/null+++ b/test/agent/fixtures/hn/item-103.html@@ -0,0 +1,105 @@+<!DOCTYPE html>+<html lang="en" op="item">+<head>+ <meta charset="utf-8">+ <title>Story Three: Zig comptime tricks | Hacker News (fixture)</title>+</head>+<body>+ <center>+ <table id="hnmain" border="0" cellpadding="0" cellspacing="0" width="85%">+ <tr>+ <td>+ <table class="fatitem" border="0">+ <tr class="athing" id="103">+ <td class="title"><span class="titleline"><a href="https://example.com/3">Story Three: Zig comptime tricks</a></span></td>+ </tr>+ </table>+ <br>+ <table border="0" class="comment-tree">+ <tr class="athing comtr" id="c1031">+ <td>+ <table border="0">+ <tr>+ <td class="default">+ <div class="comhead">+ <a href="user?id=user_103_1" class="hnuser">user_103_1</a>+ <span class="age" title="fixture"><a href="item?id=103">1 hours ago</a></span>+ </div>+ <br>+ <div class="comment"><div class="commtext c00">Comment 1 on story 103. This is deterministic fixture text.</div></div>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ <tr class="athing comtr" id="c1032">+ <td>+ <table border="0">+ <tr>+ <td class="default">+ <div class="comhead">+ <a href="user?id=user_103_2" class="hnuser">user_103_2</a>+ <span class="age" title="fixture"><a href="item?id=103">2 hours ago</a></span>+ </div>+ <br>+ <div class="comment"><div class="commtext c00">Comment 2 on story 103. This is deterministic fixture text.</div></div>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ <tr class="athing comtr" id="c1033">+ <td>+ <table border="0">+ <tr>+ <td class="default">+ <div class="comhead">+ <a href="user?id=user_103_3" class="hnuser">user_103_3</a>+ <span class="age" title="fixture"><a href="item?id=103">3 hours ago</a></span>+ </div>+ <br>+ <div class="comment"><div class="commtext c00">Comment 3 on story 103. This is deterministic fixture text.</div></div>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ <tr class="athing comtr" id="c1034">+ <td>+ <table border="0">+ <tr>+ <td class="default">+ <div class="comhead">+ <a href="user?id=user_103_4" class="hnuser">user_103_4</a>+ <span class="age" title="fixture"><a href="item?id=103">4 hours ago</a></span>+ </div>+ <br>+ <div class="comment"><div class="commtext c00">Comment 4 on story 103. This is deterministic fixture text.</div></div>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ <tr class="athing comtr" id="c1035">+ <td>+ <table border="0">+ <tr>+ <td class="default">+ <div class="comhead">+ <a href="user?id=user_103_5" class="hnuser">user_103_5</a>+ <span class="age" title="fixture"><a href="item?id=103">5 hours ago</a></span>+ </div>+ <br>+ <div class="comment"><div class="commtext c00">Comment 5 on story 103. This is deterministic fixture text.</div></div>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ </table>+ </center>+</body>+</html>diff --git a/test/agent/fixtures/hn/item-104.html b/test/agent/fixtures/hn/item-104.htmlnew file mode 100644index 000000000..9b45fb4db--- /dev/null+++ b/test/agent/fixtures/hn/item-104.html@@ -0,0 +1,105 @@+<!DOCTYPE html>+<html lang="en" op="item">+<head>+ <meta charset="utf-8">+ <title>Story Four: Deterministic replay for agents | Hacker News (fixture)</title>+</head>+<body>+ <center>+ <table id="hnmain" border="0" cellpadding="0" cellspacing="0" width="85%">+ <tr>+ <td>+ <table class="fatitem" border="0">+ <tr class="athing" id="104">+ <td class="title"><span class="titleline"><a href="https://example.com/4">Story Four: Deterministic replay for agents</a></span></td>+ </tr>+ </table>+ <br>+ <table border="0" class="comment-tree">+ <tr class="athing comtr" id="c1041">+ <td>+ <table border="0">+ <tr>+ <td class="default">+ <div class="comhead">+ <a href="user?id=user_104_1" class="hnuser">user_104_1</a>+ <span class="age" title="fixture"><a href="item?id=104">1 hours ago</a></span>+ </div>+ <br>+ <div class="comment"><div class="commtext c00">Comment 1 on story 104. This is deterministic fixture text.</div></div>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ <tr class="athing comtr" id="c1042">+ <td>+ <table border="0">+ <tr>+ <td class="default">+ <div class="comhead">+ <a href="user?id=user_104_2" class="hnuser">user_104_2</a>+ <span class="age" title="fixture"><a href="item?id=104">2 hours ago</a></span>+ </div>+ <br>+ <div class="comment"><div class="commtext c00">Comment 2 on story 104. This is deterministic fixture text.</div></div>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ <tr class="athing comtr" id="c1043">+ <td>+ <table border="0">+ <tr>+ <td class="default">+ <div class="comhead">+ <a href="user?id=user_104_3" class="hnuser">user_104_3</a>+ <span class="age" title="fixture"><a href="item?id=104">3 hours ago</a></span>+ </div>+ <br>+ <div class="comment"><div class="commtext c00">Comment 3 on story 104. This is deterministic fixture text.</div></div>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ <tr class="athing comtr" id="c1044">+ <td>+ <table border="0">+ <tr>+ <td class="default">+ <div class="comhead">+ <a href="user?id=user_104_4" class="hnuser">user_104_4</a>+ <span class="age" title="fixture"><a href="item?id=104">4 hours ago</a></span>+ </div>+ <br>+ <div class="comment"><div class="commtext c00">Comment 4 on story 104. This is deterministic fixture text.</div></div>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ <tr class="athing comtr" id="c1045">+ <td>+ <table border="0">+ <tr>+ <td class="default">+ <div class="comhead">+ <a href="user?id=user_104_5" class="hnuser">user_104_5</a>+ <span class="age" title="fixture"><a href="item?id=104">5 hours ago</a></span>+ </div>+ <br>+ <div class="comment"><div class="commtext c00">Comment 5 on story 104. This is deterministic fixture text.</div></div>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ </table>+ </center>+</body>+</html>diff --git a/test/agent/fixtures/hn/item-105.html b/test/agent/fixtures/hn/item-105.htmlnew file mode 100644index 000000000..304a0d95b--- /dev/null+++ b/test/agent/fixtures/hn/item-105.html@@ -0,0 +1,105 @@+<!DOCTYPE html>+<html lang="en" op="item">+<head>+ <meta charset="utf-8">+ <title>Story Five: The end of CDP overhead | Hacker News (fixture)</title>+</head>+<body>+ <center>+ <table id="hnmain" border="0" cellpadding="0" cellspacing="0" width="85%">+ <tr>+ <td>+ <table class="fatitem" border="0">+ <tr class="athing" id="105">+ <td class="title"><span class="titleline"><a href="https://example.com/5">Story Five: The end of CDP overhead</a></span></td>+ </tr>+ </table>+ <br>+ <table border="0" class="comment-tree">+ <tr class="athing comtr" id="c1051">+ <td>+ <table border="0">+ <tr>+ <td class="default">+ <div class="comhead">+ <a href="user?id=user_105_1" class="hnuser">user_105_1</a>+ <span class="age" title="fixture"><a href="item?id=105">1 hours ago</a></span>+ </div>+ <br>+ <div class="comment"><div class="commtext c00">Comment 1 on story 105. This is deterministic fixture text.</div></div>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ <tr class="athing comtr" id="c1052">+ <td>+ <table border="0">+ <tr>+ <td class="default">+ <div class="comhead">+ <a href="user?id=user_105_2" class="hnuser">user_105_2</a>+ <span class="age" title="fixture"><a href="item?id=105">2 hours ago</a></span>+ </div>+ <br>+ <div class="comment"><div class="commtext c00">Comment 2 on story 105. This is deterministic fixture text.</div></div>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ <tr class="athing comtr" id="c1053">+ <td>+ <table border="0">+ <tr>+ <td class="default">+ <div class="comhead">+ <a href="user?id=user_105_3" class="hnuser">user_105_3</a>+ <span class="age" title="fixture"><a href="item?id=105">3 hours ago</a></span>+ </div>+ <br>+ <div class="comment"><div class="commtext c00">Comment 3 on story 105. This is deterministic fixture text.</div></div>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ <tr class="athing comtr" id="c1054">+ <td>+ <table border="0">+ <tr>+ <td class="default">+ <div class="comhead">+ <a href="user?id=user_105_4" class="hnuser">user_105_4</a>+ <span class="age" title="fixture"><a href="item?id=105">4 hours ago</a></span>+ </div>+ <br>+ <div class="comment"><div class="commtext c00">Comment 4 on story 105. This is deterministic fixture text.</div></div>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ <tr class="athing comtr" id="c1055">+ <td>+ <table border="0">+ <tr>+ <td class="default">+ <div class="comhead">+ <a href="user?id=user_105_5" class="hnuser">user_105_5</a>+ <span class="age" title="fixture"><a href="item?id=105">5 hours ago</a></span>+ </div>+ <br>+ <div class="comment"><div class="commtext c00">Comment 5 on story 105. This is deterministic fixture text.</div></div>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ </table>+ </td>+ </tr>+ </table>+ </center>+</body>+</html>diff --git a/test/agent/golden/hn-front.json b/test/agent/golden/hn-front.jsonnew file mode 100644index 000000000..61d05386f--- /dev/null+++ b/test/agent/golden/hn-front.json@@ -0,0 +1,97 @@+[+ {+ "comments": [+ {+ "text": "Comment 1 on story 101. This is deterministic fixture text.",+ "user": "user_101_1"+ },+ {+ "text": "Comment 2 on story 101. This is deterministic fixture text.",+ "user": "user_101_2"+ },+ {+ "text": "Comment 3 on story 101. This is deterministic fixture text.",+ "user": "user_101_3"+ }+ ],+ "rank": "1.",+ "title": "Story One: Local-first software",+ "url": "https://example.com/1"+ },+ {+ "comments": [+ {+ "text": "Comment 1 on story 102. This is deterministic fixture text.",+ "user": "user_102_1"+ },+ {+ "text": "Comment 2 on story 102. This is deterministic fixture text.",+ "user": "user_102_2"+ },+ {+ "text": "Comment 3 on story 102. This is deterministic fixture text.",+ "user": "user_102_3"+ }+ ],+ "rank": "2.",+ "title": "Story Two: A tiny headless browser",+ "url": "https://example.com/2"+ },+ {+ "comments": [+ {+ "text": "Comment 1 on story 103. This is deterministic fixture text.",+ "user": "user_103_1"+ },+ {+ "text": "Comment 2 on story 103. This is deterministic fixture text.",+ "user": "user_103_2"+ },+ {+ "text": "Comment 3 on story 103. This is deterministic fixture text.",+ "user": "user_103_3"+ }+ ],+ "rank": "3.",+ "title": "Story Three: Zig comptime tricks",+ "url": "https://example.com/3"+ },+ {+ "comments": [+ {+ "text": "Comment 1 on story 104. This is deterministic fixture text.",+ "user": "user_104_1"+ },+ {+ "text": "Comment 2 on story 104. This is deterministic fixture text.",+ "user": "user_104_2"+ },+ {+ "text": "Comment 3 on story 104. This is deterministic fixture text.",+ "user": "user_104_3"+ }+ ],+ "rank": "4.",+ "title": "Story Four: Deterministic replay for agents",+ "url": "https://example.com/4"+ },+ {+ "comments": [+ {+ "text": "Comment 1 on story 105. This is deterministic fixture text.",+ "user": "user_105_1"+ },+ {+ "text": "Comment 2 on story 105. This is deterministic fixture text.",+ "user": "user_105_2"+ },+ {+ "text": "Comment 3 on story 105. This is deterministic fixture text.",+ "user": "user_105_3"+ }+ ],+ "rank": "5.",+ "title": "Story Five: The end of CDP overhead",+ "url": "https://example.com/5"+ }+]diff --git a/test/agent/run.sh b/test/agent/run.shnew file mode 100755index 000000000..727de70ba--- /dev/null+++ b/test/agent/run.sh@@ -0,0 +1,189 @@+#!/usr/bin/env bash+#+# Agent regression suite. Two layers:+#+# deterministic — replay a golden PandaScript against frozen local HTML+# fixtures and compare the returned JSON exactly to a+# golden file. No API key, no network. Runs on every PR.+#+# live — drive the real LLM agent (needs GOOGLE_API_KEY):+# * static Q&A: ask closed-form questions about a local+# fixture page, substring-match the answer.+# * HN save+replay: ask the agent to scrape live Hacker+# News and /save a reproducible script, then replay it+# token-free and validate the output against a shape+# invariant (jq), not exact values.+#+# Usage: test/agent/run.sh [deterministic|live|all|update-golden] (default: all)+#+# update-golden regenerates golden/hn-front.json from the current fixtures and+# script — review the diff before committing.+#+# Env:+# LPD path to the lightpanda binary (default: zig-out/bin/lightpanda)+# GOOGLE_API_KEY or GEMINI_API_KEY (the binary accepts both) — required for+# the live layer; live layer is skipped if neither is set+# LP_MODEL Gemini model id for the live layer (default below)+# LP_HTTP_PROXY optional proxy for the live HN call only (datacenter IPs are+# often blocked by news.ycombinator.com); localhost fixtures+# are never proxied+# MAX_TOKENS per-live-task total-token ceiling (default: 3000000).+# `total` counts cached reads, so a normal HN save is ~1M;+# this is a loose backstop against a runaway agent loop.+set -uo pipefail++LAYER="${1:-all}"++HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+REPO="$(cd "$HERE/../.." && pwd)"+LPD="${LPD:-$REPO/zig-out/bin/lightpanda}"+# The port is part of the fixture contract: scripts/hn-front.js, the golden+# file, and cases/static-qa.tsv all embed it. Not configurable.+PORT=8081+BASE="http://127.0.0.1:${PORT}"+# Pin an explicit model id — never a *-latest / *-preview alias, which drift.+LP_MODEL="${LP_MODEL:-gemini-3.5-flash}"+MAX_TOKENS="${MAX_TOKENS:-3000000}"++export LIGHTPANDA_DISABLE_TELEMETRY=true++PASS=0+FAIL=0+green() { printf '\033[32m%s\033[0m\n' "$1"; }+red() { printf '\033[31m%s\033[0m\n' "$1"; }+pass() { green "PASS: $1"; PASS=$((PASS + 1)); }+fail() { red "FAIL: $1"; FAIL=$((FAIL + 1)); }+info() { printf '\033[2m%s\033[0m\n' "$1"; }++[ -x "$LPD" ] || { red "lightpanda binary not found or not executable: $LPD"; exit 2; }++TMP="$(mktemp -d)"+SRV=""+cleanup() {+ [ -n "$SRV" ] && kill "$SRV" 2>/dev/null+ rm -rf "$TMP"+}+trap cleanup EXIT++start_server() {+ ( cd "$HERE/fixtures" && exec python3 -m http.server "$PORT" --bind 127.0.0.1 ) >/dev/null 2>&1 &+ SRV=$!+ for _ in $(seq 1 25); do+ curl -sf "$BASE/hn/front.html" -o /dev/null && return 0+ sleep 0.2+ done+ red "static server failed to start on $BASE"; exit 2+}++# --- deterministic layer -----------------------------------------------------+run_deterministic() {+ info "== deterministic layer (no API key) =="+ if ! "$LPD" agent "$HERE/scripts/hn-front.js" >"$TMP/out" 2>/dev/null; then+ fail "hn-front.js replay (non-zero exit)"; return+ fi+ if ! jq -e . "$TMP/out" >/dev/null 2>&1; then+ fail "hn-front.js replay (output is not valid JSON)"; return+ fi+ local d+ if d="$(diff <(jq -S . "$HERE/golden/hn-front.json") <(jq -S . "$TMP/out"))"; then+ pass "hn-front.js replay matches golden"+ else+ fail "hn-front.js replay differs from golden/hn-front.json"+ info " diff (golden < , actual > ):"+ printf '%s\n' "$d" | sed 's/^/ /'+ fi+}++# Grep the stable "$usage total=N" line lightpanda prints to stderr and fail+# if a single task blew past the token ceiling (runaway agent loop).+check_usage() {+ local errfile="$1" label="$2" total+ total="$(sed -n '/^\$usage /{s/.*total=\([0-9]\+\).*/\1/p;q}' "$errfile")"+ [ -n "$total" ] && info " usage: total=${total} tokens ($label)"+ if [ -n "$total" ] && [ "$total" -gt "$MAX_TOKENS" ]; then+ fail "$label exceeded token ceiling ($total > $MAX_TOKENS)"+ fi+}++# --- live layer --------------------------------------------------------------+run_live_qa() {+ info "== live layer: static Q&A (model=$LP_MODEL) =="+ while IFS=$'\t' read -r task expected; do+ [ -z "${task// }" ] && continue+ case "$task" in \#*) continue ;; esac+ timeout 300 "$LPD" agent --provider gemini --model "$LP_MODEL" --task "$task" >"$TMP/out" 2>"$TMP/err"+ if grep -qiF "$expected" "$TMP/out"; then+ pass "Q&A: expected \"$expected\""+ else+ fail "Q&A: expected \"$expected\" not found in answer"+ info " answer: $(tr '\n' ' ' <"$TMP/out" | cut -c1-200)"+ fi+ check_usage "$TMP/err" "Q&A \"$expected\""+ done <"$HERE/cases/static-qa.tsv"+}++run_live_hn() {+ info "== live layer: HN save + replay (model=$LP_MODEL) =="+ local script="$TMP/hn-live.js" task+ task="$(cat "$HERE/cases/hn-live.task")"++ local proxy_args=()+ [ -n "${LP_HTTP_PROXY:-}" ] && proxy_args=(--http-proxy "$LP_HTTP_PROXY")++ timeout 900 "$LPD" agent --provider gemini --model "$LP_MODEL" "${proxy_args[@]}" --task "$task" --save "$script" >/dev/null 2>"$TMP/err"+ check_usage "$TMP/err" "HN save"++ if [ ! -s "$script" ]; then+ fail "HN save produced no script"; return+ fi+ # A real replay script drives the page — it must navigate and extract.+ if grep -q 'goto(' "$script" && grep -q 'extract(' "$script"; then+ pass "HN saved script looks replayable (has goto + extract)"+ else+ fail "HN saved script missing goto/extract — see below"+ sed 's/^/ /' "$script"+ fi++ # Replay token-free (no --provider/--task => no LLM).+ if ! timeout 300 "$LPD" agent "$script" >"$TMP/out" 2>/dev/null; then+ fail "HN saved script failed on replay"; return+ fi+ if jq -e -f "$HERE/schemas/hn-live.jq" "$TMP/out" >/dev/null 2>&1; then+ pass "HN replay output satisfies shape invariant"+ else+ fail "HN replay output violates schemas/hn-live.jq"+ info " output: $(tr '\n' ' ' <"$TMP/out" | cut -c1-300)"+ fi+}++# --- dispatch ----------------------------------------------------------------+start_server++case "$LAYER" in+ deterministic) run_deterministic ;;+ live)+ [ -n "${GOOGLE_API_KEY:-}${GEMINI_API_KEY:-}" ] || { red "GOOGLE_API_KEY/GEMINI_API_KEY unset — cannot run live layer"; exit 2; }+ run_live_qa; run_live_hn ;;+ all)+ run_deterministic+ if [ -n "${GOOGLE_API_KEY:-}${GEMINI_API_KEY:-}" ]; then+ run_live_qa; run_live_hn+ else+ info "GOOGLE_API_KEY/GEMINI_API_KEY unset — skipping live layer"+ fi ;;+ update-golden)+ "$LPD" agent "$HERE/scripts/hn-front.js" 2>/dev/null | jq -S . >"$HERE/golden/hn-front.json"+ info "golden/hn-front.json regenerated — review the diff before committing"+ exit 0 ;;+ *) red "unknown layer: $LAYER (use deterministic|live|all|update-golden)"; exit 2 ;;+esac++echo+echo "-------------------------------------"+if [ "$FAIL" -eq 0 ]; then+ green "SUMMARY: $PASS passed, 0 failed"+ exit 0+else+ red "SUMMARY: $PASS passed, $FAIL failed"+ exit 1+fidiff --git a/test/agent/schemas/hn-live.jq b/test/agent/schemas/hn-live.jqnew file mode 100644index 000000000..c65884f79--- /dev/null+++ b/test/agent/schemas/hn-live.jq@@ -0,0 +1,24 @@+# Invariant check for the live Hacker News agent output.+#+# The output contract lives in two co-located halves: cases/hn-live.task pins+# what the agent must return, this file verifies it. Edit them together.+#+# We cannot assert exact values (live site + LLM both vary run to run), so we+# assert the *shape*: a JSON array of exactly 5 stories, each with a non-empty+# `title` string and a `comments` array of 0..3 objects, each comment having+# non-empty `user` and `text` strings. A fresh story may legitimately have no+# comments yet, but all 5 empty means extraction is broken — so at least one+# story must have a comment.+#+# Evaluates to true iff every invariant holds. Use with `jq -e -f` so the+# process exit code reflects the result.+def nonempty_string: type == "string" and (length > 0);++(type == "array")+and (length == 5)+and all(.[];+ (.title | nonempty_string)+ and (.comments | type == "array" and length <= 3)+ and (.comments | all(.[]; (.user | nonempty_string) and (.text | nonempty_string)))+)+and any(.[]; .comments | length > 0)diff --git a/test/agent/scripts/hn-front.js b/test/agent/scripts/hn-front.jsnew file mode 100644index 000000000..062b5d247--- /dev/null+++ b/test/agent/scripts/hn-front.js@@ -0,0 +1,46 @@+// Deterministic golden script: top 5 stories from the frozen HN front-page+// fixture, then the top 3 comments of each story's frozen item page.+// Navigates to the local fixture server instead of news.ycombinator.com, so+// replay is fully reproducible with no LLM and no network. Keep in sync with+// golden/hn-front.json; the port is pinned by run.sh.+const BASE = "http://127.0.0.1:8081/hn";++const page = new Page();+await page.goto(`${BASE}/front.html`);++const { stories } = page.extract({+ stories: [{+ selector: ".athing",+ limit: 5,+ fields: {+ rank: ".rank",+ title: ".titleline > a",+ url: { selector: ".titleline > a", attr: "href" },+ id: { selector: "", attr: "id" }+ }+ }]+});++const results = [];++for (const story of stories) {+ await page.goto(`${BASE}/item-${story.id}.html`);+ const { comments } = page.extract({+ comments: [{+ selector: ".comtr",+ limit: 3,+ fields: {+ user: ".hnuser",+ text: ".commtext"+ }+ }]+ });+ results.push({+ rank: story.rank,+ title: story.title,+ url: story.url,+ comments+ });+}++return results;