Skip to content

Lightpanda: the headless browser designed for AI and automation

test: consume agent regression suite from lightpanda-io/demo

Adrià Arrufatbd95d9a610b3
diff --git a/.github/workflows/agent-regression.yml b/.github/workflows/agent-regression.yml
index b506731eb..89dce4c7f 100644
--- a/.github/workflows/agent-regression.yml
+++ b/.github/workflows/agent-regression.yml
@@ -1,12 +1,16 @@
name: agent-regression
on:
+ # TEMPORARY: validate the demo-checkout wiring from this branch pre-merge;
+ # remove before merging.
+ push:
+ branches: [agent-regression]
schedule:
- cron: "17 3 * * *"
workflow_dispatch:
inputs:
model:
- description: "Gemini model id for the live layer (empty = suite default, see test/agent/run.sh)"
+ description: "Gemini model id for the live layer (empty = suite default, see agent/run.sh in lightpanda-io/demo)"
required: false
jobs:
@@ -46,6 +50,9 @@ jobs:
steps:
- uses: actions/checkout@v6
+ with:
+ repository: 'lightpanda-io/demo'
+ fetch-depth: 0
- name: download lightpanda release
uses: actions/download-artifact@v8
@@ -57,10 +64,10 @@ jobs:
- name: run agent regression suite (deterministic + live)
env:
- LPD: ${{ github.workspace }}/bin/lightpanda
+ LPD_PATH: ${{ github.workspace }}/bin/lightpanda
GEMINI_API_KEY: ${{ secrets.GEMINI_API_KEY }}
LP_MODEL: ${{ github.event.inputs.model }}
# Optional: news.ycombinator.com often blocks datacenter IPs. Reuse
# the residential proxy secret if present; empty is fine locally.
LP_HTTP_PROXY: ${{ secrets.MASSIVE_PROXY_RESIDENTIAL_US }}
- run: ./test/agent/run.sh all
+ run: ./agent/run.sh all
diff --git a/.github/workflows/e2e-test.yml b/.github/workflows/e2e-test.yml
index bb645759a..e8fc03476 100644
--- a/.github/workflows/e2e-test.yml
+++ b/.github/workflows/e2e-test.yml
@@ -13,7 +13,6 @@ on:
paths:
- ".github/**"
- "src/**"
- - "test/agent/**"
- "build.zig"
- "build.zig.zon"
@@ -29,7 +28,6 @@ on:
paths:
- ".github/**"
- "src/**"
- - "test/agent/**"
- "build.zig"
- "build.zig.zon"
@@ -412,8 +410,10 @@ jobs:
timeout-minutes: 5
steps:
- # Checkout THIS repo — agent fixtures, golden files and run.sh live here.
- uses: actions/checkout@v6
+ with:
+ repository: 'lightpanda-io/demo'
+ fetch-depth: 0
- name: download artifact
uses: actions/download-artifact@v8
@@ -423,13 +423,13 @@ jobs:
- run: chmod a+x bin/lightpanda
- # No API key: replays a golden PandaScript against frozen local fixtures
- # and diffs the returned JSON against golden/hn-front.json. The keyed
- # live layer runs separately in agent-regression.yml.
+ # No API key: replays golden PandaScripts against the demo sites and
+ # diffs the returned JSON against agent/golden/. The keyed live layer
+ # runs separately in agent-regression.yml.
- name: deterministic agent replay
env:
- LPD: ${{ github.workspace }}/bin/lightpanda
- run: ./test/agent/run.sh deterministic
+ LPD_PATH: ${{ github.workspace }}/bin/lightpanda
+ run: ./agent/run.sh deterministic
mcp-smoke:
name: mcp smoke
diff --git a/Makefile b/Makefile
index dcecf8ffa..7208bb9d3 100644
--- a/Makefile
+++ b/Makefile
@@ -128,12 +128,13 @@ end2end:
@test -d ../demo
cd ../demo && go run runner/main.go
-## Run the agent regression suite (LAYER=deterministic|live|all, default all).
-## The live layer needs GOOGLE_API_KEY or GEMINI_API_KEY; without one only
-## the deterministic layer runs. See test/agent/README.md.
+## Run the agent regression suite from ../demo (LAYER=deterministic|live|all,
+## default all). The live layer needs GOOGLE_API_KEY or GEMINI_API_KEY;
+## without one only the deterministic layer runs. See ../demo/agent/README.md.
test-agent:
+ @test -d ../demo
@test -x zig-out/bin/lightpanda || $(MAKE) build
- @./test/agent/run.sh $(LAYER)
+ @cd ../demo && ./agent/run.sh $(LAYER)
## Remove build artifacts (keeps .lp-cache/ and zig-pkg/ — slow to re-fetch)
clean:
diff --git a/test/agent/README.md b/test/agent/README.md
deleted file mode 100644
index a887e9888..000000000
--- a/test/agent/README.md
+++ /dev/null
@@ -1,84 +0,0 @@
-# Agent regression suite
-
-End-to-end regression tests for `lightpanda agent`. Two layers, so the cheap
-deterministic checks can gate every PR while the expensive keyed checks run
-nightly / on demand.
-
-```
-test/agent/
- run.sh orchestrator (bash + jq)
- fixtures/ static HTML served locally over HTTP
- facts/company.html closed-form Q&A page
- hn/front.html frozen HN front page
- hn/item-*.html frozen HN comment threads
- scripts/hn-front.js golden PandaScript (replays against the fixtures)
- golden/hn-front.json exact expected replay output
- cases/static-qa.tsv <task>\t<expected-substring> per line
- cases/hn-live.task task prompt for the live HN case (pins the contract)
- schemas/hn-live.jq shape invariant for the live HN output
-```
-
-## Layers
-
-### Deterministic (no API key, runs on every PR)
-
-Replays `scripts/hn-front.js` — a top-5-stories / top-3-comments HN scrape
-pointed at the local fixture server — and diffs the returned JSON **exactly**
-against `golden/hn-front.json`. This guards the replay engine and the
-`goto` / `extract` primitives. No LLM, no network.
-
-### Live (needs `GOOGLE_API_KEY` or `GEMINI_API_KEY`, runs nightly / on demand)
-
-Drives the real Gemini-backed agent:
-
-- **Static Q&A** — for each row in `cases/static-qa.tsv`, run
- `lightpanda agent --task "<question about a local fixture>"` and assert the
- expected substring appears in the answer. Closed-form answers make the
- substring match robust to LLM phrasing.
-- **HN save + replay** — ask the agent (task in `cases/hn-live.task`) to
- scrape live Hacker News and `/save` a reproducible script, then replay that
- script **token-free** and validate the output against `schemas/hn-live.jq`
- (a *shape* invariant: exactly 5 stories, non-empty titles, ≤3 comments each
- with non-empty `user`/`text`). We can't assert exact values because the live
- site and the LLM both vary — so the task prompt pins the output contract and
- the schema verifies it; edit those two files together.
-
-The stable `$usage total=N` line lightpanda prints to stderr is captured per
-task; a loose `MAX_TOKENS` ceiling flags a runaway agent loop.
-
-## Running locally
-
-```bash
-make build # produces zig-out/bin/lightpanda
-make test-agent # all layers (live skipped if no key)
-make test-agent LAYER=deterministic
-
-# or invoke run.sh directly:
-./test/agent/run.sh deterministic
-GEMINI_API_KEY=... ./test/agent/run.sh live
-```
-
-Environment knobs (see the header of `run.sh`): `LPD`, `LP_MODEL`,
-`LP_HTTP_PROXY`, `MAX_TOKENS`. The fixture-server port (8081) is part of the
-fixture contract — the golden script and TSV embed it — and is not
-configurable.
-
-## CI
-
-- `.github/workflows/e2e-test.yml` runs the **deterministic** layer on every PR
- (job `agent-deterministic`).
-- `.github/workflows/agent-regression.yml` runs **all** layers nightly and via
- *Run workflow* (`workflow_dispatch`). It uses the `GEMINI_API_KEY` repo secret;
- it reuses the `MASSIVE_PROXY_RESIDENTIAL_US` secret as `LP_HTTP_PROXY` for the
- live HN call (datacenter IPs are often blocked by news.ycombinator.com).
-
-## Maintenance
-
-- **Add a Q&A case:** append a `<task>\t<expected>` row to
- `cases/static-qa.tsv` (point the task at a `fixtures/` page).
-- **Change the deterministic scrape:** edit `scripts/hn-front.js` and/or the
- `fixtures/hn/*` pages, then regenerate the golden:
- ```bash
- ./test/agent/run.sh update-golden
- ```
- Review the diff before committing — the golden is the contract.
diff --git a/test/agent/cases/hn-live.task b/test/agent/cases/hn-live.task
deleted file mode 100644
index 30f46a4c9..000000000
--- a/test/agent/cases/hn-live.task
+++ /dev/null
@@ -1 +0,0 @@
-Go to https://news.ycombinator.com, collect the top 5 stories and the top 3 comments of each story. Return a JSON array of exactly 5 objects, each with keys "title" (the story title string) and "comments" (an array of at most 3 objects, each with keys "user" and "text"; leave it empty if the story has no comments yet).
diff --git a/test/agent/cases/static-qa.tsv b/test/agent/cases/static-qa.tsv
deleted file mode 100644
index 5291c9ee4..000000000
--- a/test/agent/cases/static-qa.tsv
+++ /dev/null
@@ -1,11 +0,0 @@
-# Static Q&A regression cases for the live LLM layer.
-# One case per line, TAB-separated: <task>\t<expected-substring>
-# The task tells the agent to visit a local fixture page and answer a
-# closed-form question; the expected substring must appear (case-insensitive)
-# in the agent's final answer on stdout. Keep answers unambiguous so LLM
-# phrasing does not matter — match on the load-bearing token only.
-# Lines starting with # and blank lines are ignored.
-Go to http://127.0.0.1:8081/facts/company.html and tell me only the year Acme Robotics was founded. 1987
-Go to http://127.0.0.1:8081/facts/company.html and tell me the city where Acme Robotics is headquartered. Reykjavik
-Go to http://127.0.0.1:8081/facts/company.html and tell me the full name of the CEO of Acme Robotics. Mira Voss
-Go to http://127.0.0.1:8081/facts/company.html and tell me the name of Acme Robotics' flagship product. Puffin 9
diff --git a/test/agent/fixtures/facts/company.html b/test/agent/fixtures/facts/company.html
deleted file mode 100644
index de21304a8..000000000
--- a/test/agent/fixtures/facts/company.html
+++ /dev/null
@@ -1,28 +0,0 @@
-<!DOCTYPE html>
-<html lang="en">
-<head>
- <meta charset="utf-8">
- <title>Acme Robotics — Company Facts</title>
-</head>
-<body>
- <main>
- <h1>Acme Robotics</h1>
- <p>Acme Robotics is a fictional company used only for automated testing.</p>
-
- <table id="facts">
- <tbody>
- <tr><th>Founded</th><td id="founded">1987</td></tr>
- <tr><th>Headquarters</th><td id="hq">Reykjavik, Iceland</td></tr>
- <tr><th>CEO</th><td id="ceo">Dr. Mira Voss</td></tr>
- <tr><th>Employees</th><td id="employees">4,213</td></tr>
- <tr><th>Flagship product</th><td id="product">The Puffin 9 autonomous drone</td></tr>
- </tbody>
- </table>
-
- <section id="motto">
- <h2>Company motto</h2>
- <p>"We build calm machines for a loud world."</p>
- </section>
- </main>
-</body>
-</html>
diff --git a/test/agent/fixtures/hn/front.html b/test/agent/fixtures/hn/front.html
deleted file mode 100644
index 4d00120b1..000000000
--- a/test/agent/fixtures/hn/front.html
+++ /dev/null
@@ -1,94 +0,0 @@
-<!DOCTYPE html>
-<html lang="en" op="news">
-<head>
- <meta charset="utf-8">
- <title>Hacker News (fixture)</title>
-</head>
-<body>
- <center>
- <table id="hnmain" border="0" cellpadding="0" cellspacing="0" width="85%">
- <tr>
- <td>
- <table border="0" cellpadding="0" cellspacing="0">
- <tr class="athing" id="101">
- <td align="right" valign="top" class="title"><span class="rank">1.</span></td>
- <td valign="top" class="votelinks"><a href="vote?id=101"><div class="votearrow" title="upvote"></div></a></td>
- <td class="title"><span class="titleline"><a href="https://example.com/1">Story One: Local-first software</a><span class="sitebit comhead"> (<a href="from?site=example.com"><span class="sitestr">example.com</span></a>)</span></span></td>
- </tr>
- <tr>
- <td colspan="2"></td>
- <td class="subtext"><span class="subline"><span class="score" id="score_101">210 points</span> by <a href="user?id=alice" class="hnuser">alice</a></span></td>
- </tr>
- <tr class="spacer" style="height:5px"></tr>
-
- <tr class="athing" id="102">
- <td align="right" valign="top" class="title"><span class="rank">2.</span></td>
- <td valign="top" class="votelinks"><a href="vote?id=102"><div class="votearrow" title="upvote"></div></a></td>
- <td class="title"><span class="titleline"><a href="https://example.com/2">Story Two: A tiny headless browser</a><span class="sitebit comhead"> (<a href="from?site=example.com"><span class="sitestr">example.com</span></a>)</span></span></td>
- </tr>
- <tr>
- <td colspan="2"></td>
- <td class="subtext"><span class="subline"><span class="score" id="score_102">184 points</span> by <a href="user?id=bob" class="hnuser">bob</a></span></td>
- </tr>
- <tr class="spacer" style="height:5px"></tr>
-
- <tr class="athing" id="103">
- <td align="right" valign="top" class="title"><span class="rank">3.</span></td>
- <td valign="top" class="votelinks"><a href="vote?id=103"><div class="votearrow" title="upvote"></div></a></td>
- <td class="title"><span class="titleline"><a href="https://example.com/3">Story Three: Zig comptime tricks</a><span class="sitebit comhead"> (<a href="from?site=example.com"><span class="sitestr">example.com</span></a>)</span></span></td>
- </tr>
- <tr>
- <td colspan="2"></td>
- <td class="subtext"><span class="subline"><span class="score" id="score_103">153 points</span> by <a href="user?id=carol" class="hnuser">carol</a></span></td>
- </tr>
- <tr class="spacer" style="height:5px"></tr>
-
- <tr class="athing" id="104">
- <td align="right" valign="top" class="title"><span class="rank">4.</span></td>
- <td valign="top" class="votelinks"><a href="vote?id=104"><div class="votearrow" title="upvote"></div></a></td>
- <td class="title"><span class="titleline"><a href="https://example.com/4">Story Four: Deterministic replay for agents</a><span class="sitebit comhead"> (<a href="from?site=example.com"><span class="sitestr">example.com</span></a>)</span></span></td>
- </tr>
- <tr>
- <td colspan="2"></td>
- <td class="subtext"><span class="subline"><span class="score" id="score_104">121 points</span> by <a href="user?id=dave" class="hnuser">dave</a></span></td>
- </tr>
- <tr class="spacer" style="height:5px"></tr>
-
- <tr class="athing" id="105">
- <td align="right" valign="top" class="title"><span class="rank">5.</span></td>
- <td valign="top" class="votelinks"><a href="vote?id=105"><div class="votearrow" title="upvote"></div></a></td>
- <td class="title"><span class="titleline"><a href="https://example.com/5">Story Five: The end of CDP overhead</a><span class="sitebit comhead"> (<a href="from?site=example.com"><span class="sitestr">example.com</span></a>)</span></span></td>
- </tr>
- <tr>
- <td colspan="2"></td>
- <td class="subtext"><span class="subline"><span class="score" id="score_105">98 points</span> by <a href="user?id=erin" class="hnuser">erin</a></span></td>
- </tr>
- <tr class="spacer" style="height:5px"></tr>
-
- <tr class="athing" id="106">
- <td align="right" valign="top" class="title"><span class="rank">6.</span></td>
- <td valign="top" class="votelinks"><a href="vote?id=106"><div class="votearrow" title="upvote"></div></a></td>
- <td class="title"><span class="titleline"><a href="https://example.com/6">Story Six: Should not be scraped</a></span></td>
- </tr>
- <tr>
- <td colspan="2"></td>
- <td class="subtext"><span class="subline"><span class="score" id="score_106">80 points</span> by <a href="user?id=frank" class="hnuser">frank</a></span></td>
- </tr>
- <tr class="spacer" style="height:5px"></tr>
-
- <tr class="athing" id="107">
- <td align="right" valign="top" class="title"><span class="rank">7.</span></td>
- <td valign="top" class="votelinks"><a href="vote?id=107"><div class="votearrow" title="upvote"></div></a></td>
- <td class="title"><span class="titleline"><a href="https://example.com/7">Story Seven: Also should not be scraped</a></span></td>
- </tr>
- <tr>
- <td colspan="2"></td>
- <td class="subtext"><span class="subline"><span class="score" id="score_107">61 points</span> by <a href="user?id=grace" class="hnuser">grace</a></span></td>
- </tr>
- </table>
- </td>
- </tr>
- </table>
- </center>
-</body>
-</html>
diff --git a/test/agent/fixtures/hn/item-101.html b/test/agent/fixtures/hn/item-101.html
deleted file mode 100644
index 3ad817be6..000000000
--- a/test/agent/fixtures/hn/item-101.html
+++ /dev/null
@@ -1,105 +0,0 @@
-<!DOCTYPE html>
-<html lang="en" op="item">
-<head>
- <meta charset="utf-8">
- <title>Story One: Local-first software | Hacker News (fixture)</title>
-</head>
-<body>
- <center>
- <table id="hnmain" border="0" cellpadding="0" cellspacing="0" width="85%">
- <tr>
- <td>
- <table class="fatitem" border="0">
- <tr class="athing" id="101">
- <td class="title"><span class="titleline"><a href="https://example.com/1">Story One: Local-first software</a></span></td>
- </tr>
- </table>
- <br>
- <table border="0" class="comment-tree">
- <tr class="athing comtr" id="c1011">
- <td>
- <table border="0">
- <tr>
- <td class="default">
- <div class="comhead">
- <a href="user?id=user_101_1" class="hnuser">user_101_1</a>
- <span class="age" title="fixture"><a href="item?id=101">1 hours ago</a></span>
- </div>
- <br>
- <div class="comment"><div class="commtext c00">Comment 1 on story 101. This is deterministic fixture text.</div></div>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- <tr class="athing comtr" id="c1012">
- <td>
- <table border="0">
- <tr>
- <td class="default">
- <div class="comhead">
- <a href="user?id=user_101_2" class="hnuser">user_101_2</a>
- <span class="age" title="fixture"><a href="item?id=101">2 hours ago</a></span>
- </div>
- <br>
- <div class="comment"><div class="commtext c00">Comment 2 on story 101. This is deterministic fixture text.</div></div>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- <tr class="athing comtr" id="c1013">
- <td>
- <table border="0">
- <tr>
- <td class="default">
- <div class="comhead">
- <a href="user?id=user_101_3" class="hnuser">user_101_3</a>
- <span class="age" title="fixture"><a href="item?id=101">3 hours ago</a></span>
- </div>
- <br>
- <div class="comment"><div class="commtext c00">Comment 3 on story 101. This is deterministic fixture text.</div></div>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- <tr class="athing comtr" id="c1014">
- <td>
- <table border="0">
- <tr>
- <td class="default">
- <div class="comhead">
- <a href="user?id=user_101_4" class="hnuser">user_101_4</a>
- <span class="age" title="fixture"><a href="item?id=101">4 hours ago</a></span>
- </div>
- <br>
- <div class="comment"><div class="commtext c00">Comment 4 on story 101. This is deterministic fixture text.</div></div>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- <tr class="athing comtr" id="c1015">
- <td>
- <table border="0">
- <tr>
- <td class="default">
- <div class="comhead">
- <a href="user?id=user_101_5" class="hnuser">user_101_5</a>
- <span class="age" title="fixture"><a href="item?id=101">5 hours ago</a></span>
- </div>
- <br>
- <div class="comment"><div class="commtext c00">Comment 5 on story 101. This is deterministic fixture text.</div></div>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- </table>
- </center>
-</body>
-</html>
diff --git a/test/agent/fixtures/hn/item-102.html b/test/agent/fixtures/hn/item-102.html
deleted file mode 100644
index a4650eb63..000000000
--- a/test/agent/fixtures/hn/item-102.html
+++ /dev/null
@@ -1,105 +0,0 @@
-<!DOCTYPE html>
-<html lang="en" op="item">
-<head>
- <meta charset="utf-8">
- <title>Story Two: A tiny headless browser | Hacker News (fixture)</title>
-</head>
-<body>
- <center>
- <table id="hnmain" border="0" cellpadding="0" cellspacing="0" width="85%">
- <tr>
- <td>
- <table class="fatitem" border="0">
- <tr class="athing" id="102">
- <td class="title"><span class="titleline"><a href="https://example.com/2">Story Two: A tiny headless browser</a></span></td>
- </tr>
- </table>
- <br>
- <table border="0" class="comment-tree">
- <tr class="athing comtr" id="c1021">
- <td>
- <table border="0">
- <tr>
- <td class="default">
- <div class="comhead">
- <a href="user?id=user_102_1" class="hnuser">user_102_1</a>
- <span class="age" title="fixture"><a href="item?id=102">1 hours ago</a></span>
- </div>
- <br>
- <div class="comment"><div class="commtext c00">Comment 1 on story 102. This is deterministic fixture text.</div></div>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- <tr class="athing comtr" id="c1022">
- <td>
- <table border="0">
- <tr>
- <td class="default">
- <div class="comhead">
- <a href="user?id=user_102_2" class="hnuser">user_102_2</a>
- <span class="age" title="fixture"><a href="item?id=102">2 hours ago</a></span>
- </div>
- <br>
- <div class="comment"><div class="commtext c00">Comment 2 on story 102. This is deterministic fixture text.</div></div>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- <tr class="athing comtr" id="c1023">
- <td>
- <table border="0">
- <tr>
- <td class="default">
- <div class="comhead">
- <a href="user?id=user_102_3" class="hnuser">user_102_3</a>
- <span class="age" title="fixture"><a href="item?id=102">3 hours ago</a></span>
- </div>
- <br>
- <div class="comment"><div class="commtext c00">Comment 3 on story 102. This is deterministic fixture text.</div></div>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- <tr class="athing comtr" id="c1024">
- <td>
- <table border="0">
- <tr>
- <td class="default">
- <div class="comhead">
- <a href="user?id=user_102_4" class="hnuser">user_102_4</a>
- <span class="age" title="fixture"><a href="item?id=102">4 hours ago</a></span>
- </div>
- <br>
- <div class="comment"><div class="commtext c00">Comment 4 on story 102. This is deterministic fixture text.</div></div>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- <tr class="athing comtr" id="c1025">
- <td>
- <table border="0">
- <tr>
- <td class="default">
- <div class="comhead">
- <a href="user?id=user_102_5" class="hnuser">user_102_5</a>
- <span class="age" title="fixture"><a href="item?id=102">5 hours ago</a></span>
- </div>
- <br>
- <div class="comment"><div class="commtext c00">Comment 5 on story 102. This is deterministic fixture text.</div></div>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- </table>
- </center>
-</body>
-</html>
diff --git a/test/agent/fixtures/hn/item-103.html b/test/agent/fixtures/hn/item-103.html
deleted file mode 100644
index 7ae4af487..000000000
--- a/test/agent/fixtures/hn/item-103.html
+++ /dev/null
@@ -1,105 +0,0 @@
-<!DOCTYPE html>
-<html lang="en" op="item">
-<head>
- <meta charset="utf-8">
- <title>Story Three: Zig comptime tricks | Hacker News (fixture)</title>
-</head>
-<body>
- <center>
- <table id="hnmain" border="0" cellpadding="0" cellspacing="0" width="85%">
- <tr>
- <td>
- <table class="fatitem" border="0">
- <tr class="athing" id="103">
- <td class="title"><span class="titleline"><a href="https://example.com/3">Story Three: Zig comptime tricks</a></span></td>
- </tr>
- </table>
- <br>
- <table border="0" class="comment-tree">
- <tr class="athing comtr" id="c1031">
- <td>
- <table border="0">
- <tr>
- <td class="default">
- <div class="comhead">
- <a href="user?id=user_103_1" class="hnuser">user_103_1</a>
- <span class="age" title="fixture"><a href="item?id=103">1 hours ago</a></span>
- </div>
- <br>
- <div class="comment"><div class="commtext c00">Comment 1 on story 103. This is deterministic fixture text.</div></div>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- <tr class="athing comtr" id="c1032">
- <td>
- <table border="0">
- <tr>
- <td class="default">
- <div class="comhead">
- <a href="user?id=user_103_2" class="hnuser">user_103_2</a>
- <span class="age" title="fixture"><a href="item?id=103">2 hours ago</a></span>
- </div>
- <br>
- <div class="comment"><div class="commtext c00">Comment 2 on story 103. This is deterministic fixture text.</div></div>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- <tr class="athing comtr" id="c1033">
- <td>
- <table border="0">
- <tr>
- <td class="default">
- <div class="comhead">
- <a href="user?id=user_103_3" class="hnuser">user_103_3</a>
- <span class="age" title="fixture"><a href="item?id=103">3 hours ago</a></span>
- </div>
- <br>
- <div class="comment"><div class="commtext c00">Comment 3 on story 103. This is deterministic fixture text.</div></div>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- <tr class="athing comtr" id="c1034">
- <td>
- <table border="0">
- <tr>
- <td class="default">
- <div class="comhead">
- <a href="user?id=user_103_4" class="hnuser">user_103_4</a>
- <span class="age" title="fixture"><a href="item?id=103">4 hours ago</a></span>
- </div>
- <br>
- <div class="comment"><div class="commtext c00">Comment 4 on story 103. This is deterministic fixture text.</div></div>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- <tr class="athing comtr" id="c1035">
- <td>
- <table border="0">
- <tr>
- <td class="default">
- <div class="comhead">
- <a href="user?id=user_103_5" class="hnuser">user_103_5</a>
- <span class="age" title="fixture"><a href="item?id=103">5 hours ago</a></span>
- </div>
- <br>
- <div class="comment"><div class="commtext c00">Comment 5 on story 103. This is deterministic fixture text.</div></div>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- </table>
- </center>
-</body>
-</html>
diff --git a/test/agent/fixtures/hn/item-104.html b/test/agent/fixtures/hn/item-104.html
deleted file mode 100644
index 9b45fb4db..000000000
--- a/test/agent/fixtures/hn/item-104.html
+++ /dev/null
@@ -1,105 +0,0 @@
-<!DOCTYPE html>
-<html lang="en" op="item">
-<head>
- <meta charset="utf-8">
- <title>Story Four: Deterministic replay for agents | Hacker News (fixture)</title>
-</head>
-<body>
- <center>
- <table id="hnmain" border="0" cellpadding="0" cellspacing="0" width="85%">
- <tr>
- <td>
- <table class="fatitem" border="0">
- <tr class="athing" id="104">
- <td class="title"><span class="titleline"><a href="https://example.com/4">Story Four: Deterministic replay for agents</a></span></td>
- </tr>
- </table>
- <br>
- <table border="0" class="comment-tree">
- <tr class="athing comtr" id="c1041">
- <td>
- <table border="0">
- <tr>
- <td class="default">
- <div class="comhead">
- <a href="user?id=user_104_1" class="hnuser">user_104_1</a>
- <span class="age" title="fixture"><a href="item?id=104">1 hours ago</a></span>
- </div>
- <br>
- <div class="comment"><div class="commtext c00">Comment 1 on story 104. This is deterministic fixture text.</div></div>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- <tr class="athing comtr" id="c1042">
- <td>
- <table border="0">
- <tr>
- <td class="default">
- <div class="comhead">
- <a href="user?id=user_104_2" class="hnuser">user_104_2</a>
- <span class="age" title="fixture"><a href="item?id=104">2 hours ago</a></span>
- </div>
- <br>
- <div class="comment"><div class="commtext c00">Comment 2 on story 104. This is deterministic fixture text.</div></div>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- <tr class="athing comtr" id="c1043">
- <td>
- <table border="0">
- <tr>
- <td class="default">
- <div class="comhead">
- <a href="user?id=user_104_3" class="hnuser">user_104_3</a>
- <span class="age" title="fixture"><a href="item?id=104">3 hours ago</a></span>
- </div>
- <br>
- <div class="comment"><div class="commtext c00">Comment 3 on story 104. This is deterministic fixture text.</div></div>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- <tr class="athing comtr" id="c1044">
- <td>
- <table border="0">
- <tr>
- <td class="default">
- <div class="comhead">
- <a href="user?id=user_104_4" class="hnuser">user_104_4</a>
- <span class="age" title="fixture"><a href="item?id=104">4 hours ago</a></span>
- </div>
- <br>
- <div class="comment"><div class="commtext c00">Comment 4 on story 104. This is deterministic fixture text.</div></div>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- <tr class="athing comtr" id="c1045">
- <td>
- <table border="0">
- <tr>
- <td class="default">
- <div class="comhead">
- <a href="user?id=user_104_5" class="hnuser">user_104_5</a>
- <span class="age" title="fixture"><a href="item?id=104">5 hours ago</a></span>
- </div>
- <br>
- <div class="comment"><div class="commtext c00">Comment 5 on story 104. This is deterministic fixture text.</div></div>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- </table>
- </center>
-</body>
-</html>
diff --git a/test/agent/fixtures/hn/item-105.html b/test/agent/fixtures/hn/item-105.html
deleted file mode 100644
index 304a0d95b..000000000
--- a/test/agent/fixtures/hn/item-105.html
+++ /dev/null
@@ -1,105 +0,0 @@
-<!DOCTYPE html>
-<html lang="en" op="item">
-<head>
- <meta charset="utf-8">
- <title>Story Five: The end of CDP overhead | Hacker News (fixture)</title>
-</head>
-<body>
- <center>
- <table id="hnmain" border="0" cellpadding="0" cellspacing="0" width="85%">
- <tr>
- <td>
- <table class="fatitem" border="0">
- <tr class="athing" id="105">
- <td class="title"><span class="titleline"><a href="https://example.com/5">Story Five: The end of CDP overhead</a></span></td>
- </tr>
- </table>
- <br>
- <table border="0" class="comment-tree">
- <tr class="athing comtr" id="c1051">
- <td>
- <table border="0">
- <tr>
- <td class="default">
- <div class="comhead">
- <a href="user?id=user_105_1" class="hnuser">user_105_1</a>
- <span class="age" title="fixture"><a href="item?id=105">1 hours ago</a></span>
- </div>
- <br>
- <div class="comment"><div class="commtext c00">Comment 1 on story 105. This is deterministic fixture text.</div></div>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- <tr class="athing comtr" id="c1052">
- <td>
- <table border="0">
- <tr>
- <td class="default">
- <div class="comhead">
- <a href="user?id=user_105_2" class="hnuser">user_105_2</a>
- <span class="age" title="fixture"><a href="item?id=105">2 hours ago</a></span>
- </div>
- <br>
- <div class="comment"><div class="commtext c00">Comment 2 on story 105. This is deterministic fixture text.</div></div>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- <tr class="athing comtr" id="c1053">
- <td>
- <table border="0">
- <tr>
- <td class="default">
- <div class="comhead">
- <a href="user?id=user_105_3" class="hnuser">user_105_3</a>
- <span class="age" title="fixture"><a href="item?id=105">3 hours ago</a></span>
- </div>
- <br>
- <div class="comment"><div class="commtext c00">Comment 3 on story 105. This is deterministic fixture text.</div></div>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- <tr class="athing comtr" id="c1054">
- <td>
- <table border="0">
- <tr>
- <td class="default">
- <div class="comhead">
- <a href="user?id=user_105_4" class="hnuser">user_105_4</a>
- <span class="age" title="fixture"><a href="item?id=105">4 hours ago</a></span>
- </div>
- <br>
- <div class="comment"><div class="commtext c00">Comment 4 on story 105. This is deterministic fixture text.</div></div>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- <tr class="athing comtr" id="c1055">
- <td>
- <table border="0">
- <tr>
- <td class="default">
- <div class="comhead">
- <a href="user?id=user_105_5" class="hnuser">user_105_5</a>
- <span class="age" title="fixture"><a href="item?id=105">5 hours ago</a></span>
- </div>
- <br>
- <div class="comment"><div class="commtext c00">Comment 5 on story 105. This is deterministic fixture text.</div></div>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- </table>
- </td>
- </tr>
- </table>
- </center>
-</body>
-</html>
diff --git a/test/agent/golden/hn-front.json b/test/agent/golden/hn-front.json
deleted file mode 100644
index 61d05386f..000000000
--- a/test/agent/golden/hn-front.json
+++ /dev/null
@@ -1,97 +0,0 @@
-[
- {
- "comments": [
- {
- "text": "Comment 1 on story 101. This is deterministic fixture text.",
- "user": "user_101_1"
- },
- {
- "text": "Comment 2 on story 101. This is deterministic fixture text.",
- "user": "user_101_2"
- },
- {
- "text": "Comment 3 on story 101. This is deterministic fixture text.",
- "user": "user_101_3"
- }
- ],
- "rank": "1.",
- "title": "Story One: Local-first software",
- "url": "https://example.com/1"
- },
- {
- "comments": [
- {
- "text": "Comment 1 on story 102. This is deterministic fixture text.",
- "user": "user_102_1"
- },
- {
- "text": "Comment 2 on story 102. This is deterministic fixture text.",
- "user": "user_102_2"
- },
- {
- "text": "Comment 3 on story 102. This is deterministic fixture text.",
- "user": "user_102_3"
- }
- ],
- "rank": "2.",
- "title": "Story Two: A tiny headless browser",
- "url": "https://example.com/2"
- },
- {
- "comments": [
- {
- "text": "Comment 1 on story 103. This is deterministic fixture text.",
- "user": "user_103_1"
- },
- {
- "text": "Comment 2 on story 103. This is deterministic fixture text.",
- "user": "user_103_2"
- },
- {
- "text": "Comment 3 on story 103. This is deterministic fixture text.",
- "user": "user_103_3"
- }
- ],
- "rank": "3.",
- "title": "Story Three: Zig comptime tricks",
- "url": "https://example.com/3"
- },
- {
- "comments": [
- {
- "text": "Comment 1 on story 104. This is deterministic fixture text.",
- "user": "user_104_1"
- },
- {
- "text": "Comment 2 on story 104. This is deterministic fixture text.",
- "user": "user_104_2"
- },
- {
- "text": "Comment 3 on story 104. This is deterministic fixture text.",
- "user": "user_104_3"
- }
- ],
- "rank": "4.",
- "title": "Story Four: Deterministic replay for agents",
- "url": "https://example.com/4"
- },
- {
- "comments": [
- {
- "text": "Comment 1 on story 105. This is deterministic fixture text.",
- "user": "user_105_1"
- },
- {
- "text": "Comment 2 on story 105. This is deterministic fixture text.",
- "user": "user_105_2"
- },
- {
- "text": "Comment 3 on story 105. This is deterministic fixture text.",
- "user": "user_105_3"
- }
- ],
- "rank": "5.",
- "title": "Story Five: The end of CDP overhead",
- "url": "https://example.com/5"
- }
-]
diff --git a/test/agent/run.sh b/test/agent/run.sh
deleted file mode 100755
index 1734b64e6..000000000
--- a/test/agent/run.sh
+++ /dev/null
@@ -1,199 +0,0 @@
-#!/usr/bin/env bash
-#
-# Agent regression suite. Two layers:
-#
-# deterministic — replay a golden PandaScript against frozen local HTML
-# fixtures and compare the returned JSON exactly to a
-# golden file. No API key, no network. Runs on every PR.
-#
-# live — drive the real LLM agent (needs GOOGLE_API_KEY):
-# * static Q&A: ask closed-form questions about a local
-# fixture page, substring-match the answer.
-# * HN save+replay: ask the agent to scrape live Hacker
-# News and /save a reproducible script, then replay it
-# token-free and validate the output against a shape
-# invariant (jq), not exact values.
-#
-# Usage: test/agent/run.sh [deterministic|live|all|update-golden] (default: all)
-#
-# update-golden regenerates golden/hn-front.json from the current fixtures and
-# script — review the diff before committing.
-#
-# Env:
-# LPD path to the lightpanda binary (default: zig-out/bin/lightpanda)
-# GOOGLE_API_KEY or GEMINI_API_KEY (the binary accepts both) — required for
-# the live layer; live layer is skipped if neither is set
-# LP_MODEL Gemini model id for the live layer (default below)
-# LP_HTTP_PROXY optional proxy for the live HN call only (datacenter IPs are
-# often blocked by news.ycombinator.com); localhost fixtures
-# are never proxied
-# MAX_TOKENS per-live-task total-token ceiling (default: 3000000).
-# `total` counts cached reads, so a normal HN save is ~1M;
-# this is a loose backstop against a runaway agent loop.
-set -uo pipefail
-
-LAYER="${1:-all}"
-
-HERE="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
-REPO="$(cd "$HERE/../.." && pwd)"
-LPD="${LPD:-$REPO/zig-out/bin/lightpanda}"
-# The port is part of the fixture contract: scripts/hn-front.js, the golden
-# file, and cases/static-qa.tsv all embed it. Not configurable.
-PORT=8081
-BASE="http://127.0.0.1:${PORT}"
-# Pin an explicit model id — never a *-latest / *-preview alias, which drift.
-LP_MODEL="${LP_MODEL:-gemini-3.5-flash}"
-MAX_TOKENS="${MAX_TOKENS:-3000000}"
-
-export LIGHTPANDA_DISABLE_TELEMETRY=true
-
-PASS=0
-FAIL=0
-green() { printf '\033[32m%s\033[0m\n' "$1"; }
-red() { printf '\033[31m%s\033[0m\n' "$1"; }
-pass() { green "PASS: $1"; PASS=$((PASS + 1)); }
-fail() { red "FAIL: $1"; FAIL=$((FAIL + 1)); }
-info() { printf '\033[2m%s\033[0m\n' "$1"; }
-
-[ -x "$LPD" ] || { red "lightpanda binary not found or not executable: $LPD"; exit 2; }
-
-TMP="$(mktemp -d)"
-SRV=""
-cleanup() {
- [ -n "$SRV" ] && kill "$SRV" 2>/dev/null
- rm -rf "$TMP"
-}
-trap cleanup EXIT
-
-start_server() {
- ( cd "$HERE/fixtures" && exec python3 -m http.server "$PORT" --bind 127.0.0.1 ) >/dev/null 2>&1 &
- SRV=$!
- for _ in $(seq 1 25); do
- curl -sf "$BASE/hn/front.html" -o /dev/null && return 0
- sleep 0.2
- done
- red "static server failed to start on $BASE"; exit 2
-}
-
-# --- deterministic layer -----------------------------------------------------
-run_deterministic() {
- info "== deterministic layer (no API key) =="
- if ! "$LPD" agent "$HERE/scripts/hn-front.js" >"$TMP/out" 2>/dev/null; then
- fail "hn-front.js replay (non-zero exit)"; return
- fi
- if ! jq -e . "$TMP/out" >/dev/null 2>&1; then
- fail "hn-front.js replay (output is not valid JSON)"; return
- fi
- local d
- if d="$(diff <(jq -S . "$HERE/golden/hn-front.json") <(jq -S . "$TMP/out"))"; then
- pass "hn-front.js replay matches golden"
- else
- fail "hn-front.js replay differs from golden/hn-front.json"
- info " diff (golden < , actual > ):"
- printf '%s\n' "$d" | sed 's/^/ /'
- fi
-}
-
-# Surface the agent's own error output on failure — an instant empty answer
-# usually means an API error (bad key, quota, restriction) that only appears
-# on stderr.
-show_err() {
- sed -n '/^\$usage /!p' "$1" | tail -3 | sed 's/^/ /'
-}
-
-# Grep the stable "$usage total=N" line lightpanda prints to stderr and fail
-# if a single task blew past the token ceiling (runaway agent loop).
-check_usage() {
- local errfile="$1" label="$2" total
- total="$(sed -n '/^\$usage /{s/.*total=\([0-9]\+\).*/\1/p;q}' "$errfile")"
- [ -n "$total" ] && info " usage: total=${total} tokens ($label)"
- if [ -n "$total" ] && [ "$total" -gt "$MAX_TOKENS" ]; then
- fail "$label exceeded token ceiling ($total > $MAX_TOKENS)"
- fi
-}
-
-# --- live layer --------------------------------------------------------------
-run_live_qa() {
- info "== live layer: static Q&A (model=$LP_MODEL) =="
- while IFS=$'\t' read -r task expected; do
- [ -z "${task// }" ] && continue
- case "$task" in \#*) continue ;; esac
- timeout 300 "$LPD" agent --provider gemini --model "$LP_MODEL" --task "$task" >"$TMP/out" 2>"$TMP/err"
- if grep -qiF "$expected" "$TMP/out"; then
- pass "Q&A: expected \"$expected\""
- else
- fail "Q&A: expected \"$expected\" not found in answer"
- info " answer: $(tr '\n' ' ' <"$TMP/out" | cut -c1-200)"
- show_err "$TMP/err"
- fi
- check_usage "$TMP/err" "Q&A \"$expected\""
- done <"$HERE/cases/static-qa.tsv"
-}
-
-run_live_hn() {
- info "== live layer: HN save + replay (model=$LP_MODEL) =="
- local script="$TMP/hn-live.js" task
- task="$(cat "$HERE/cases/hn-live.task")"
-
- local proxy_args=()
- [ -n "${LP_HTTP_PROXY:-}" ] && proxy_args=(--http-proxy "$LP_HTTP_PROXY")
-
- timeout 900 "$LPD" agent --provider gemini --model "$LP_MODEL" "${proxy_args[@]}" --task "$task" --save "$script" >/dev/null 2>"$TMP/err"
- check_usage "$TMP/err" "HN save"
-
- if [ ! -s "$script" ]; then
- fail "HN save produced no script"
- show_err "$TMP/err"
- return
- fi
- # A real replay script drives the page — it must navigate and extract.
- if grep -q 'goto(' "$script" && grep -q 'extract(' "$script"; then
- pass "HN saved script looks replayable (has goto + extract)"
- else
- fail "HN saved script missing goto/extract — see below"
- sed 's/^/ /' "$script"
- fi
-
- # Replay token-free (no --provider/--task => no LLM).
- if ! timeout 300 "$LPD" agent "$script" >"$TMP/out" 2>/dev/null; then
- fail "HN saved script failed on replay"; return
- fi
- if jq -e -f "$HERE/schemas/hn-live.jq" "$TMP/out" >/dev/null 2>&1; then
- pass "HN replay output satisfies shape invariant"
- else
- fail "HN replay output violates schemas/hn-live.jq"
- info " output: $(tr '\n' ' ' <"$TMP/out" | cut -c1-300)"
- fi
-}
-
-# --- dispatch ----------------------------------------------------------------
-start_server
-
-case "$LAYER" in
- deterministic) run_deterministic ;;
- live)
- [ -n "${GOOGLE_API_KEY:-}${GEMINI_API_KEY:-}" ] || { red "GOOGLE_API_KEY/GEMINI_API_KEY unset — cannot run live layer"; exit 2; }
- run_live_qa; run_live_hn ;;
- all)
- run_deterministic
- if [ -n "${GOOGLE_API_KEY:-}${GEMINI_API_KEY:-}" ]; then
- run_live_qa; run_live_hn
- else
- info "GOOGLE_API_KEY/GEMINI_API_KEY unset — skipping live layer"
- fi ;;
- update-golden)
- "$LPD" agent "$HERE/scripts/hn-front.js" 2>/dev/null | jq -S . >"$HERE/golden/hn-front.json"
- info "golden/hn-front.json regenerated — review the diff before committing"
- exit 0 ;;
- *) red "unknown layer: $LAYER (use deterministic|live|all|update-golden)"; exit 2 ;;
-esac
-
-echo
-echo "-------------------------------------"
-if [ "$FAIL" -eq 0 ]; then
- green "SUMMARY: $PASS passed, 0 failed"
- exit 0
-else
- red "SUMMARY: $PASS passed, $FAIL failed"
- exit 1
-fi
diff --git a/test/agent/schemas/hn-live.jq b/test/agent/schemas/hn-live.jq
deleted file mode 100644
index c65884f79..000000000
--- a/test/agent/schemas/hn-live.jq
+++ /dev/null
@@ -1,24 +0,0 @@
-# Invariant check for the live Hacker News agent output.
-#
-# The output contract lives in two co-located halves: cases/hn-live.task pins
-# what the agent must return, this file verifies it. Edit them together.
-#
-# We cannot assert exact values (live site + LLM both vary run to run), so we
-# assert the *shape*: a JSON array of exactly 5 stories, each with a non-empty
-# `title` string and a `comments` array of 0..3 objects, each comment having
-# non-empty `user` and `text` strings. A fresh story may legitimately have no
-# comments yet, but all 5 empty means extraction is broken — so at least one
-# story must have a comment.
-#
-# Evaluates to true iff every invariant holds. Use with `jq -e -f` so the
-# process exit code reflects the result.
-def nonempty_string: type == "string" and (length > 0);
-
-(type == "array")
-and (length == 5)
-and all(.[];
- (.title | nonempty_string)
- and (.comments | type == "array" and length <= 3)
- and (.comments | all(.[]; (.user | nonempty_string) and (.text | nonempty_string)))
-)
-and any(.[]; .comments | length > 0)
diff --git a/test/agent/scripts/hn-front.js b/test/agent/scripts/hn-front.js
deleted file mode 100644
index 062b5d247..000000000
--- a/test/agent/scripts/hn-front.js
+++ /dev/null
@@ -1,46 +0,0 @@
-// Deterministic golden script: top 5 stories from the frozen HN front-page
-// fixture, then the top 3 comments of each story's frozen item page.
-// Navigates to the local fixture server instead of news.ycombinator.com, so
-// replay is fully reproducible with no LLM and no network. Keep in sync with
-// golden/hn-front.json; the port is pinned by run.sh.
-const BASE = "http://127.0.0.1:8081/hn";
-
-const page = new Page();
-await page.goto(`${BASE}/front.html`);
-
-const { stories } = page.extract({
- stories: [{
- selector: ".athing",
- limit: 5,
- fields: {
- rank: ".rank",
- title: ".titleline > a",
- url: { selector: ".titleline > a", attr: "href" },
- id: { selector: "", attr: "id" }
- }
- }]
-});
-
-const results = [];
-
-for (const story of stories) {
- await page.goto(`${BASE}/item-${story.id}.html`);
- const { comments } = page.extract({
- comments: [{
- selector: ".comtr",
- limit: 3,
- fields: {
- user: ".hnuser",
- text: ".commtext"
- }
- }]
- });
- results.push({
- rank: story.rank,
- title: story.title,
- url: story.url,
- comments
- });
-}
-
-return results;