High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
test(extract): corpus e2e mirror passes article, infobox and search on a 0-hit floor; assert live hits equal the offline corpus per entry (PIN-435)
Luigi Agosti070571fad48c
diff --git a/internal/extract/corpus_test.go b/internal/extract/corpus_test.goindex d4ecef81..c87599d6 100644--- a/internal/extract/corpus_test.go+++ b/internal/extract/corpus_test.go@@ -3,6 +3,7 @@ package extractimport ("bytes""encoding/json"+ "flag""math""os""path/filepath"@@ -22,6 +23,8 @@ const (e2eManifest = "../../tests/e2e/fixtures/corpus/manifest.json")+var updateManifest = flag.Bool("update", false, "rewrite the e2e corpus manifest with the offline hits and data of every mirrored entry")+type corpusEntry struct {name stringnodes []observe.A11yNode@@ -69,7 +72,7 @@ func TestCorpus(t *testing.T) {func TestCorpus_E2EMirrorMatchesUnitCorpus(t *testing.T) {manifest := readE2EManifest(t)entries := loadCorpus(t)- if len(manifest) != len(entries) {+ if len(manifest) != len(entries) && !*updateManifest {t.Errorf("%s lists %d entries, the unit corpus has %d", e2eManifest, len(manifest), len(entries))}for _, e := range entries {@@ -77,18 +80,17 @@ func TestCorpus_E2EMirrorMatchesUnitCorpus(t *testing.T) {_, statErr := os.Stat(filepath.Join(e2eCorpusDir, e.name+".html"))mirrored := statErr == nilswitch {- case !listed:- t.Errorf("%s is missing from %s", e.name, e2eManifest)- case m.Offline != "" && m.MinHits != nil:- t.Errorf("%s is both offline-only and carries minHits", e.name)case m.Offline != "":if mirrored {t.Errorf("%s is offline-only but has an e2e HTML fixture", e.name)}- case m.MinHits == nil:- t.Errorf("%s needs either minHits or an offline reason", e.name)+ if m.Hits != nil || m.Data != nil {+ t.Errorf("%s is offline-only but carries offline hits or data", e.name)+ }+ case !mirrored && !listed:+ t.Errorf("%s is missing from %s", e.name, e2eManifest)case !mirrored:- t.Errorf("%s carries minHits but has no e2e HTML fixture", e.name)+ t.Errorf("%s needs either an e2e HTML fixture or an offline reason", e.name)default:for _, file := range []string{"schema.json", "expected.json"} {unit := mustRead(t, filepath.Join(corpusDir, e.name, file))@@ -97,11 +99,39 @@ func TestCorpus_E2EMirrorMatchesUnitCorpus(t *testing.T) {t.Errorf("%s: e2e copy of %s differs from the unit corpus", e.name, file)}}- if hits := countHits(scoreEntry(t, e)); hits < *m.MinHits {- t.Errorf("%s: offline hits %d fell below the e2e minHits %d", e.name, hits, *m.MinHits)+ data := resolveEntry(t, e)+ hits := countHits(compareObject("", data, e.expected))+ if *updateManifest {+ manifest[e.name] = e2eEntry{Hits: &hits, Data: data}+ continue}+ checkManifestEntry(t, e.name, m, hits, data)+ }+ }+ if *updateManifest {+ writeE2EManifest(t, manifest)+ }+}++func checkManifestEntry(t *testing.T, name string, m e2eEntry, hits int, data map[string]any) {+ t.Helper()+ switch {+ case m.Hits == nil:+ t.Errorf("%s: %s carries no offline hits; run go test ./internal/extract -run TestCorpus_E2EMirror -update", name, e2eManifest)+ case *m.Hits != hits:+ t.Errorf("%s: %s records %d offline hits, the offline run scores %d; run go test ./internal/extract -run TestCorpus_E2EMirror -update", name, e2eManifest, *m.Hits, hits)+ }+ if reflect.DeepEqual(m.Data, data) {+ return+ }+ reported := map[string]bool{}+ for _, o := range append(compareObject("", m.Data, data), swapped(compareObject("", data, m.Data))...) {+ if !o.hit && !reported[o.path] {+ reported[o.path] = true+ t.Errorf("%s: %s records %s=%s, the offline run extracts %s", name, e2eManifest, o.path, show(o.got), show(o.want))}}+ t.Errorf("%s: %s data differs from the offline run; run go test ./internal/extract -run TestCorpus_E2EMirror -update", name, e2eManifest)}func loadCorpus(t *testing.T) []corpusEntry {@@ -136,6 +166,11 @@ func loadCorpus(t *testing.T) []corpusEntry {}func scoreEntry(t *testing.T, e corpusEntry) []fieldOutcome {+ t.Helper()+ return compareObject("", resolveEntry(t, e), e.expected)+}++func resolveEntry(t *testing.T, e corpusEntry) map[string]any {t.Helper()raw, err := json.Marshal(Resolve(e.schema, e.nodes, Options{}).Data)if err != nil {@@ -145,7 +180,7 @@ func scoreEntry(t *testing.T, e corpusEntry) []fieldOutcome {if err := json.Unmarshal(raw, &got); err != nil {t.Fatalf("%s: decode data: %v", e.name, err)}- return compareObject("", got, e.expected)+ return got}func compareObject(prefix string, got, want map[string]any) []fieldOutcome {@@ -175,6 +210,13 @@ func compareObject(prefix string, got, want map[string]any) []fieldOutcome {return out}+func swapped(outcomes []fieldOutcome) []fieldOutcome {+ for i := range outcomes {+ outcomes[i].got, outcomes[i].want = outcomes[i].want, outcomes[i].got+ }+ return outcomes+}+func countHits(outcomes []fieldOutcome) int {hits := 0for _, o := range outcomes {@@ -220,8 +262,9 @@ func readBaseline(t *testing.T) float64 {}type e2eEntry struct {- MinHits *int `json:"minHits"`- Offline string `json:"offline"`+ Hits *int `json:"hits,omitempty"`+ Data map[string]any `json:"data,omitempty"`+ Offline string `json:"offline,omitempty"`}func readE2EManifest(t *testing.T) map[string]e2eEntry {@@ -235,6 +278,22 @@ func readE2EManifest(t *testing.T) map[string]e2eEntry {return manifest.Entries}+func writeE2EManifest(t *testing.T, entries map[string]e2eEntry) {+ t.Helper()+ var buf bytes.Buffer+ enc := json.NewEncoder(&buf)+ enc.SetEscapeHTML(false)+ enc.SetIndent("", " ")+ if err := enc.Encode(struct {+ Entries map[string]e2eEntry `json:"entries"`+ }{entries}); err != nil {+ t.Fatalf("encode %s: %v", e2eManifest, err)+ }+ if err := os.WriteFile(e2eManifest, buf.Bytes(), 0o644); err != nil {+ t.Fatalf("write %s: %v", e2eManifest, err)+ }+}+func mustRead(t *testing.T, path string) []byte {t.Helper()data, err := os.ReadFile(path)diff --git a/internal/extract/testdata/corpus/README.md b/internal/extract/testdata/corpus/README.mdindex 3db3278e..29500621 100644--- a/internal/extract/testdata/corpus/README.md+++ b/internal/extract/testdata/corpus/README.md@@ -84,13 +84,21 @@ removed, the stylesheets reduced to the rules that match the page and inlined,images replaced by a 1x1 pixel) plus byte-identical copies of `schema.json` and`expected.json`. Each copy was checked to produce the same node list (depth,role, name, text, value) as the committed snapshot. Its `manifest.json` gives-each mirrored entry a `minHits` floor, the offline hit count at commit time, and-each offline-only entry the reason it is not mirrored.-`TestCorpus_E2EMirrorMatchesUnitCorpus` keeps the copies, the manifest and the-floors in step with this directory, and the-`tests/e2e/scenarios/api/extract-corpus-extended.sh` scenario runs each mirrored-page through the live pipeline, reporting every field and failing an entry-whose hits fall below its floor.+each mirrored entry the offline run's `hits` against `expected.json` and the+`data` it extracts, and each offline-only entry the reason it is not mirrored.+`TestCorpus_E2EMirrorMatchesUnitCorpus` keeps the copies and the manifest in+step with this directory and fails when a recorded `hits` or `data` differs+from the offline run. After a change that moves the offline result, regenerate+the manifest rather than editing it:++```+go test ./internal/extract/ -run TestCorpus_E2EMirror -update+```++The `tests/e2e/scenarios/api/extract-corpus-extended.sh` scenario runs each+mirrored page through the live pipeline, reports every field, and fails an+entry whose live hits differ from the recorded `hits` or whose live `data`+differs from the recorded `data`, printing each field that drifted.## Attribution and licencesdiff --git a/tests/e2e/fixtures/corpus/manifest.json b/tests/e2e/fixtures/corpus/manifest.jsonindex 5a1a8deb..638da54b 100644--- a/tests/e2e/fixtures/corpus/manifest.json+++ b/tests/e2e/fixtures/corpus/manifest.json@@ -1,12 +1,160 @@{"entries": {- "article": { "minHits": 0 },- "infobox": { "minHits": 0 },- "listing": { "minHits": 11 },- "login": { "minHits": 2 },- "pricing": { "offline": "github.com/pricing is GitHub, Inc. page content that cannot be redistributed" },- "product": { "minHits": 1 },- "search": { "minHits": 0 },- "stories": { "offline": "the news.ycombinator.com front page is Y Combinator and user-submitted content that cannot be redistributed" }+ "article": {+ "hits": 0,+ "data": {+ "headline": "Contents",+ "published": "Published"+ }+ },+ "infobox": {+ "hits": 0,+ "data": {+ "easiestRoute": "Easiest route",+ "location": "Location",+ "name": "Contents",+ "parentRange": "Parent range"+ }+ },+ "listing": {+ "hits": 11,+ "data": {+ "books": [+ {+ "availability": "In stock"+ },+ {+ "availability": "In stock"+ },+ {+ "availability": "In stock"+ },+ {+ "availability": "In stock"+ },+ {+ "availability": "In stock"+ },+ {+ "availability": "In stock"+ },+ {+ "availability": "In stock"+ },+ {+ "availability": "In stock",+ "title": "A Year in Provence ..."+ },+ {+ "availability": "In stock"+ },+ {+ "availability": "In stock"+ },+ {+ "availability": "In stock"+ }+ ]+ }+ },+ "login": {+ "hits": 2,+ "data": {+ "passwordLabel": "Username Password",+ "quotesSource": "Quotes by:",+ "site": "Quotes to Scrape",+ "submit": "Login",+ "usernameLabel": "Username Password"+ }+ },+ "pricing": {+ "offline": "github.com/pricing is GitHub, Inc. page content that cannot be redistributed"+ },+ "product": {+ "hits": 1,+ "data": {+ "availability": "Availability",+ "description": "Product Description",+ "productType": "Product Type",+ "title": "A Light in the Attic",+ "upc": "UPC"+ }+ },+ "search": {+ "hits": 0,+ "data": {+ "results": [+ {+ "words": 0+ },+ {+ "title": "Thumbnail for PhantomJS",+ "words": 6+ },+ {+ "words": 0+ },+ {+ "title": "Thumbnail for Web browser",+ "words": 34+ },+ {+ "title": "Thumbnail for AI browser",+ "words": 8+ },+ {+ "words": 0+ },+ {+ "words": 0+ },+ {+ "words": 0+ },+ {+ "words": 0+ },+ {+ "title": "Thumbnail for Archive.today",+ "words": 44+ },+ {+ "words": 0+ },+ {+ "words": 0+ },+ {+ "title": "Thumbnail for CodeSandbox",+ "words": 6+ },+ {+ "words": 0+ },+ {+ "words": 0+ },+ {+ "words": 0+ },+ {+ "words": 0+ },+ {+ "words": 4+ },+ {+ "title": "Thumbnail for Denial-of-service attack",+ "words": -2017+ },+ {+ "words": 0+ }+ ]+ }+ },+ "stories": {+ "offline": "the news.ycombinator.com front page is Y Combinator and user-submitted content that cannot be redistributed"+ }}}diff --git a/tests/e2e/scenarios/api/extract-corpus-extended.sh b/tests/e2e/scenarios/api/extract-corpus-extended.shindex a33ebe9a..2cd90362 100755--- a/tests/e2e/scenarios/api/extract-corpus-extended.sh+++ b/tests/e2e/scenarios/api/extract-corpus-extended.sh@@ -1,5 +1,5 @@#!/bin/bash-# extract-corpus-extended.sh — the live /extract pipeline on each mirrored corpus page scores at least the offline unit corpus.+# extract-corpus-extended.sh — the live /extract pipeline on each mirrored corpus page extracts what the offline unit corpus does.GROUP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"source "${GROUP_DIR}/../../helpers/api.sh"@@ -24,8 +24,15 @@ def outcomes($got; $want; $prefix):[outcomes($got; $want; "")]'+CORPUS_DRIFT='+def pathname: map(if type == "number" then "[\(.)]" else ".\(.)" end) | join("") | ltrimstr(".");+[($live | paths(scalars)), ($offline | paths(scalars))] | unique |+map(. as $p | {path: ($p | pathname), live: ($live | getpath($p)), offline: ($offline | getpath($p))}) |+map(select(.live != .offline))+'+# ─────────────────────────────────────────────────────────────────-start_test "extract-corpus: live render, snapshot and resolve score at least the offline corpus"+start_test "extract-corpus: live render, snapshot and resolve match the offline corpus"for NAME in $(jq -r '.entries | keys[]' "$CORPUS_MANIFEST"); doOFFLINE_REASON=$(jq -r --arg n "$NAME" '.entries[$n].offline // empty' "$CORPUS_MANIFEST")@@ -33,7 +40,8 @@ for NAME in $(jq -r '.entries | keys[]' "$CORPUS_MANIFEST"); doskip_assert "${NAME}: offline-only, skipped: ${OFFLINE_REASON}"continuefi- MIN_HITS=$(jq -r --arg n "$NAME" '.entries[$n].minHits' "$CORPUS_MANIFEST")+ OFFLINE_HITS=$(jq -r --arg n "$NAME" '.entries[$n].hits' "$CORPUS_MANIFEST")+ OFFLINE_DATA=$(jq -c --arg n "$NAME" '.entries[$n].data' "$CORPUS_MANIFEST")pt_post /navigate -d "{\"url\":\"${FIXTURES_URL}/corpus/${NAME}.html\"}" >/dev/nullassert_ok "${NAME}: navigate to corpus/${NAME}.html"@@ -48,10 +56,15 @@ for NAME in $(jq -r '.entries | keys[]' "$CORPUS_MANIFEST"); doHITS=$(echo "$OUTCOMES" | jq '[.[] | select(.hit)] | length')TOTAL=$(echo "$OUTCOMES" | jq 'length')- if [ "$HITS" -ge "$MIN_HITS" ]; then- pass_assert "${NAME}: pass, ${HITS}/${TOTAL} fields match expected.json (offline floor ${MIN_HITS})"+ SAME=$(echo "$RESULT" | jq --argjson offline "$OFFLINE_DATA" '.data == $offline')+ DRIFT=$(echo "$RESULT" | jq -c --argjson offline "$OFFLINE_DATA" '.data as $live | '"$CORPUS_DRIFT")+ echo "$DRIFT" | jq -r '.[] | " drift \(.path) live=\(.live | tojson) offline=\(.offline | tojson)" | .[0:160]'+ DRIFTED=$(echo "$DRIFT" | jq 'length')++ if [ "$HITS" -eq "$OFFLINE_HITS" ] && [ "$SAME" = "true" ]; then+ pass_assert "${NAME}: pass, ${HITS}/${TOTAL} fields match expected.json and every field equals the offline run"else- fail_assert "${NAME}: fail, ${HITS}/${TOTAL} fields match expected.json, below the offline floor ${MIN_HITS}"+ fail_assert "${NAME}: fail, ${HITS}/${TOTAL} fields match expected.json (offline ${OFFLINE_HITS}), ${DRIFTED} fields differ from the offline run"fidone