Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

test(extract): corpus e2e mirror passes article, infobox and search on a 0-hit floor; assert live hits equal the offline corpus per entry (PIN-435)

Luigi Agosti070571fad48c
diff --git a/internal/extract/corpus_test.go b/internal/extract/corpus_test.go
index d4ecef81..c87599d6 100644
--- a/internal/extract/corpus_test.go
+++ b/internal/extract/corpus_test.go
@@ -3,6 +3,7 @@ package extract
import (
"bytes"
"encoding/json"
+ "flag"
"math"
"os"
"path/filepath"
@@ -22,6 +23,8 @@ const (
e2eManifest = "../../tests/e2e/fixtures/corpus/manifest.json"
)
+var updateManifest = flag.Bool("update", false, "rewrite the e2e corpus manifest with the offline hits and data of every mirrored entry")
+
type corpusEntry struct {
name string
nodes []observe.A11yNode
@@ -69,7 +72,7 @@ func TestCorpus(t *testing.T) {
func TestCorpus_E2EMirrorMatchesUnitCorpus(t *testing.T) {
manifest := readE2EManifest(t)
entries := loadCorpus(t)
- if len(manifest) != len(entries) {
+ if len(manifest) != len(entries) && !*updateManifest {
t.Errorf("%s lists %d entries, the unit corpus has %d", e2eManifest, len(manifest), len(entries))
}
for _, e := range entries {
@@ -77,18 +80,17 @@ func TestCorpus_E2EMirrorMatchesUnitCorpus(t *testing.T) {
_, statErr := os.Stat(filepath.Join(e2eCorpusDir, e.name+".html"))
mirrored := statErr == nil
switch {
- case !listed:
- t.Errorf("%s is missing from %s", e.name, e2eManifest)
- case m.Offline != "" && m.MinHits != nil:
- t.Errorf("%s is both offline-only and carries minHits", e.name)
case m.Offline != "":
if mirrored {
t.Errorf("%s is offline-only but has an e2e HTML fixture", e.name)
}
- case m.MinHits == nil:
- t.Errorf("%s needs either minHits or an offline reason", e.name)
+ if m.Hits != nil || m.Data != nil {
+ t.Errorf("%s is offline-only but carries offline hits or data", e.name)
+ }
+ case !mirrored && !listed:
+ t.Errorf("%s is missing from %s", e.name, e2eManifest)
case !mirrored:
- t.Errorf("%s carries minHits but has no e2e HTML fixture", e.name)
+ t.Errorf("%s needs either an e2e HTML fixture or an offline reason", e.name)
default:
for _, file := range []string{"schema.json", "expected.json"} {
unit := mustRead(t, filepath.Join(corpusDir, e.name, file))
@@ -97,11 +99,39 @@ func TestCorpus_E2EMirrorMatchesUnitCorpus(t *testing.T) {
t.Errorf("%s: e2e copy of %s differs from the unit corpus", e.name, file)
}
}
- if hits := countHits(scoreEntry(t, e)); hits < *m.MinHits {
- t.Errorf("%s: offline hits %d fell below the e2e minHits %d", e.name, hits, *m.MinHits)
+ data := resolveEntry(t, e)
+ hits := countHits(compareObject("", data, e.expected))
+ if *updateManifest {
+ manifest[e.name] = e2eEntry{Hits: &hits, Data: data}
+ continue
}
+ checkManifestEntry(t, e.name, m, hits, data)
+ }
+ }
+ if *updateManifest {
+ writeE2EManifest(t, manifest)
+ }
+}
+
+func checkManifestEntry(t *testing.T, name string, m e2eEntry, hits int, data map[string]any) {
+ t.Helper()
+ switch {
+ case m.Hits == nil:
+ t.Errorf("%s: %s carries no offline hits; run go test ./internal/extract -run TestCorpus_E2EMirror -update", name, e2eManifest)
+ case *m.Hits != hits:
+ t.Errorf("%s: %s records %d offline hits, the offline run scores %d; run go test ./internal/extract -run TestCorpus_E2EMirror -update", name, e2eManifest, *m.Hits, hits)
+ }
+ if reflect.DeepEqual(m.Data, data) {
+ return
+ }
+ reported := map[string]bool{}
+ for _, o := range append(compareObject("", m.Data, data), swapped(compareObject("", data, m.Data))...) {
+ if !o.hit && !reported[o.path] {
+ reported[o.path] = true
+ t.Errorf("%s: %s records %s=%s, the offline run extracts %s", name, e2eManifest, o.path, show(o.got), show(o.want))
}
}
+ t.Errorf("%s: %s data differs from the offline run; run go test ./internal/extract -run TestCorpus_E2EMirror -update", name, e2eManifest)
}
func loadCorpus(t *testing.T) []corpusEntry {
@@ -136,6 +166,11 @@ func loadCorpus(t *testing.T) []corpusEntry {
}
func scoreEntry(t *testing.T, e corpusEntry) []fieldOutcome {
+ t.Helper()
+ return compareObject("", resolveEntry(t, e), e.expected)
+}
+
+func resolveEntry(t *testing.T, e corpusEntry) map[string]any {
t.Helper()
raw, err := json.Marshal(Resolve(e.schema, e.nodes, Options{}).Data)
if err != nil {
@@ -145,7 +180,7 @@ func scoreEntry(t *testing.T, e corpusEntry) []fieldOutcome {
if err := json.Unmarshal(raw, &got); err != nil {
t.Fatalf("%s: decode data: %v", e.name, err)
}
- return compareObject("", got, e.expected)
+ return got
}
func compareObject(prefix string, got, want map[string]any) []fieldOutcome {
@@ -175,6 +210,13 @@ func compareObject(prefix string, got, want map[string]any) []fieldOutcome {
return out
}
+func swapped(outcomes []fieldOutcome) []fieldOutcome {
+ for i := range outcomes {
+ outcomes[i].got, outcomes[i].want = outcomes[i].want, outcomes[i].got
+ }
+ return outcomes
+}
+
func countHits(outcomes []fieldOutcome) int {
hits := 0
for _, o := range outcomes {
@@ -220,8 +262,9 @@ func readBaseline(t *testing.T) float64 {
}
type e2eEntry struct {
- MinHits *int `json:"minHits"`
- Offline string `json:"offline"`
+ Hits *int `json:"hits,omitempty"`
+ Data map[string]any `json:"data,omitempty"`
+ Offline string `json:"offline,omitempty"`
}
func readE2EManifest(t *testing.T) map[string]e2eEntry {
@@ -235,6 +278,22 @@ func readE2EManifest(t *testing.T) map[string]e2eEntry {
return manifest.Entries
}
+func writeE2EManifest(t *testing.T, entries map[string]e2eEntry) {
+ t.Helper()
+ var buf bytes.Buffer
+ enc := json.NewEncoder(&buf)
+ enc.SetEscapeHTML(false)
+ enc.SetIndent("", " ")
+ if err := enc.Encode(struct {
+ Entries map[string]e2eEntry `json:"entries"`
+ }{entries}); err != nil {
+ t.Fatalf("encode %s: %v", e2eManifest, err)
+ }
+ if err := os.WriteFile(e2eManifest, buf.Bytes(), 0o644); err != nil {
+ t.Fatalf("write %s: %v", e2eManifest, err)
+ }
+}
+
func mustRead(t *testing.T, path string) []byte {
t.Helper()
data, err := os.ReadFile(path)
diff --git a/internal/extract/testdata/corpus/README.md b/internal/extract/testdata/corpus/README.md
index 3db3278e..29500621 100644
--- a/internal/extract/testdata/corpus/README.md
+++ b/internal/extract/testdata/corpus/README.md
@@ -84,13 +84,21 @@ removed, the stylesheets reduced to the rules that match the page and inlined,
images replaced by a 1x1 pixel) plus byte-identical copies of `schema.json` and
`expected.json`. Each copy was checked to produce the same node list (depth,
role, name, text, value) as the committed snapshot. Its `manifest.json` gives
-each mirrored entry a `minHits` floor, the offline hit count at commit time, and
-each offline-only entry the reason it is not mirrored.
-`TestCorpus_E2EMirrorMatchesUnitCorpus` keeps the copies, the manifest and the
-floors in step with this directory, and the
-`tests/e2e/scenarios/api/extract-corpus-extended.sh` scenario runs each mirrored
-page through the live pipeline, reporting every field and failing an entry
-whose hits fall below its floor.
+each mirrored entry the offline run's `hits` against `expected.json` and the
+`data` it extracts, and each offline-only entry the reason it is not mirrored.
+`TestCorpus_E2EMirrorMatchesUnitCorpus` keeps the copies and the manifest in
+step with this directory and fails when a recorded `hits` or `data` differs
+from the offline run. After a change that moves the offline result, regenerate
+the manifest rather than editing it:
+
+```
+go test ./internal/extract/ -run TestCorpus_E2EMirror -update
+```
+
+The `tests/e2e/scenarios/api/extract-corpus-extended.sh` scenario runs each
+mirrored page through the live pipeline, reports every field, and fails an
+entry whose live hits differ from the recorded `hits` or whose live `data`
+differs from the recorded `data`, printing each field that drifted.
## Attribution and licences
diff --git a/tests/e2e/fixtures/corpus/manifest.json b/tests/e2e/fixtures/corpus/manifest.json
index 5a1a8deb..638da54b 100644
--- a/tests/e2e/fixtures/corpus/manifest.json
+++ b/tests/e2e/fixtures/corpus/manifest.json
@@ -1,12 +1,160 @@
{
"entries": {
- "article": { "minHits": 0 },
- "infobox": { "minHits": 0 },
- "listing": { "minHits": 11 },
- "login": { "minHits": 2 },
- "pricing": { "offline": "github.com/pricing is GitHub, Inc. page content that cannot be redistributed" },
- "product": { "minHits": 1 },
- "search": { "minHits": 0 },
- "stories": { "offline": "the news.ycombinator.com front page is Y Combinator and user-submitted content that cannot be redistributed" }
+ "article": {
+ "hits": 0,
+ "data": {
+ "headline": "Contents",
+ "published": "Published"
+ }
+ },
+ "infobox": {
+ "hits": 0,
+ "data": {
+ "easiestRoute": "Easiest route",
+ "location": "Location",
+ "name": "Contents",
+ "parentRange": "Parent range"
+ }
+ },
+ "listing": {
+ "hits": 11,
+ "data": {
+ "books": [
+ {
+ "availability": "In stock"
+ },
+ {
+ "availability": "In stock"
+ },
+ {
+ "availability": "In stock"
+ },
+ {
+ "availability": "In stock"
+ },
+ {
+ "availability": "In stock"
+ },
+ {
+ "availability": "In stock"
+ },
+ {
+ "availability": "In stock"
+ },
+ {
+ "availability": "In stock",
+ "title": "A Year in Provence ..."
+ },
+ {
+ "availability": "In stock"
+ },
+ {
+ "availability": "In stock"
+ },
+ {
+ "availability": "In stock"
+ }
+ ]
+ }
+ },
+ "login": {
+ "hits": 2,
+ "data": {
+ "passwordLabel": "Username Password",
+ "quotesSource": "Quotes by:",
+ "site": "Quotes to Scrape",
+ "submit": "Login",
+ "usernameLabel": "Username Password"
+ }
+ },
+ "pricing": {
+ "offline": "github.com/pricing is GitHub, Inc. page content that cannot be redistributed"
+ },
+ "product": {
+ "hits": 1,
+ "data": {
+ "availability": "Availability",
+ "description": "Product Description",
+ "productType": "Product Type",
+ "title": "A Light in the Attic",
+ "upc": "UPC"
+ }
+ },
+ "search": {
+ "hits": 0,
+ "data": {
+ "results": [
+ {
+ "words": 0
+ },
+ {
+ "title": "Thumbnail for PhantomJS",
+ "words": 6
+ },
+ {
+ "words": 0
+ },
+ {
+ "title": "Thumbnail for Web browser",
+ "words": 34
+ },
+ {
+ "title": "Thumbnail for AI browser",
+ "words": 8
+ },
+ {
+ "words": 0
+ },
+ {
+ "words": 0
+ },
+ {
+ "words": 0
+ },
+ {
+ "words": 0
+ },
+ {
+ "title": "Thumbnail for Archive.today",
+ "words": 44
+ },
+ {
+ "words": 0
+ },
+ {
+ "words": 0
+ },
+ {
+ "title": "Thumbnail for CodeSandbox",
+ "words": 6
+ },
+ {
+ "words": 0
+ },
+ {
+ "words": 0
+ },
+ {
+ "words": 0
+ },
+ {
+ "words": 0
+ },
+ {
+ "words": 4
+ },
+ {
+ "title": "Thumbnail for Denial-of-service attack",
+ "words": -2017
+ },
+ {
+ "words": 0
+ }
+ ]
+ }
+ },
+ "stories": {
+ "offline": "the news.ycombinator.com front page is Y Combinator and user-submitted content that cannot be redistributed"
+ }
}
}
diff --git a/tests/e2e/scenarios/api/extract-corpus-extended.sh b/tests/e2e/scenarios/api/extract-corpus-extended.sh
index a33ebe9a..2cd90362 100755
--- a/tests/e2e/scenarios/api/extract-corpus-extended.sh
+++ b/tests/e2e/scenarios/api/extract-corpus-extended.sh
@@ -1,5 +1,5 @@
#!/bin/bash
-# extract-corpus-extended.sh — the live /extract pipeline on each mirrored corpus page scores at least the offline unit corpus.
+# extract-corpus-extended.sh — the live /extract pipeline on each mirrored corpus page extracts what the offline unit corpus does.
GROUP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
source "${GROUP_DIR}/../../helpers/api.sh"
@@ -24,8 +24,15 @@ def outcomes($got; $want; $prefix):
[outcomes($got; $want; "")]
'
+CORPUS_DRIFT='
+def pathname: map(if type == "number" then "[\(.)]" else ".\(.)" end) | join("") | ltrimstr(".");
+[($live | paths(scalars)), ($offline | paths(scalars))] | unique |
+map(. as $p | {path: ($p | pathname), live: ($live | getpath($p)), offline: ($offline | getpath($p))}) |
+map(select(.live != .offline))
+'
+
# ─────────────────────────────────────────────────────────────────
-start_test "extract-corpus: live render, snapshot and resolve score at least the offline corpus"
+start_test "extract-corpus: live render, snapshot and resolve match the offline corpus"
for NAME in $(jq -r '.entries | keys[]' "$CORPUS_MANIFEST"); do
OFFLINE_REASON=$(jq -r --arg n "$NAME" '.entries[$n].offline // empty' "$CORPUS_MANIFEST")
@@ -33,7 +40,8 @@ for NAME in $(jq -r '.entries | keys[]' "$CORPUS_MANIFEST"); do
skip_assert "${NAME}: offline-only, skipped: ${OFFLINE_REASON}"
continue
fi
- MIN_HITS=$(jq -r --arg n "$NAME" '.entries[$n].minHits' "$CORPUS_MANIFEST")
+ OFFLINE_HITS=$(jq -r --arg n "$NAME" '.entries[$n].hits' "$CORPUS_MANIFEST")
+ OFFLINE_DATA=$(jq -c --arg n "$NAME" '.entries[$n].data' "$CORPUS_MANIFEST")
pt_post /navigate -d "{\"url\":\"${FIXTURES_URL}/corpus/${NAME}.html\"}" >/dev/null
assert_ok "${NAME}: navigate to corpus/${NAME}.html"
@@ -48,10 +56,15 @@ for NAME in $(jq -r '.entries | keys[]' "$CORPUS_MANIFEST"); do
HITS=$(echo "$OUTCOMES" | jq '[.[] | select(.hit)] | length')
TOTAL=$(echo "$OUTCOMES" | jq 'length')
- if [ "$HITS" -ge "$MIN_HITS" ]; then
- pass_assert "${NAME}: pass, ${HITS}/${TOTAL} fields match expected.json (offline floor ${MIN_HITS})"
+ SAME=$(echo "$RESULT" | jq --argjson offline "$OFFLINE_DATA" '.data == $offline')
+ DRIFT=$(echo "$RESULT" | jq -c --argjson offline "$OFFLINE_DATA" '.data as $live | '"$CORPUS_DRIFT")
+ echo "$DRIFT" | jq -r '.[] | " drift \(.path) live=\(.live | tojson) offline=\(.offline | tojson)" | .[0:160]'
+ DRIFTED=$(echo "$DRIFT" | jq 'length')
+
+ if [ "$HITS" -eq "$OFFLINE_HITS" ] && [ "$SAME" = "true" ]; then
+ pass_assert "${NAME}: pass, ${HITS}/${TOTAL} fields match expected.json and every field equals the offline run"
else
- fail_assert "${NAME}: fail, ${HITS}/${TOTAL} fields match expected.json, below the offline floor ${MIN_HITS}"
+ fail_assert "${NAME}: fail, ${HITS}/${TOTAL} fields match expected.json (offline ${OFFLINE_HITS}), ${DRIFTED} fields differ from the offline run"
fi
done