Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

test: add scrape e2e scenario and scrape-site fixture

Luigi Agosti7765a5065891
diff --git a/tests/e2e/config/pinchtab.json b/tests/e2e/config/pinchtab.json
index c546d2c1..c4d8a846 100644
--- a/tests/e2e/config/pinchtab.json
+++ b/tests/e2e/config/pinchtab.json
@@ -49,6 +49,7 @@
"127.0.0.1",
"::1",
"fixtures",
+ "scrapesite",
"httpbin.org"
],
"downloadAllowedDomains": ["fixtures", "httpbin.org"],
diff --git a/tests/e2e/docker-compose-multi.yml b/tests/e2e/docker-compose-multi.yml
index d8c02091..70bf8c21 100644
--- a/tests/e2e/docker-compose-multi.yml
+++ b/tests/e2e/docker-compose-multi.yml
@@ -134,6 +134,9 @@ services:
aliases:
- fixtures
- pivot-target
+ # Dedicated host whose nginx vhost roots at the scrape-site/ subdir,
+ # so a seaportal crawl of http://scrapesite/ finds the scrape fixture.
+ - scrapesite
healthcheck:
# Verify BOTH that nginx is up AND that the custom config is the one
# actually loaded. /get is a synthetic endpoint defined only in
diff --git a/tests/e2e/docker-compose.yml b/tests/e2e/docker-compose.yml
index 81d0b2bc..2296c89f 100644
--- a/tests/e2e/docker-compose.yml
+++ b/tests/e2e/docker-compose.yml
@@ -39,6 +39,9 @@ services:
aliases:
- fixtures
- pivot-target
+ # Dedicated host whose nginx vhost roots at the scrape-site/ subdir,
+ # so a seaportal crawl of http://scrapesite/ finds the scrape fixture.
+ - scrapesite
healthcheck:
# Verify BOTH that nginx is up AND that the custom config is the one
# actually loaded. /get is a synthetic endpoint defined only in
diff --git a/tests/e2e/fixtures/scrape-site/about.html b/tests/e2e/fixtures/scrape-site/about.html
new file mode 100644
index 00000000..ca7b2b41
--- /dev/null
+++ b/tests/e2e/fixtures/scrape-site/about.html
@@ -0,0 +1,33 @@
+<!doctype html>
+<!-- Static content page: full text in the initial HTML, so HTTP extraction is
+ sufficient and the report should mark this page source=http. -->
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <meta name="description" content="About the scrape fixture site.">
+ <link rel="icon" href="data:,">
+ <title>About — Scrape Fixture Site</title>
+</head>
+<body>
+ <nav aria-label="Site pages"><a href="index.html">Home</a></nav>
+ <main>
+ <article>
+ <h1>About this fixture</h1>
+ <p>The scrape fixture site is a deterministic set of pages used by the
+ end-to-end tests for the PinchTab scrape command. It is served by the
+ same nginx container that serves the other fixtures, and its pages never
+ change between runs, which keeps the assertions stable.</p>
+ <p>This particular page is entirely server-rendered. Everything you can
+ read here is delivered in the first HTTP response, with no client-side
+ rendering involved. That makes it a good control case: the scraper's
+ HTTP extraction should capture the full text, the routing logic should
+ decide that no browser is needed, and the page should appear in the
+ report with its content source recorded as HTTP.</p>
+ <p>Keeping a few plain pages alongside the JavaScript-only application page
+ lets a single scrape run demonstrate both halves of the pipeline at once:
+ cheap HTTP extraction where it is enough, and browser rendering only
+ where it is actually required.</p>
+ </article>
+ </main>
+</body>
+</html>
diff --git a/tests/e2e/fixtures/scrape-site/app.html b/tests/e2e/fixtures/scrape-site/app.html
new file mode 100644
index 00000000..01dd0958
--- /dev/null
+++ b/tests/e2e/fixtures/scrape-site/app.html
@@ -0,0 +1,44 @@
+<!doctype html>
+<!-- JS-only application page: the initial HTML is an empty shell, so a plain
+ HTTP fetch extracts almost nothing (thin content) and the scraper should
+ route this page to a real browser. Once JavaScript runs, #app is filled
+ with a full article that browser extraction recovers — the report should
+ then mark this page source=browser. readability strips <script>, so the
+ article text below is NOT visible to static HTTP extraction. -->
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <meta name="description" content="A JavaScript-rendered page on the scrape fixture site.">
+ <link rel="icon" href="data:,">
+ <title>App — Scrape Fixture Site</title>
+</head>
+<body>
+ <nav aria-label="Site pages"><a href="index.html">Home</a></nav>
+ <main>
+ <div id="app">Loading…</div>
+ </main>
+ <script>
+ // Build the real content in the browser, the way a single-page app would.
+ // This runs on parse, so the DOM carries the article well before the
+ // scraper reads outerHTML.
+ document.getElementById('app').innerHTML = [
+ '<article>',
+ '<h1>Rendered by JavaScript</h1>',
+ '<p>This page had no readable content in its initial HTML response. A plain',
+ ' HTTP fetch of it sees only a loading placeholder, which is why the scraper',
+ ' should decide the page needs a real browser and route it accordingly.</p>',
+ '<p>Everything in this article was produced by client-side JavaScript after',
+ ' the document loaded. When PinchTab renders the page in a real browser and',
+ ' extracts the resulting DOM, this text becomes the page content and the',
+ ' report records the content source as browser rather than HTTP.</p>',
+ '<p>The paragraphs are intentionally long so that browser extraction has a',
+ ' substantial article to recover, comfortably above the threshold that the',
+ ' readability pass needs to treat this as the main content of the page.</p>',
+ '<p>This is the single page on the scrape fixture site that exercises the',
+ ' browser-enrichment half of the pipeline end to end: thin over HTTP, whole',
+ ' once JavaScript has run.</p>',
+ '</article>',
+ ].join('');
+ </script>
+</body>
+</html>
diff --git a/tests/e2e/fixtures/scrape-site/guide.html b/tests/e2e/fixtures/scrape-site/guide.html
new file mode 100644
index 00000000..adbd8654
--- /dev/null
+++ b/tests/e2e/fixtures/scrape-site/guide.html
@@ -0,0 +1,30 @@
+<!doctype html>
+<!-- Static content page: like about.html, its full text is server-rendered so
+ the report should mark it source=http. -->
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <meta name="description" content="A short guide on the scrape fixture site.">
+ <link rel="icon" href="data:,">
+ <title>Guide — Scrape Fixture Site</title>
+</head>
+<body>
+ <nav aria-label="Site pages"><a href="index.html">Home</a></nav>
+ <main>
+ <article>
+ <h1>A short guide</h1>
+ <p>This guide page adds a third server-rendered document to the fixture so
+ the crawl discovers more than a couple of URLs and the report's page
+ tree has something to group. Its text, like the other static pages, is
+ present in the initial HTML response and needs no JavaScript to read.</p>
+ <p>When the scrape command runs against this site, the guide should be
+ fetched over HTTP, extracted successfully, and reported as HTTP-sourced.
+ It exists mainly to give the preview mode several pages to summarise and
+ to give the expand mode more than one candidate URL to choose from.</p>
+ <p>The deliberately wordy paragraphs ensure the extracted content comfortably
+ exceeds the thin-content threshold, so the routing logic leaves this page
+ on the HTTP path instead of escalating it to the browser.</p>
+ </article>
+ </main>
+</body>
+</html>
diff --git a/tests/e2e/fixtures/scrape-site/index.html b/tests/e2e/fixtures/scrape-site/index.html
new file mode 100644
index 00000000..85ed0035
--- /dev/null
+++ b/tests/e2e/fixtures/scrape-site/index.html
@@ -0,0 +1,45 @@
+<!doctype html>
+<!-- scrape-site homepage: rich static content that HTTP extraction handles on
+ its own (no browser needed). Nav links let the crawler discover every
+ page. This site deliberately mixes static pages with one JS-only page
+ (app.html) so the scrape scenario can exercise HTTP-first extraction plus
+ browser-enrichment routing. -->
+<html lang="en">
+<head>
+ <meta charset="utf-8">
+ <meta name="description" content="Deterministic scrape fixture site for PinchTab e2e tests.">
+ <link rel="icon" href="data:,">
+ <title>Scrape Fixture Site</title>
+</head>
+<body>
+ <header>Scrape fixture site</header>
+ <nav aria-label="Site pages">
+ <ul>
+ <li><a href="index.html">Home</a></li>
+ <li><a href="about.html">About</a></li>
+ <li><a href="guide.html">Guide</a></li>
+ <li><a href="app.html">App (JS-rendered)</a></li>
+ </ul>
+ </nav>
+ <main>
+ <article>
+ <h1>The scrape fixture site</h1>
+ <p>This site exists so the PinchTab scrape command can be tested against a
+ small but realistic tree of pages. The homepage, the about page, and the
+ guide are ordinary server-rendered HTML: their full text is present in
+ the initial response, so a plain HTTP fetch extracts everything and no
+ browser is required to read them.</p>
+ <p>One page is different on purpose. The application page ships an empty
+ shell and builds its real content in the browser with JavaScript, the
+ way a single-page app would. A plain HTTP fetch of that page sees almost
+ nothing, so the scraper should route it to a real browser, render it,
+ and recover the content that JavaScript produced.</p>
+ <p>Because every page links to every other page, the crawler can discover
+ the whole site starting from this homepage without needing a sitemap.
+ The content here is intentionally verbose so that HTTP extraction clears
+ the thin-content threshold and this page stays HTTP-sourced in the
+ report rather than being sent to the browser.</p>
+ </article>
+ </main>
+</body>
+</html>
diff --git a/tests/e2e/nginx/default.conf b/tests/e2e/nginx/default.conf
index f35c347a..e62c7eb8 100644
--- a/tests/e2e/nginx/default.conf
+++ b/tests/e2e/nginx/default.conf
@@ -47,3 +47,22 @@ server {
try_files $uri $uri/ =404;
}
}
+
+# Dedicated vhost for the scrape fixture. seaportal's crawler seeds discovery
+# from the host ROOT (scheme://host/), not the given path, so a scrape fixture
+# living in a subdirectory of the shared `fixtures` host is never reached. This
+# vhost roots the `scrapesite` hostname at the scrape-site/ subdirectory so a
+# crawl of http://scrapesite/ discovers exactly the scrape fixture pages.
+# Reached via the `scrapesite` network alias on the fixtures service.
+server {
+ listen 80;
+ listen [::]:80;
+ server_name scrapesite;
+
+ root /usr/share/nginx/html/scrape-site;
+ index index.html;
+
+ location / {
+ try_files $uri $uri/ =404;
+ }
+}
diff --git a/tests/e2e/scenarios/cli/scrape-basic.sh b/tests/e2e/scenarios/cli/scrape-basic.sh
new file mode 100755
index 00000000..b032d5f9
--- /dev/null
+++ b/tests/e2e/scenarios/cli/scrape-basic.sh
@@ -0,0 +1,160 @@
+#!/bin/bash
+# scrape-basic.sh — pinchtab scrape: HTTP-first crawl, browser-enrichment
+# routing, and the preview → expand flow.
+#
+# Runs against the dedicated scrape-site fixture, which mixes server-rendered
+# pages (index/about/guide — HTTP extraction is enough) with one JavaScript-only
+# page (app.html — thin over HTTP, whole once rendered). That mix lets one run
+# prove both halves of the pipeline: cheap HTTP where it suffices, browser
+# rendering only where it is required.
+
+GROUP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+source "${GROUP_DIR}/../../helpers/cli.sh"
+
+# scrapesite is a dedicated nginx vhost (see tests/e2e/nginx/default.conf) that
+# roots at the scrape-site/ fixture dir. seaportal crawls from the host root, so
+# the scrape fixture needs its own host rather than a subdirectory of `fixtures`.
+SITE_URL="http://scrapesite/index.html"
+
+# ─────────────────────────────────────────────────────────────────
+start_test "scrape crawls the site with HTTP-first extraction and browser routing"
+
+pt_ok scrape "$SITE_URL" --timeout 30 --json
+
+if echo "$PT_OUT" | jq -e '.schemaVersion == "1.0"' >/dev/null 2>&1; then
+ pass_assert "report carries schema version"
+else
+ fail_assert "report carries schema version"
+fi
+
+PAGE_COUNT=$(echo "$PT_OUT" | jq '.pages | length' 2>/dev/null)
+if [ "${PAGE_COUNT:-0}" -ge 4 ] 2>/dev/null; then
+ pass_assert "crawl discovered the linked pages (got: $PAGE_COUNT)"
+else
+ fail_assert "crawl discovered the linked pages (got: $PAGE_COUNT)"
+fi
+
+if echo "$PT_OUT" | jq -e '[.pages[].source] | all(. == "http" or . == "browser")' >/dev/null 2>&1; then
+ pass_assert "every page records its content source"
+else
+ fail_assert "every page records its content source"
+fi
+
+if echo "$PT_OUT" | jq -e '[.pages[] | select(.error == null)] | all(.markdown | length > 0)' >/dev/null 2>&1; then
+ pass_assert "every loaded page has markdown content"
+else
+ fail_assert "every loaded page has markdown content"
+fi
+
+if echo "$PT_OUT" | jq -e '.pageGroups | length > 0' >/dev/null 2>&1; then
+ pass_assert "report carries the page tree"
+else
+ fail_assert "report carries the page tree"
+fi
+
+# The JS-only page is thin over HTTP and must be recovered by the browser.
+if echo "$PT_OUT" | jq -e '.summary.browserPages >= 1' >/dev/null 2>&1; then
+ pass_assert "at least one page was browser-rendered"
+else
+ fail_assert "at least one page was browser-rendered"
+fi
+
+if echo "$PT_OUT" | jq -e '.pages[] | select(.url | endswith("app.html")) | .source == "browser"' >/dev/null 2>&1; then
+ pass_assert "the JS-only page was recovered via the browser"
+else
+ fail_assert "the JS-only page was recovered via the browser"
+fi
+
+end_test
+
+# ─────────────────────────────────────────────────────────────────
+start_test "scrape --no-browser records routing verdicts without rendering"
+
+pt_ok scrape "$SITE_URL" --timeout 30 --no-browser --json
+
+if echo "$PT_OUT" | jq -e '[.pages[].source] | all(. == "http")' >/dev/null 2>&1; then
+ pass_assert "no page was browser-rendered"
+else
+ fail_assert "no page was browser-rendered"
+fi
+
+if echo "$PT_OUT" | jq -e '.summary.browserPages == 0' >/dev/null 2>&1; then
+ pass_assert "summary counts zero browser pages"
+else
+ fail_assert "summary counts zero browser pages"
+fi
+
+# The routing verdict is still recorded even though nothing was rendered.
+if echo "$PT_OUT" | jq -e '.pages[] | select(.url | endswith("app.html")) | .browserRecommended == true' >/dev/null 2>&1; then
+ pass_assert "the JS-only page is still flagged as needing the browser"
+else
+ fail_assert "the JS-only page is still flagged as needing the browser"
+fi
+
+end_test
+
+# ─────────────────────────────────────────────────────────────────
+start_test "scrape --preview outlines pages without full bodies"
+
+pt_ok scrape "$SITE_URL" --timeout 30 --preview --json
+
+if echo "$PT_OUT" | jq -e '.pages | length >= 4' >/dev/null 2>&1; then
+ pass_assert "preview discovered the linked pages"
+else
+ fail_assert "preview discovered the linked pages"
+fi
+
+if echo "$PT_OUT" | jq -e '[.pages[].markdown // ""] | all(. == "")' >/dev/null 2>&1; then
+ pass_assert "preview withholds full page bodies"
+else
+ fail_assert "preview withholds full page bodies"
+fi
+
+if echo "$PT_OUT" | jq -e '[.pages[] | select(.error == null)] | any(.charCount > 0)' >/dev/null 2>&1; then
+ pass_assert "preview reports content size (charCount)"
+else
+ fail_assert "preview reports content size (charCount)"
+fi
+
+if echo "$PT_OUT" | jq -e '.summary.browserPages == 0' >/dev/null 2>&1; then
+ pass_assert "preview renders nothing in the browser"
+else
+ fail_assert "preview renders nothing in the browser"
+fi
+
+# The outline still tells the caller which pages would need the browser.
+if echo "$PT_OUT" | jq -e '.pages[] | select(.url | endswith("app.html")) | .browserRecommended == true' >/dev/null 2>&1; then
+ pass_assert "preview flags the JS-only page as needing the browser"
+else
+ fail_assert "preview flags the JS-only page as needing the browser"
+fi
+
+# Pick the JS-only page to expand next: it proves expand recovers via browser.
+EXPAND_URL=$(echo "$PT_OUT" | jq -r '.pages[] | select(.url | endswith("app.html")) | .url' 2>/dev/null | head -1)
+
+end_test
+
+# ─────────────────────────────────────────────────────────────────
+start_test "scrape --only expands a chosen URL at full fidelity"
+
+pt_ok scrape "$SITE_URL" --timeout 30 --only "$EXPAND_URL" --json
+
+if echo "$PT_OUT" | jq -e --arg u "$EXPAND_URL" '[.pages[].url] == [$u]' >/dev/null 2>&1; then
+ pass_assert "expand scraped exactly the chosen URL"
+else
+ fail_assert "expand scraped exactly the chosen URL"
+fi
+
+if echo "$PT_OUT" | jq -e '.pages[0].source == "browser"' >/dev/null 2>&1; then
+ pass_assert "expanded JS-only page was rendered in the browser"
+else
+ fail_assert "expanded JS-only page was rendered in the browser"
+fi
+
+if echo "$PT_OUT" | jq -e '.pages[0].markdown | contains("JavaScript")' >/dev/null 2>&1; then
+ pass_assert "expanded page carries the JavaScript-rendered content"
+else
+ fail_assert "expanded page carries the JavaScript-rendered content"
+fi
+
+end_test