High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
test: add scrape e2e scenario and scrape-site fixture
Luigi Agosti7765a5065891
diff --git a/tests/e2e/config/pinchtab.json b/tests/e2e/config/pinchtab.jsonindex c546d2c1..c4d8a846 100644--- a/tests/e2e/config/pinchtab.json+++ b/tests/e2e/config/pinchtab.json@@ -49,6 +49,7 @@"127.0.0.1","::1","fixtures",+ "scrapesite","httpbin.org"],"downloadAllowedDomains": ["fixtures", "httpbin.org"],diff --git a/tests/e2e/docker-compose-multi.yml b/tests/e2e/docker-compose-multi.ymlindex d8c02091..70bf8c21 100644--- a/tests/e2e/docker-compose-multi.yml+++ b/tests/e2e/docker-compose-multi.yml@@ -134,6 +134,9 @@ services:aliases:- fixtures- pivot-target+ # Dedicated host whose nginx vhost roots at the scrape-site/ subdir,+ # so a seaportal crawl of http://scrapesite/ finds the scrape fixture.+ - scrapesitehealthcheck:# Verify BOTH that nginx is up AND that the custom config is the one# actually loaded. /get is a synthetic endpoint defined only indiff --git a/tests/e2e/docker-compose.yml b/tests/e2e/docker-compose.ymlindex 81d0b2bc..2296c89f 100644--- a/tests/e2e/docker-compose.yml+++ b/tests/e2e/docker-compose.yml@@ -39,6 +39,9 @@ services:aliases:- fixtures- pivot-target+ # Dedicated host whose nginx vhost roots at the scrape-site/ subdir,+ # so a seaportal crawl of http://scrapesite/ finds the scrape fixture.+ - scrapesitehealthcheck:# Verify BOTH that nginx is up AND that the custom config is the one# actually loaded. /get is a synthetic endpoint defined only indiff --git a/tests/e2e/fixtures/scrape-site/about.html b/tests/e2e/fixtures/scrape-site/about.htmlnew file mode 100644index 00000000..ca7b2b41--- /dev/null+++ b/tests/e2e/fixtures/scrape-site/about.html@@ -0,0 +1,33 @@+<!doctype html>+<!-- Static content page: full text in the initial HTML, so HTTP extraction is+ sufficient and the report should mark this page source=http. -->+<html lang="en">+<head>+ <meta charset="utf-8">+ <meta name="description" content="About the scrape fixture site.">+ <link rel="icon" href="data:,">+ <title>About — Scrape Fixture Site</title>+</head>+<body>+ <nav aria-label="Site pages"><a href="index.html">Home</a></nav>+ <main>+ <article>+ <h1>About this fixture</h1>+ <p>The scrape fixture site is a deterministic set of pages used by the+ end-to-end tests for the PinchTab scrape command. It is served by the+ same nginx container that serves the other fixtures, and its pages never+ change between runs, which keeps the assertions stable.</p>+ <p>This particular page is entirely server-rendered. Everything you can+ read here is delivered in the first HTTP response, with no client-side+ rendering involved. That makes it a good control case: the scraper's+ HTTP extraction should capture the full text, the routing logic should+ decide that no browser is needed, and the page should appear in the+ report with its content source recorded as HTTP.</p>+ <p>Keeping a few plain pages alongside the JavaScript-only application page+ lets a single scrape run demonstrate both halves of the pipeline at once:+ cheap HTTP extraction where it is enough, and browser rendering only+ where it is actually required.</p>+ </article>+ </main>+</body>+</html>diff --git a/tests/e2e/fixtures/scrape-site/app.html b/tests/e2e/fixtures/scrape-site/app.htmlnew file mode 100644index 00000000..01dd0958--- /dev/null+++ b/tests/e2e/fixtures/scrape-site/app.html@@ -0,0 +1,44 @@+<!doctype html>+<!-- JS-only application page: the initial HTML is an empty shell, so a plain+ HTTP fetch extracts almost nothing (thin content) and the scraper should+ route this page to a real browser. Once JavaScript runs, #app is filled+ with a full article that browser extraction recovers — the report should+ then mark this page source=browser. readability strips <script>, so the+ article text below is NOT visible to static HTTP extraction. -->+<html lang="en">+<head>+ <meta charset="utf-8">+ <meta name="description" content="A JavaScript-rendered page on the scrape fixture site.">+ <link rel="icon" href="data:,">+ <title>App — Scrape Fixture Site</title>+</head>+<body>+ <nav aria-label="Site pages"><a href="index.html">Home</a></nav>+ <main>+ <div id="app">Loading…</div>+ </main>+ <script>+ // Build the real content in the browser, the way a single-page app would.+ // This runs on parse, so the DOM carries the article well before the+ // scraper reads outerHTML.+ document.getElementById('app').innerHTML = [+ '<article>',+ '<h1>Rendered by JavaScript</h1>',+ '<p>This page had no readable content in its initial HTML response. A plain',+ ' HTTP fetch of it sees only a loading placeholder, which is why the scraper',+ ' should decide the page needs a real browser and route it accordingly.</p>',+ '<p>Everything in this article was produced by client-side JavaScript after',+ ' the document loaded. When PinchTab renders the page in a real browser and',+ ' extracts the resulting DOM, this text becomes the page content and the',+ ' report records the content source as browser rather than HTTP.</p>',+ '<p>The paragraphs are intentionally long so that browser extraction has a',+ ' substantial article to recover, comfortably above the threshold that the',+ ' readability pass needs to treat this as the main content of the page.</p>',+ '<p>This is the single page on the scrape fixture site that exercises the',+ ' browser-enrichment half of the pipeline end to end: thin over HTTP, whole',+ ' once JavaScript has run.</p>',+ '</article>',+ ].join('');+ </script>+</body>+</html>diff --git a/tests/e2e/fixtures/scrape-site/guide.html b/tests/e2e/fixtures/scrape-site/guide.htmlnew file mode 100644index 00000000..adbd8654--- /dev/null+++ b/tests/e2e/fixtures/scrape-site/guide.html@@ -0,0 +1,30 @@+<!doctype html>+<!-- Static content page: like about.html, its full text is server-rendered so+ the report should mark it source=http. -->+<html lang="en">+<head>+ <meta charset="utf-8">+ <meta name="description" content="A short guide on the scrape fixture site.">+ <link rel="icon" href="data:,">+ <title>Guide — Scrape Fixture Site</title>+</head>+<body>+ <nav aria-label="Site pages"><a href="index.html">Home</a></nav>+ <main>+ <article>+ <h1>A short guide</h1>+ <p>This guide page adds a third server-rendered document to the fixture so+ the crawl discovers more than a couple of URLs and the report's page+ tree has something to group. Its text, like the other static pages, is+ present in the initial HTML response and needs no JavaScript to read.</p>+ <p>When the scrape command runs against this site, the guide should be+ fetched over HTTP, extracted successfully, and reported as HTTP-sourced.+ It exists mainly to give the preview mode several pages to summarise and+ to give the expand mode more than one candidate URL to choose from.</p>+ <p>The deliberately wordy paragraphs ensure the extracted content comfortably+ exceeds the thin-content threshold, so the routing logic leaves this page+ on the HTTP path instead of escalating it to the browser.</p>+ </article>+ </main>+</body>+</html>diff --git a/tests/e2e/fixtures/scrape-site/index.html b/tests/e2e/fixtures/scrape-site/index.htmlnew file mode 100644index 00000000..85ed0035--- /dev/null+++ b/tests/e2e/fixtures/scrape-site/index.html@@ -0,0 +1,45 @@+<!doctype html>+<!-- scrape-site homepage: rich static content that HTTP extraction handles on+ its own (no browser needed). Nav links let the crawler discover every+ page. This site deliberately mixes static pages with one JS-only page+ (app.html) so the scrape scenario can exercise HTTP-first extraction plus+ browser-enrichment routing. -->+<html lang="en">+<head>+ <meta charset="utf-8">+ <meta name="description" content="Deterministic scrape fixture site for PinchTab e2e tests.">+ <link rel="icon" href="data:,">+ <title>Scrape Fixture Site</title>+</head>+<body>+ <header>Scrape fixture site</header>+ <nav aria-label="Site pages">+ <ul>+ <li><a href="index.html">Home</a></li>+ <li><a href="about.html">About</a></li>+ <li><a href="guide.html">Guide</a></li>+ <li><a href="app.html">App (JS-rendered)</a></li>+ </ul>+ </nav>+ <main>+ <article>+ <h1>The scrape fixture site</h1>+ <p>This site exists so the PinchTab scrape command can be tested against a+ small but realistic tree of pages. The homepage, the about page, and the+ guide are ordinary server-rendered HTML: their full text is present in+ the initial response, so a plain HTTP fetch extracts everything and no+ browser is required to read them.</p>+ <p>One page is different on purpose. The application page ships an empty+ shell and builds its real content in the browser with JavaScript, the+ way a single-page app would. A plain HTTP fetch of that page sees almost+ nothing, so the scraper should route it to a real browser, render it,+ and recover the content that JavaScript produced.</p>+ <p>Because every page links to every other page, the crawler can discover+ the whole site starting from this homepage without needing a sitemap.+ The content here is intentionally verbose so that HTTP extraction clears+ the thin-content threshold and this page stays HTTP-sourced in the+ report rather than being sent to the browser.</p>+ </article>+ </main>+</body>+</html>diff --git a/tests/e2e/nginx/default.conf b/tests/e2e/nginx/default.confindex f35c347a..e62c7eb8 100644--- a/tests/e2e/nginx/default.conf+++ b/tests/e2e/nginx/default.conf@@ -47,3 +47,22 @@ server {try_files $uri $uri/ =404;}}++# Dedicated vhost for the scrape fixture. seaportal's crawler seeds discovery+# from the host ROOT (scheme://host/), not the given path, so a scrape fixture+# living in a subdirectory of the shared `fixtures` host is never reached. This+# vhost roots the `scrapesite` hostname at the scrape-site/ subdirectory so a+# crawl of http://scrapesite/ discovers exactly the scrape fixture pages.+# Reached via the `scrapesite` network alias on the fixtures service.+server {+ listen 80;+ listen [::]:80;+ server_name scrapesite;++ root /usr/share/nginx/html/scrape-site;+ index index.html;++ location / {+ try_files $uri $uri/ =404;+ }+}diff --git a/tests/e2e/scenarios/cli/scrape-basic.sh b/tests/e2e/scenarios/cli/scrape-basic.shnew file mode 100755index 00000000..b032d5f9--- /dev/null+++ b/tests/e2e/scenarios/cli/scrape-basic.sh@@ -0,0 +1,160 @@+#!/bin/bash+# scrape-basic.sh — pinchtab scrape: HTTP-first crawl, browser-enrichment+# routing, and the preview → expand flow.+#+# Runs against the dedicated scrape-site fixture, which mixes server-rendered+# pages (index/about/guide — HTTP extraction is enough) with one JavaScript-only+# page (app.html — thin over HTTP, whole once rendered). That mix lets one run+# prove both halves of the pipeline: cheap HTTP where it suffices, browser+# rendering only where it is required.++GROUP_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"+source "${GROUP_DIR}/../../helpers/cli.sh"++# scrapesite is a dedicated nginx vhost (see tests/e2e/nginx/default.conf) that+# roots at the scrape-site/ fixture dir. seaportal crawls from the host root, so+# the scrape fixture needs its own host rather than a subdirectory of `fixtures`.+SITE_URL="http://scrapesite/index.html"++# ─────────────────────────────────────────────────────────────────+start_test "scrape crawls the site with HTTP-first extraction and browser routing"++pt_ok scrape "$SITE_URL" --timeout 30 --json++if echo "$PT_OUT" | jq -e '.schemaVersion == "1.0"' >/dev/null 2>&1; then+ pass_assert "report carries schema version"+else+ fail_assert "report carries schema version"+fi++PAGE_COUNT=$(echo "$PT_OUT" | jq '.pages | length' 2>/dev/null)+if [ "${PAGE_COUNT:-0}" -ge 4 ] 2>/dev/null; then+ pass_assert "crawl discovered the linked pages (got: $PAGE_COUNT)"+else+ fail_assert "crawl discovered the linked pages (got: $PAGE_COUNT)"+fi++if echo "$PT_OUT" | jq -e '[.pages[].source] | all(. == "http" or . == "browser")' >/dev/null 2>&1; then+ pass_assert "every page records its content source"+else+ fail_assert "every page records its content source"+fi++if echo "$PT_OUT" | jq -e '[.pages[] | select(.error == null)] | all(.markdown | length > 0)' >/dev/null 2>&1; then+ pass_assert "every loaded page has markdown content"+else+ fail_assert "every loaded page has markdown content"+fi++if echo "$PT_OUT" | jq -e '.pageGroups | length > 0' >/dev/null 2>&1; then+ pass_assert "report carries the page tree"+else+ fail_assert "report carries the page tree"+fi++# The JS-only page is thin over HTTP and must be recovered by the browser.+if echo "$PT_OUT" | jq -e '.summary.browserPages >= 1' >/dev/null 2>&1; then+ pass_assert "at least one page was browser-rendered"+else+ fail_assert "at least one page was browser-rendered"+fi++if echo "$PT_OUT" | jq -e '.pages[] | select(.url | endswith("app.html")) | .source == "browser"' >/dev/null 2>&1; then+ pass_assert "the JS-only page was recovered via the browser"+else+ fail_assert "the JS-only page was recovered via the browser"+fi++end_test++# ─────────────────────────────────────────────────────────────────+start_test "scrape --no-browser records routing verdicts without rendering"++pt_ok scrape "$SITE_URL" --timeout 30 --no-browser --json++if echo "$PT_OUT" | jq -e '[.pages[].source] | all(. == "http")' >/dev/null 2>&1; then+ pass_assert "no page was browser-rendered"+else+ fail_assert "no page was browser-rendered"+fi++if echo "$PT_OUT" | jq -e '.summary.browserPages == 0' >/dev/null 2>&1; then+ pass_assert "summary counts zero browser pages"+else+ fail_assert "summary counts zero browser pages"+fi++# The routing verdict is still recorded even though nothing was rendered.+if echo "$PT_OUT" | jq -e '.pages[] | select(.url | endswith("app.html")) | .browserRecommended == true' >/dev/null 2>&1; then+ pass_assert "the JS-only page is still flagged as needing the browser"+else+ fail_assert "the JS-only page is still flagged as needing the browser"+fi++end_test++# ─────────────────────────────────────────────────────────────────+start_test "scrape --preview outlines pages without full bodies"++pt_ok scrape "$SITE_URL" --timeout 30 --preview --json++if echo "$PT_OUT" | jq -e '.pages | length >= 4' >/dev/null 2>&1; then+ pass_assert "preview discovered the linked pages"+else+ fail_assert "preview discovered the linked pages"+fi++if echo "$PT_OUT" | jq -e '[.pages[].markdown // ""] | all(. == "")' >/dev/null 2>&1; then+ pass_assert "preview withholds full page bodies"+else+ fail_assert "preview withholds full page bodies"+fi++if echo "$PT_OUT" | jq -e '[.pages[] | select(.error == null)] | any(.charCount > 0)' >/dev/null 2>&1; then+ pass_assert "preview reports content size (charCount)"+else+ fail_assert "preview reports content size (charCount)"+fi++if echo "$PT_OUT" | jq -e '.summary.browserPages == 0' >/dev/null 2>&1; then+ pass_assert "preview renders nothing in the browser"+else+ fail_assert "preview renders nothing in the browser"+fi++# The outline still tells the caller which pages would need the browser.+if echo "$PT_OUT" | jq -e '.pages[] | select(.url | endswith("app.html")) | .browserRecommended == true' >/dev/null 2>&1; then+ pass_assert "preview flags the JS-only page as needing the browser"+else+ fail_assert "preview flags the JS-only page as needing the browser"+fi++# Pick the JS-only page to expand next: it proves expand recovers via browser.+EXPAND_URL=$(echo "$PT_OUT" | jq -r '.pages[] | select(.url | endswith("app.html")) | .url' 2>/dev/null | head -1)++end_test++# ─────────────────────────────────────────────────────────────────+start_test "scrape --only expands a chosen URL at full fidelity"++pt_ok scrape "$SITE_URL" --timeout 30 --only "$EXPAND_URL" --json++if echo "$PT_OUT" | jq -e --arg u "$EXPAND_URL" '[.pages[].url] == [$u]' >/dev/null 2>&1; then+ pass_assert "expand scraped exactly the chosen URL"+else+ fail_assert "expand scraped exactly the chosen URL"+fi++if echo "$PT_OUT" | jq -e '.pages[0].source == "browser"' >/dev/null 2>&1; then+ pass_assert "expanded JS-only page was rendered in the browser"+else+ fail_assert "expanded JS-only page was rendered in the browser"+fi++if echo "$PT_OUT" | jq -e '.pages[0].markdown | contains("JavaScript")' >/dev/null 2>&1; then+ pass_assert "expanded page carries the JavaScript-rendered content"+else+ fail_assert "expanded page carries the JavaScript-rendered content"+fi++end_test