High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
package scrape
import (
"context"
"errors"
"fmt"
"net/url"
"strings"
"sync"
"time"
"unicode/utf8"
seaportal "github.com/pinchtab/seaportal"
)
// DefaultConcurrency is the number of pages browser-rendered in parallel
// when the caller does not choose one; MaxConcurrency caps what a caller may
// request (each worker drives its own browser tab).
const (
DefaultConcurrency = 2
MaxConcurrency = 8
)
// MinContentChars mirrors seaportal's static-ok confidence threshold: an
// HTTP extraction shorter than this is treated as a probable JS shell and
// the page is routed to the browser.
const MinContentChars = 500
// Crawler produces the seaportal site crawl for Run. Isolated behind a
// function type so tests can fake the crawl and handlers can bake in
// timeouts without Run knowing about seaportal options.
type Crawler func(ctx context.Context) (*seaportal.ScrapeResult, error)
// BrowserRenderer returns the rendered HTML for url from a real browser
// tab. Failures come back as errors and are recorded per page, never
// aborting the run.
type BrowserRenderer func(url string) (string, error)
// RunOptions configures a scrape run.
type RunOptions struct {
// Concurrency is the number of pages browser-rendered in parallel,
// clamped to [1, MaxConcurrency].
Concurrency int
// EnrichAll browser-renders every reachable page, overriding routing.
EnrichAll bool
// NoBrowser skips browser enrichment entirely (HTTP crawl only);
// routing verdicts are still recorded on each page.
NoBrowser bool
// Preview produces an outline: the HTTP crawl and routing verdicts, but
// no browser enrichment and no full page bodies. Each page's Markdown is
// withheld and replaced by CharCount plus a leading Snippet, so a caller
// can survey a large site cheaply and then expand chosen pages (Input.URL
// list) at full fidelity.
Preview bool
}
// SnippetChars is how many characters of a page's content the preview keeps
// as a stand-in for the withheld body.
const SnippetChars = 240
// CrawlGuard applies pinchtab's navigation security stack to the HTTP crawl
// so seaportal's fetches (robots.txt, sitemaps, discovered links, pages) go
// through the same URL vetting as browser navigation.
type CrawlGuard struct {
// ValidateURL is pinchtab's per-URL gate (navguard resolution checks +
// IDPI domain rules). It runs before every crawl fetch and on every
// redirect hop, and must be safe for concurrent use.
ValidateURL func(url string) error
// TrustedResolveCIDRs mirrors the runtime config escape hatch for hosts
// that legitimately resolve to non-public addresses.
TrustedResolveCIDRs []string
// MaxRedirects caps redirect hops when > 0; otherwise seaportal's
// default cap applies (a crawl never inherits "unlimited").
MaxRedirects int
}
// Policy renders the guard as a seaportal SecurityPolicy: the secure
// defaults (scheme allowlist, redirect cap + per-hop revalidation, body and
// decompression caps) with private-IP enforcement delegated to ValidateURL,
// which owns pinchtab's richer semantics (IDPI-allowed internal domains,
// trusted CIDRs).
func (g CrawlGuard) Policy() *seaportal.SecurityPolicy {
p := seaportal.DefaultSecurityPolicy()
p.TrustedResolveCIDRs = g.TrustedResolveCIDRs
if g.MaxRedirects > 0 {
p.MaxRedirects = g.MaxRedirects
}
if g.ValidateURL != nil {
p.BlockPrivateIPs = false
p.URLFilter = func(_ context.Context, rawURL string) error {
return g.ValidateURL(rawURL)
}
}
return p
}
// SiteCrawler is the default Crawler: seaportal.ScrapeSite over input with
// every fetch gated by guard. timeout <= 0 keeps seaportal's default
// overall deadline.
func SiteCrawler(input Input, timeout time.Duration, guard CrawlGuard) Crawler {
return func(ctx context.Context) (*seaportal.ScrapeResult, error) {
opts := &seaportal.ScrapeOptions{
BaseURL: input.URL,
MaxPages: input.MaxPages,
MaxPerPattern: input.MaxPerPattern,
IncludePatterns: input.IncludePatterns,
ExcludePatterns: input.ExcludePatterns,
Timeout: timeout,
Security: guard.Policy(),
}
return seaportal.ScrapeSite(ctx, opts)
}
}
// URLListCrawler is the Crawler for expand mode: instead of discovering a
// site, it fetches an explicit set of URLs over HTTP and extracts each, so
// Run can route and browser-render exactly the pages a caller chose from a
// prior preview. Every fetch goes through guard.Policy() — the same SSRF and
// redirect protection as the crawl. Per-URL fetch failures become failed
// pages, never aborting the run; the crawl only errors when the list is empty.
func URLListCrawler(urls []string, timeout time.Duration, guard CrawlGuard) Crawler {
return func(ctx context.Context) (*seaportal.ScrapeResult, error) {
clean := dedupeURLs(urls)
if len(clean) == 0 {
return nil, errors.New("no urls to expand")
}
policy := guard.Policy()
pages := make([]seaportal.PageObject, 0, len(clean))
for _, u := range clean {
pages = append(pages, fetchOne(ctx, u, timeout, policy))
}
return &seaportal.ScrapeResult{
Site: seaportal.SiteInfo{BaseURL: originOf(clean[0])},
Pages: pages,
}, nil
}
}
// fetchOne HTTP-fetches url through the security policy and extracts it into
// a PageObject. Fetch or extraction failures are recorded on the page's Error
// so the run continues and the page can still route to the browser.
func fetchOne(ctx context.Context, url string, timeout time.Duration, policy *seaportal.SecurityPolicy) seaportal.PageObject {
body, hdr, status, err := seaportal.FetchBytes(ctx, url, seaportal.FetchBytesOptions{
Security: policy,
Timeout: timeout,
})
if err != nil {
return seaportal.PageObject{URL: url, Status: status, Error: err.Error()}
}
r := seaportal.FromHTML(string(body), url)
p := seaportal.PageObject{
URL: url,
Status: status,
ContentType: hdr.Get("Content-Type"),
Title: r.Title,
Markdown: r.Content,
Error: r.Error,
}
if r.Description != "" {
p.Meta = map[string]string{"description": r.Description}
}
return p
}
// blockedStatuses are HTTP statuses where a real browser (with its stealth
// and challenge handling) may succeed where a plain HTTP client was refused.
var blockedStatuses = map[int]bool{401: true, 403: true, 407: true, 429: true, 503: true}
// NeedsBrowser is the routing verdict for one HTTP-extracted page: whether
// the browser should re-render it, and why. Not-found pages never route to
// the browser — re-rendering a 404 cannot recover content.
func NeedsBrowser(p Page) (bool, []string) {
if p.StatusCode == 404 || p.StatusCode == 410 {
return false, nil
}
var reasons []string
if p.Error != "" {
reasons = append(reasons, "fetch-error")
}
if blockedStatuses[p.StatusCode] {
reasons = append(reasons, fmt.Sprintf("blocked-status:%d", p.StatusCode))
}
if len(reasons) == 0 && len(strings.TrimSpace(p.Markdown)) < MinContentChars {
reasons = append(reasons, "thin-content")
}
return len(reasons) > 0, reasons
}
// Run executes the scrape pipeline: crawl the site over HTTP, route each
// page, then browser-render the routed pages with bounded concurrency and
// re-extract their content from the rendered HTML. Page failures are report
// data, not errors — Run only errors when the crawl itself fails or finds
// nothing.
func Run(ctx context.Context, input Input, opts RunOptions, crawl Crawler, render BrowserRenderer) (Report, error) {
res, err := crawl(ctx)
if err != nil {
return Report{}, fmt.Errorf("site crawl: %w", err)
}
if len(res.Pages) == 0 {
return Report{}, errors.New("no pages discovered")
}
pages := make([]Page, len(res.Pages))
enrich := make([]bool, len(res.Pages))
for i, sp := range res.Pages {
p := fromSeaportal(sp)
needs, reasons := NeedsBrowser(p)
p.BrowserRecommended = needs
p.BrowserReasons = reasons
if opts.EnrichAll && !needs && p.StatusCode != 404 && p.StatusCode != 410 {
needs, p.BrowserReasons = true, []string{"enrich-all"}
}
if opts.Preview {
p.CharCount = utf8.RuneCountInString(p.Markdown)
p.Snippet = snippet(p.Markdown, SnippetChars)
p.Markdown = ""
}
pages[i] = p
enrich[i] = needs && !opts.NoBrowser && !opts.Preview && render != nil
}
sem := make(chan struct{}, clampConcurrency(opts.Concurrency))
var wg sync.WaitGroup
for i := range pages {
if !enrich[i] {
continue
}
wg.Add(1)
go func(p *Page) {
defer wg.Done()
sem <- struct{}{}
defer func() { <-sem }()
enrichPage(p, render)
}(&pages[i])
}
wg.Wait()
return Report{
SchemaVersion: SchemaVersion,
GeneratedAt: time.Now().UTC(),
Input: input,
Site: SiteInfo{
BaseURL: res.Site.BaseURL,
Title: res.Site.Title,
SitemapFound: res.Site.SitemapFound,
TotalURLsInSitemap: res.Site.TotalURLsInSitemap,
SampledPages: len(pages),
},
PageGroups: fromSeaportalGroups(res.PageGroups),
Pages: pages,
Summary: summarize(pages, res.Summary.Recommendations),
}, nil
}
// PageMarkdown is one rendered document converted to Markdown through
// seaportal: the body with the title and description the converter recovered,
// plus its error. Markdown is empty when the converter errored or produced no
// content, so a caller can fall back to raw text.
type PageMarkdown struct {
Markdown string
Title string
Description string
Err string
}
// ToMarkdown converts one document's HTML to Markdown through the seaportal
// engine. Markdown is blank when the converter errors or yields no content;
// Title, Description and Err are always reported. It is the single owner of the
// (html, url) → Markdown conversion shared by enrichPage and the /text handler.
func ToMarkdown(html, url string) PageMarkdown {
r := seaportal.FromHTML(html, url)
md := PageMarkdown{Title: r.Title, Description: r.Description, Err: r.Error}
if strings.TrimSpace(r.Content) != "" {
md.Markdown = r.Content
}
return md
}
// enrichPage renders p in the browser and replaces its content with the
// extraction over the rendered HTML. The HTTP extraction is kept whenever
// the browser path fails or yields nothing.
func enrichPage(p *Page, render BrowserRenderer) {
html, err := render(p.URL)
if err != nil {
p.BrowserError = err.Error()
return
}
md := ToMarkdown(html, p.URL)
if md.Err != "" {
p.BrowserError = md.Err
return
}
if md.Markdown == "" {
p.BrowserError = "browser extraction produced no content"
return
}
p.Markdown = md.Markdown
p.Source = SourceBrowser
// The browser reached the page and produced content, so an HTTP fetch
// failure no longer marks the page as failed.
p.Error = ""
if md.Title != "" {
p.Title = md.Title
}
if p.Meta == nil && md.Description != "" {
p.Meta = map[string]string{"description": md.Description}
}
}
func fromSeaportal(sp seaportal.PageObject) Page {
return Page{
URL: sp.URL,
Title: sp.Title,
StatusCode: sp.Status,
ContentType: sp.ContentType,
Markdown: sp.Markdown,
Meta: sp.Meta,
Schema: sp.Schema,
InternalLinks: sp.InternalLinks,
ExternalLinks: sp.ExternalLinks,
Source: SourceHTTP,
Error: sp.Error,
}
}
// fromSeaportalGroups keeps the site tree as URL references; page content
// lives once in Report.Pages.
func fromSeaportalGroups(groups []seaportal.PageGroup) []PageGroup {
out := make([]PageGroup, 0, len(groups))
for _, g := range groups {
urls := make([]string, 0, len(g.Pages))
for _, p := range g.Pages {
urls = append(urls, p.URL)
}
out = append(out, PageGroup{
Pattern: g.Pattern,
Total: g.TotalInSitemap,
Sampled: g.Sampled,
URLs: urls,
})
}
return out
}
// ThinContentChars is the post-enrichment thin-content threshold: a page whose final
// extraction is shorter than this has little usable text.
//
// PinchTab OWNS this number. seaportal applies the same 160 at the pinned version, but
// its constant is unexported AND inside another module's internal/ tree, so no import,
// linkname or reflection can reach it — claiming to mirror it would be a promise this
// code cannot keep. Owning it means a future upstream change becomes a visible decision
// here rather than a silent divergence with nothing failing.
const ThinContentChars = 160
// regeneratedRecommendations are the upstream recommendations this package rebuilds
// from the final pages, matched on the stable phrase in each — the errors/4xx line and
// the thin-content line.
//
// THE RULE, recorded where the choice is made, so the next upstream recommendation is
// classified rather than guessed at: a recommendation derived from PAGE CONTENT must be
// regenerated after enrichment, because the browser phase changes exactly that; one
// derived from CRAWL SCOPE is forwarded, because the browser phase changes nothing about
// which URLs were sampled. Both sitemap lines are scope and forward by matching nothing
// here; the errors and thin-content lines are content, and both are regenerated below
// from the final pages.
//
// An unrecognised recommendation is FORWARDED, since dropping advice that is still true
// is the worse failure — which is why this list names what is DROPPED, so a new upstream
// line takes the forwarding default until it is classified here.
var regeneratedRecommendations = []string{"extractable text", "returned errors"}
// summarize rolls the FINAL pages up. Every field including the prose comes from this one
// slice: the recommendations used to be carried in verbatim from the HTTP crawl, so the
// numbers were post-render and the sentences pre-render inside one object — a run that
// enriched every page still advised the reader to consider enrichment, and counted a page
// as having little text while reporting its 2012 characters two lines above.
func summarize(pages []Page, inherited []string) Summary {
s := Summary{ContentTypes: map[string]int{}}
for _, p := range pages {
// A failed page is not an ordinary content sample: keep its error body out of
// the contentTypes taxonomy so a consumer is not told a 404 is a `page`.
if !PageFailed(p) && p.ContentType != "" {
s.ContentTypes[p.ContentType]++
}
switch {
case PageFailed(p):
s.FailedPages++
case p.Source == SourceBrowser:
s.BrowserPages++
default:
s.HTTPPages++
}
}
if len(s.ContentTypes) == 0 {
s.ContentTypes = nil
}
s.Recommendations = recommend(pages, s, inherited)
return s
}
// pageFailed reports whether a page is a failure rather than a content sample: a
// transport error, or a 4xx/5xx response the browser did not recover. A 3xx or an
// unset (zero) status is not a failure — the threshold is >= 400 — so a redirect
// artefact or a page that never carried a status stays an ordinary page. A page the
// browser rendered into content (Source browser, which enrichPage sets only on
// success and where it clears Error) is not failed even if its HTTP status was
// >= 400, mirroring enrichPage's rule that a recovered fetch failure no longer marks
// the page failed; its statusCode is left intact. This is the one owner of the rule,
// read by both the summary partition and the thin-content count.
func PageFailed(p Page) bool {
if p.Error != "" {
return true
}
return p.StatusCode >= 400 && p.Source != SourceBrowser
}
// recommend keeps the crawl-scope advice the HTTP phase produced and regenerates the
// page-content advice from the pages as they finally stand.
func recommend(pages []Page, s Summary, inherited []string) []string {
var recs []string
if s.FailedPages > 0 {
recs = append(recs, fmt.Sprintf("%d of %d pages returned errors or 4xx/5xx responses", s.FailedPages, len(pages)))
}
if thin, unenriched := thinPages(pages); thin > 0 {
// The advice to enrich only survives while it is still actionable. A page the
// browser already rendered — or tried to and failed — has had that remedy
// applied, so repeating it sends the reader on a re-run that changes nothing.
line := fmt.Sprintf("%d pages have little extractable text (possible SPA/JS-only)", thin)
if unenriched > 0 {
line += "; consider PinchTab enrichment"
}
recs = append(recs, line)
}
for _, rec := range inherited {
if !regeneratedLocally(rec) {
recs = append(recs, rec)
}
}
return recs
}
// thinPages counts the pages whose FINAL content is thin, and how many of those never
// had a browser attempt — the second number is what decides whether advising enrichment
// is still useful. A failed page is not also thin: it has no content for a reason that
// the errors line already reports.
func thinPages(pages []Page) (thin, unenriched int) {
for _, p := range pages {
if PageFailed(p) {
continue
}
if contentChars(p) >= ThinContentChars {
continue
}
thin++
if p.Source != SourceBrowser && p.BrowserError == "" {
unenriched++
}
}
return thin, unenriched
}
// contentChars is a page's extracted length, read from CharCount in preview mode where
// the body is deliberately withheld and Markdown is empty for every page.
func contentChars(p Page) int {
if p.Markdown != "" {
return utf8.RuneCountInString(strings.TrimSpace(p.Markdown))
}
return p.CharCount
}
func regeneratedLocally(rec string) bool {
for _, phrase := range regeneratedRecommendations {
if strings.Contains(rec, phrase) {
return true
}
}
return false
}
// clampConcurrency normalizes a requested concurrency into [1, MaxConcurrency].
func clampConcurrency(n int) int {
switch {
case n < 1:
return DefaultConcurrency
case n > MaxConcurrency:
return MaxConcurrency
default:
return n
}
}
// snippet returns the first max characters of s with runs of whitespace
// collapsed to single spaces, appending an ellipsis when it truncated.
func snippet(s string, max int) string {
s = strings.Join(strings.Fields(s), " ")
if utf8.RuneCountInString(s) <= max {
return s
}
runes := []rune(s)
return strings.TrimSpace(string(runes[:max])) + "…"
}
// dedupeURLs trims, drops blanks, and removes duplicate URLs while keeping
// first-seen order.
func dedupeURLs(urls []string) []string {
seen := make(map[string]bool, len(urls))
out := make([]string, 0, len(urls))
for _, u := range urls {
u = strings.TrimSpace(u)
if u == "" || seen[u] {
continue
}
seen[u] = true
out = append(out, u)
}
return out
}
// originOf returns the scheme://host of a URL, or the raw string when it does
// not parse — used only to label the expand result's base URL.
func originOf(raw string) string {
u, err := url.Parse(raw)
if err != nil || u.Host == "" {
return raw
}
return u.Scheme + "://" + u.Host
}