Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

6.1 KB
Raw Blame
package handlers

import (
	"context"
	"encoding/json"
	"fmt"
	"net/http"
	"strings"
	"time"

	"github.com/pinchtab/pinchtab/internal/bridge"
	"github.com/pinchtab/pinchtab/internal/bridge/observe"
	"github.com/pinchtab/pinchtab/internal/config"
	"github.com/pinchtab/pinchtab/internal/httpx"
	"github.com/pinchtab/pinchtab/internal/scrape"
)

const (
	scrapeRenderTimeout = 60 * time.Second
	// scrapeRunTimeout bounds a whole scrape run (HTTP crawl plus every
	// browser-rendered page) and is how far the response write deadline is
	// extended past the server's default WriteTimeout. Matches the CLI's
	// client-side scrape timeout.
	scrapeRunTimeout = 15 * time.Minute
)

type scrapeRequest struct {
	URL             string   `json:"url"`
	Browser         string   `json:"browser"`
	MaxPages        int      `json:"maxPages"`
	MaxPerPattern   int      `json:"maxPerPattern"`
	IncludePatterns []string `json:"includePatterns"`
	ExcludePatterns []string `json:"excludePatterns"`
	Concurrency     int      `json:"concurrency"`
	EnrichAll       bool     `json:"enrichAll"`
	NoBrowser       bool     `json:"noBrowser"`
	TimeoutSeconds  int      `json:"timeoutSeconds"`
	// Preview returns an outline (routing verdicts, per-page snippet and
	// charCount) without browser rendering or full page bodies.
	Preview bool `json:"preview"`
	// Only, when set, expands exactly these URLs at full fidelity instead of
	// discovering the site — the drill-down half of a preview.
	Only []string `json:"only"`
}

// @Endpoint POST /scrape
func (h *Handlers) HandleScrape(w http.ResponseWriter, r *http.Request) {
	var req scrapeRequest
	if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxBodySize)).Decode(&req); err != nil {
		httpx.Error(w, 400, fmt.Errorf("decode: %w", err))
		return
	}
	req.URL = strings.TrimSpace(req.URL)
	if req.URL == "" {
		httpx.Error(w, 400, fmt.Errorf("url required"))
		return
	}

	routing, ok := h.resolveNavigateBrowser(w, r, "", strings.TrimSpace(req.Browser))
	if !ok {
		return
	}
	if _, err := h.validateAuditTarget(req.URL, routing.EffectiveCfg); err != nil {
		httpx.Error(w, 400, err)
		return
	}
	for _, u := range req.Only {
		if strings.TrimSpace(u) == "" {
			continue
		}
		if _, err := h.validateAuditTarget(u, routing.EffectiveCfg); err != nil {
			httpx.Error(w, 400, fmt.Errorf("expand target %q: %w", u, err))
			return
		}
	}
	// A preview never renders, so it needs no browser; expand and full runs do.
	if !req.NoBrowser && !req.Preview && !h.ensureBrowserOrRespond(w, routing.EffectiveCfg) {
		return
	}

	// A multi-page run legitimately outlives the server's WriteTimeout;
	// extend the write deadline and bound the whole run to the same window.
	httpx.ExtendWriteDeadline(w, scrapeRunTimeout)
	runCtx, runCancel := context.WithTimeout(r.Context(), scrapeRunTimeout)
	defer runCancel()

	input := scrape.Input{
		URL:             req.URL,
		MaxPages:        req.MaxPages,
		MaxPerPattern:   req.MaxPerPattern,
		IncludePatterns: req.IncludePatterns,
		ExcludePatterns: req.ExcludePatterns,
	}
	guard := h.crawlGuard(routing.EffectiveCfg)
	renderer := func(url string) (string, error) {
		targets, err := h.validateAuditTarget(url, routing.EffectiveCfg)
		if err != nil {
			return "", err
		}
		return h.renderPageHTML(runCtx, targets.url, routing.EffectiveCfg, targets)
	}

	// Expand a chosen URL set instead of discovering the site when --only is
	// given; otherwise crawl from the base URL.
	timeout := time.Duration(req.TimeoutSeconds) * time.Second
	crawler := scrape.SiteCrawler(input, timeout, guard)
	if len(req.Only) > 0 {
		crawler = scrape.URLListCrawler(req.Only, timeout, guard)
	}

	report, err := scrape.Run(
		runCtx,
		input,
		scrape.RunOptions{
			Concurrency: req.Concurrency,
			EnrichAll:   req.EnrichAll,
			NoBrowser:   req.NoBrowser,
			Preview:     req.Preview,
		},
		crawler,
		renderer,
	)
	if err != nil {
		httpx.Error(w, 400, err)
		return
	}
	httpx.JSON(w, 200, report)
}

// crawlGuard adapts this instance's navigation security stack (navguard
// resolution checks + IDPI domain rules + trusted CIDRs) into the guard the
// seaportal HTTP crawl applies to every fetch and redirect hop.
func (h *Handlers) crawlGuard(cfg *config.RuntimeConfig) scrape.CrawlGuard {
	return scrape.CrawlGuard{
		ValidateURL: func(url string) error {
			_, err := h.validateAuditTarget(url, cfg)
			return err
		},
		TrustedResolveCIDRs: cfg.TrustedResolveCIDRs,
		MaxRedirects:        cfg.MaxRedirects,
	}
}

// renderPageHTML navigates a fresh tab to url and returns the rendered
// document HTML after the page settles. The same navigation guard and
// error-page detection as auditPage apply; the tab is always closed.
func (h *Handlers) renderPageHTML(clientCtx context.Context, url string, cfg *config.RuntimeConfig, targets navTargets) (string, error) {
	tabID, tabCtx, _, err := h.Bridge.CreateTab("")
	if err != nil {
		return "", fmt.Errorf("new tab: %w", err)
	}
	defer func() { _ = h.Bridge.CloseTab(tabID) }()

	navTimeout := cfg.NavigateTimeout
	if navTimeout <= 0 {
		navTimeout = 30 * time.Second
	}
	navCtx, navCancel := context.WithTimeout(tabCtx, navTimeout)
	defer navCancel()

	navGuard, err := installNavigateRuntimeGuardWithBridge(h.Bridge, navCtx, navCancel, targets.target, targets.trustedCIDRs)
	if err != nil {
		return "", fmt.Errorf("navigation guard: %w", err)
	}

	if _, navErr := h.Bridge.Navigate(navCtx, url, bridge.NavigateParams{MaxRedirects: cfg.MaxRedirects}); navErr != nil {
		if navGuard != nil {
			if blockedErr := navGuard.blocked(); blockedErr != nil {
				navErr = blockedErr
			}
		}
		return "", navErr
	}

	if cur, urlErr := h.Bridge.CurrentURL(navCtx); urlErr == nil && strings.HasPrefix(cur, "chrome-error://") {
		return "", h.documentNetError(tabID, url)
	}

	cCtx, cCancel := context.WithTimeout(tabCtx, scrapeRenderTimeout)
	defer cCancel()
	go httpx.CancelOnClientDone(clientCtx, cCancel)

	// Let JS rendering and late subresources land before reading the DOM.
	_, _ = observe.WaitForQuietWindow(cCtx, 500*time.Millisecond, 5*time.Second)

	var html string
	if err := h.Bridge.Evaluate(cCtx, "document.documentElement.outerHTML", &html, bridge.EvalOpts{}); err != nil {
		return "", fmt.Errorf("read rendered html: %w", err)
	}
	return html, nil
}