High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
package handlers
import (
"context"
"fmt"
"net/http"
"os"
"path/filepath"
"strconv"
"strings"
"sync"
"time"
"github.com/pinchtab/pinchtab/internal/bridge"
"github.com/pinchtab/pinchtab/internal/bridge/observe"
"github.com/pinchtab/pinchtab/internal/fileout"
"github.com/pinchtab/pinchtab/internal/httpx"
)
// maxExportBodyBytes caps the size of a single response body included in an export.
const maxExportBodyBytes = 10 << 20 // 10 MB
// maxExportStreamDuration caps how long a streaming export can run before auto-closing.
const maxExportStreamDuration = 30 * time.Minute
// bodyFetchConcurrency limits parallel CDP GetResponseBody calls to avoid tying up the tab.
const bodyFetchConcurrency = 4
// entryFinishWait caps how long a live-stream entry may stay pending while waiting for
// responseReceived/loadingFinished before it is exported best-effort.
const entryFinishWait = 10 * time.Second
// staleTmpAge is the minimum age of a .tmp file before it's considered orphaned.
const staleTmpAge = 5 * time.Minute
// CleanupStaleTmpExports removes orphaned .tmp files from the exports directory.
// Called at startup to handle files left behind by a crash or hard kill.
func CleanupStaleTmpExports(stateDir string) {
exportDir := filepath.Join(stateDir, "exports")
entries, err := os.ReadDir(exportDir)
if err != nil {
return
}
cutoff := time.Now().Add(-staleTmpAge)
for _, entry := range entries {
if entry.IsDir() || !strings.HasSuffix(entry.Name(), ".tmp") {
continue
}
info, err := entry.Info()
if err != nil || info.ModTime().After(cutoff) {
continue // skip files that might still be in-flight
}
_ = os.Remove(filepath.Join(exportDir, entry.Name()))
}
}
// HandleNetworkExport exports captured network data in a registered format (HAR, NDJSON, etc.).
//
// @Endpoint GET /network/export
// @Description Export captured network entries in HAR 1.2, NDJSON, or other registered formats
//
// @Param tabId string query Tab ID (optional, uses current tab)
// @Param format string query Export format: "har" (default), "ndjson", or any registered format
// @Param output string query "file" to save to disk (optional)
// @Param path string query Filename when output=file (optional, auto-generated if omitted)
// @Param body string query "true" to include response bodies (can be slow)
// @Param redact string query "false" to include sensitive headers like cookies (default: redacted)
// @Param filter string query URL pattern filter
// @Param method string query HTTP method filter
// @Param status string query Status code range filter (e.g. "4xx")
// @Param type string query Resource type filter
// @Param limit string query Maximum entries to export
//
// @Response 200 application/har+json|application/x-ndjson Exported data (encoded incrementally, in capture order)
// @Response 200 application/json File save result when output=file
// @Response 400 application/json Invalid format or parameters
// @Response 423 application/json Tab is locked
// @Response 500 application/json Export error
func (h *Handlers) HandleNetworkExport(w http.ResponseWriter, r *http.Request) {
if !h.ensureBrowserReady(w) {
return
}
ec, ok := h.resolveExportContext(w, r, "network-export")
if !ok {
return
}
nm := h.Bridge.NetworkMonitor()
if nm == nil {
// No monitor: emit an empty document so callers still get a valid export.
enc := ec.factory("PinchTab", h.version())
w.Header().Set("Content-Type", enc.ContentType())
if err := enc.Start(w); err != nil {
return
}
_ = enc.Finish()
return
}
buf, ok := h.ensureCaptureBuffer(w, r, nm, ec.tabCtx, ec.resolvedTabID)
if !ok {
return
}
filter := parseNetworkFilter(r)
entries := buf.List(filter)
if filter.Limit > 0 && len(entries) > filter.Limit {
entries = entries[len(entries)-filter.Limit:]
}
includeBody := r.URL.Query().Get("body") == "true"
redactHeaders := r.URL.Query().Get("redact") != "false"
output := r.URL.Query().Get("output")
fetchCtx, fetchCancel := context.WithTimeout(ec.tabCtx, h.Config.ActionTimeout)
defer fetchCancel()
go httpx.CancelOnClientDone(r.Context(), fetchCancel)
enc := ec.factory("PinchTab", h.version())
// Entries are encoded incrementally in capture order as their bodies resolve;
// streamExportEntries caps in-memory entries at bodyFetchConcurrency rather than
// buffering the whole []observe.ExportEntry before encoding.
if output == "file" {
if err := h.writeExportFile(w, r, enc, ec.formatName, func(emit func(observe.ExportEntry) error) error {
return h.streamExportEntries(fetchCtx, nm, entries, includeBody, redactHeaders, emit)
}); err != nil {
httpx.Error(w, 500, fmt.Errorf("write file: %w", err))
}
return
}
w.Header().Set("Content-Type", enc.ContentType())
if err := enc.Start(w); err != nil {
return
}
if err := h.streamExportEntries(fetchCtx, nm, entries, includeBody, redactHeaders, enc.Encode); err != nil {
return
}
_ = enc.Finish()
}
// exportContext is the resolved prelude shared by the two export endpoints.
type exportContext struct {
tabCtx context.Context
resolvedTabID string
factory observe.ExportEncoderFactory
formatName string
}
// resolveExportContext runs the post-browser export prelude shared by
// /network/export and /network/export/stream: tab resolution + domain policy,
// tab-lease enforcement, read-request bookkeeping, and export-format resolution.
// The caller runs the browser-ready guard (and, for the streaming variant, the
// path-required check) first so each endpoint keeps its own ordering. On any
// failure it writes the response and returns ok=false.
func (h *Handlers) resolveExportContext(w http.ResponseWriter, r *http.Request, label string) (exportContext, bool) {
tabCtx, resolvedTabID, ok := h.resolveNetworkTab(w, r)
if !ok {
return exportContext{}, false
}
owner := resolveOwner(r, "")
if err := h.enforceTabLease(resolvedTabID, owner); err != nil {
httpx.ErrorCode(w, http.StatusLocked, "tab_locked", err.Error(), false, nil)
return exportContext{}, false
}
h.recordReadRequest(r, label, resolvedTabID)
formatName := r.URL.Query().Get("format")
if formatName == "" {
formatName = "har"
}
factory := observe.GetFormat(formatName)
if factory == nil {
message := fmt.Sprintf("unknown export format %q", formatName)
httpx.JSONError(w, 400, "unknown_format", message, map[string]any{
"code": "unknown_format",
"error": message,
"available": observe.ListFormats(),
})
return exportContext{}, false
}
return exportContext{tabCtx: tabCtx, resolvedTabID: resolvedTabID, factory: factory, formatName: formatName}, true
}
// resolveExportBody returns the response body and base64 flag for an export entry,
// preferring the body already retained in the capture buffer (no CDP round-trip,
// and it survives the resource being evicted from Chrome) and only falling back to
// a live GetResponseBody fetch when nothing is retained.
func resolveExportBody(ctx context.Context, nm *bridge.NetworkMonitor, entry bridge.NetworkEntry) (string, bool) {
if entry.BodyRetained {
return clampExportBody(entry.ResponseBody, entry.Base64Encoded)
}
body, b64, _ := bridge.GetResponseBody(ctx, entry.RequestID)
return clampExportBody(body, b64)
}
// streamExportEntries converts entries to ExportEntry values and invokes emit for
// each, in the original capture order, fetching response bodies through a bounded
// concurrent pipeline. A producer goroutine cannot run more than bodyFetchConcurrency
// entries ahead of the consumer, so at most that many entries are materialized in
// memory at once — a large capture is never buffered whole. It returns the first
// emit error after draining outstanding fetches so no goroutine leaks.
func (h *Handlers) streamExportEntries(
ctx context.Context,
nm *bridge.NetworkMonitor,
entries []bridge.NetworkEntry,
includeBody, redactHeaders bool,
emit func(observe.ExportEntry) error,
) error {
results := make([]chan observe.ExportEntry, len(entries))
for i := range results {
results[i] = make(chan observe.ExportEntry, 1)
}
// A slot is acquired before a body fetch starts and released by the consumer
// once it has encoded that entry, capping in-flight (fetched-but-unencoded)
// entries — and therefore memory — at bodyFetchConcurrency.
sem := make(chan struct{}, bodyFetchConcurrency)
go func() {
var wg sync.WaitGroup
for i, entry := range entries {
sem <- struct{}{}
wg.Add(1)
go func(idx int, ent bridge.NetworkEntry) {
defer wg.Done()
var body string
var b64 bool
if includeBody && ent.Finished && !ent.Failed {
body, b64 = resolveExportBody(ctx, nm, ent)
}
results[idx] <- toExportEntry(ent, body, b64, redactHeaders)
}(i, entry)
}
wg.Wait()
}()
var emitErr error
for i := range results {
entry := <-results[i]
if emitErr == nil {
if err := emit(entry); err != nil {
emitErr = err
}
}
// Always release the slot, even after an emit error, so producers blocked
// on sem can finish (drain to avoid leaking goroutines).
<-sem
}
return emitErr
}
// resolveExportFile sanitizes userPath into the server-controlled exports dir
// (filepath.Base + SafeCreatePath + abs-prefix containment), creates the dir,
// and opens the .tmp file 0600. status is the HTTP code for err (400 for
// path/sanitization failures, 500 for filesystem failures). This is the ONLY
// path both export writers may use — do not re-implement it.
func (h *Handlers) resolveExportFile(userPath string) (absPath, tmpPath string, f *os.File, status int, err error) {
exportDir := filepath.Join(h.Config.StateDir, "exports")
if err = os.MkdirAll(exportDir, 0750); err != nil {
return "", "", nil, 500, fmt.Errorf("create dir: %w", err)
}
safeName := filepath.Base(userPath)
safePath, err := httpx.SafeCreatePath(exportDir, safeName)
if err != nil {
return "", "", nil, 400, fmt.Errorf("invalid path: %w", err)
}
absBase, _ := filepath.Abs(exportDir)
absPath, err = filepath.Abs(safePath)
if err != nil || !strings.HasPrefix(absPath, absBase+string(filepath.Separator)) {
return "", "", nil, 400, fmt.Errorf("path escapes export directory")
}
tmpPath = absPath + ".tmp"
f, err = os.OpenFile(tmpPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)
if err != nil {
return "", "", nil, 500, fmt.Errorf("create file: %w", err)
}
return absPath, tmpPath, f, 200, nil
}
// clampExportBody drops a response body that exceeds the export size cap.
func clampExportBody(body string, b64 bool) (string, bool) {
if len(body) > maxExportBodyBytes {
return "", false
}
return body, b64
}
// toExportEntry converts a captured entry to an export entry, redacting
// sensitive request/response headers when redactHeaders is set.
func toExportEntry(entry bridge.NetworkEntry, body string, b64, redactHeaders bool) observe.ExportEntry {
e := observe.NetworkEntryToExport(entry, body, b64)
if redactHeaders {
e.Request.Headers = observe.RedactSensitiveHeaders(e.Request.Headers)
e.Response.Headers = observe.RedactSensitiveHeaders(e.Response.Headers)
}
return e
}
// writeExportFile encodes an export to a temp file via encodeAll (which pushes
// each ExportEntry through emit), then atomically renames it into place. encodeAll
// drives the incremental producer so entries are never buffered whole in memory.
func (h *Handlers) writeExportFile(
w http.ResponseWriter,
r *http.Request,
enc observe.ExportEncoder,
formatName string,
encodeAll func(emit func(observe.ExportEntry) error) error,
) error {
userPath := r.URL.Query().Get("path")
autoNamed := userPath == ""
// A reservation is a real 0-byte file, so every exit that is not the completed
// rename has to remove it. Left behind it poisons the name twice over: the obvious
// name holds an empty export, and the next export in the same second is pushed onto
// a -1 suffix. Releasing it from a deferred check rather than at each return is what
// makes that exhaustive — a branch added later inherits it instead of forgetting it.
reservedPath, committed := "", false
defer func() {
if !committed && reservedPath != "" {
_ = os.Remove(reservedPath)
}
}()
if autoNamed {
// Reserve the generated name before resolveExportFile derives the tmp path from
// it. Two auto-named exports in the same second used to rename onto one file —
// and share one <path>.tmp while writing, so the loser was corrupt as well as
// lost. The reservation is a 0-byte file the final rename replaces. A caller
// ?path= is untouched and keeps overwriting.
exportDir := filepath.Join(h.Config.StateDir, "exports")
if err := os.MkdirAll(exportDir, 0750); err != nil {
return fmt.Errorf("create dir: %w", err)
}
path, err := fileout.ReserveUnique(exportDir, "network-"+exportTimestamp(), enc.FileExtension())
if err != nil {
return fmt.Errorf("reserve export name: %w", err)
}
reservedPath = path
userPath = filepath.Base(reservedPath)
}
absPath, tmpPath, f, _, err := h.resolveExportFile(userPath)
if err != nil {
return err
}
cleanup := func() {
_ = f.Close()
_ = os.Remove(tmpPath)
}
if err := enc.Start(f); err != nil {
cleanup()
return err
}
count := 0
if err := encodeAll(func(entry observe.ExportEntry) error {
if err := enc.Encode(entry); err != nil {
return err
}
count++
return nil
}); err != nil {
cleanup()
return err
}
if err := enc.Finish(); err != nil {
cleanup()
return err
}
if err := f.Close(); err != nil {
_ = os.Remove(tmpPath)
return err
}
if err := os.Rename(tmpPath, absPath); err != nil {
_ = os.Remove(tmpPath)
return err
}
committed = true
httpx.JSON(w, 200, map[string]any{
"path": absPath,
"entries": count,
"format": formatName,
})
return nil
}
// HandleTabNetworkExport handles GET /tabs/{id}/network/export.
func (h *Handlers) HandleTabNetworkExport(w http.ResponseWriter, r *http.Request) {
h.withPathTabID(w, r, h.HandleNetworkExport)
}
func parseNetworkFilter(r *http.Request) bridge.NetworkFilter {
f := bridge.NetworkFilter{
URLPattern: r.URL.Query().Get("filter"),
Method: r.URL.Query().Get("method"),
StatusRange: r.URL.Query().Get("status"),
ResourceType: r.URL.Query().Get("type"),
}
if v := r.URL.Query().Get("limit"); v != "" {
if n, err := strconv.Atoi(v); err == nil && n > 0 {
f.Limit = n
}
}
return f
}
func (h *Handlers) version() string {
if h.Version != "" {
return h.Version
}
return "dev"
}