High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
feat: add network export HTTP endpoints
Aria8f2e93cd7e72
diff --git a/internal/handlers/handlers.go b/internal/handlers/handlers.goindex f355e839..378a9dc9 100644--- a/internal/handlers/handlers.go+++ b/internal/handlers/handlers.go@@ -163,10 +163,14 @@ func (h *Handlers) RegisterRoutes(mux *http.ServeMux, doShutdown func()) {mux.HandleFunc("GET /clipboard/paste", h.HandleClipboardPaste)mux.HandleFunc("GET /network", h.HandleNetwork)mux.HandleFunc("GET /network/stream", h.HandleNetworkStream)+ mux.HandleFunc("GET /network/export", h.HandleNetworkExport)+ mux.HandleFunc("GET /network/export/stream", h.HandleNetworkExportStream)mux.HandleFunc("GET /network/{requestId}", h.HandleNetworkByID)mux.HandleFunc("POST /network/clear", h.HandleNetworkClear)mux.HandleFunc("GET /tabs/{id}/network", h.HandleTabNetwork)mux.HandleFunc("GET /tabs/{id}/network/stream", h.HandleTabNetworkStream)+ mux.HandleFunc("GET /tabs/{id}/network/export", h.HandleTabNetworkExport)+ mux.HandleFunc("GET /tabs/{id}/network/export/stream", h.HandleTabNetworkExportStream)mux.HandleFunc("GET /tabs/{id}/network/{requestId}", h.HandleTabNetworkByID)mux.HandleFunc("POST /dialog", h.HandleDialog)mux.HandleFunc("POST /tabs/{id}/dialog", h.HandleTabDialog)diff --git a/internal/handlers/network_export.go b/internal/handlers/network_export.gonew file mode 100644index 00000000..18bd0567--- /dev/null+++ b/internal/handlers/network_export.go@@ -0,0 +1,402 @@+package handlers++import (+ "fmt"+ "net/http"+ "os"+ "path/filepath"+ "strconv"+ "time"++ "github.com/pinchtab/pinchtab/internal/bridge"+ "github.com/pinchtab/pinchtab/internal/bridge/observe"+ "github.com/pinchtab/pinchtab/internal/httpx"+)++// HandleNetworkExport exports captured network data in a registered format (HAR, NDJSON, etc.).+//+// @Endpoint GET /network/export+// @Description Export captured network entries in HAR 1.2, NDJSON, or other registered formats+//+// @Param tabId string query Tab ID (optional, uses current tab)+// @Param format string query Export format: "har" (default), "ndjson", or any registered format+// @Param output string query "file" to save to disk (optional)+// @Param path string query Filename when output=file (optional, auto-generated if omitted)+// @Param body string query "true" to include response bodies (can be slow)+// @Param filter string query URL pattern filter+// @Param method string query HTTP method filter+// @Param status string query Status code range filter (e.g. "4xx")+// @Param type string query Resource type filter+// @Param limit string query Maximum entries to export+//+// @Response 200 application/har+json|application/x-ndjson Exported data (streamed)+// @Response 200 application/json File save result when output=file+// @Response 400 application/json Invalid format or parameters+// @Response 500 application/json Export error+func (h *Handlers) HandleNetworkExport(w http.ResponseWriter, r *http.Request) {+ if err := h.ensureChrome(); err != nil {+ httpx.Error(w, 500, fmt.Errorf("chrome initialization: %w", err))+ return+ }++ tabID := r.URL.Query().Get("tabId")+ tabCtx, resolvedTabID, err := h.tabContext(r, tabID)+ if err != nil {+ httpx.Error(w, 404, err)+ return+ }+ if _, ok := h.enforceCurrentTabDomainPolicy(w, r, tabCtx, resolvedTabID); !ok {+ return+ }++ // Resolve format from registry+ formatName := r.URL.Query().Get("format")+ if formatName == "" {+ formatName = "har"+ }+ factory := observe.GetFormat(formatName)+ if factory == nil {+ httpx.JSON(w, 400, map[string]any{+ "code": "unknown_format",+ "error": fmt.Sprintf("unknown export format %q", formatName),+ "available": observe.ListFormats(),+ })+ return+ }++ // Get network entries+ nm := h.Bridge.NetworkMonitor()+ if nm == nil {+ // No monitor — export empty+ enc := factory("PinchTab", h.version())+ w.Header().Set("Content-Type", enc.ContentType())+ _ = enc.Start(w)+ _ = enc.Finish()+ return+ }++ bufferSize := parseBufferSize(r)+ buf := nm.GetBuffer(resolvedTabID)+ if buf == nil {+ if err := nm.StartCaptureWithSize(tabCtx, resolvedTabID, bufferSize); err != nil {+ httpx.Error(w, 500, fmt.Errorf("start capture: %w", err))+ return+ }+ buf = nm.GetBuffer(resolvedTabID)+ }++ filter := parseNetworkFilter(r)+ entries := buf.List(filter)+ if filter.Limit > 0 && len(entries) > filter.Limit {+ entries = entries[len(entries)-filter.Limit:]+ }++ includeBody := r.URL.Query().Get("body") == "true"+ output := r.URL.Query().Get("output")++ // Convert entries+ exportEntries := make([]observe.ExportEntry, 0, len(entries))+ for _, entry := range entries {+ var body string+ var b64 bool+ if includeBody && entry.Finished && !entry.Failed {+ body, b64, _ = nm.GetResponseBody(tabCtx, entry.RequestID)+ }+ exportEntries = append(exportEntries, observe.NetworkEntryToExport(entry, body, b64))+ }++ enc := factory("PinchTab", h.version())++ if output == "file" {+ if err := h.writeExportFile(w, r, enc, exportEntries, formatName); err != nil {+ httpx.Error(w, 500, fmt.Errorf("write file: %w", err))+ }+ return+ }++ // Stream to response+ w.Header().Set("Content-Type", enc.ContentType())+ if err := enc.Start(w); err != nil {+ return+ }+ for _, entry := range exportEntries {+ if err := enc.Encode(entry); err != nil {+ return+ }+ }+ _ = enc.Finish()+}++func (h *Handlers) writeExportFile(+ w http.ResponseWriter,+ r *http.Request,+ enc observe.ExportEncoder,+ entries []observe.ExportEntry,+ formatName string,+) error {+ userPath := r.URL.Query().Get("path")+ if userPath == "" {+ ts := time.Now().Format("20060102-150405")+ userPath = fmt.Sprintf("network-%s%s", ts, enc.FileExtension())+ }++ dir := filepath.Join(h.Config.StateDir, "exports")+ if err := os.MkdirAll(dir, 0750); err != nil {+ return fmt.Errorf("create dir: %w", err)+ }++ finalPath := filepath.Join(dir, filepath.Base(userPath))+ tmpPath := finalPath + ".tmp"++ f, err := os.OpenFile(tmpPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)+ if err != nil {+ return fmt.Errorf("create temp file: %w", err)+ }++ if err := enc.Start(f); err != nil {+ f.Close()+ os.Remove(tmpPath)+ return err+ }+ for _, entry := range entries {+ if err := enc.Encode(entry); err != nil {+ f.Close()+ os.Remove(tmpPath)+ return err+ }+ }+ if err := enc.Finish(); err != nil {+ f.Close()+ os.Remove(tmpPath)+ return err+ }+ if err := f.Close(); err != nil {+ os.Remove(tmpPath)+ return err+ }++ if err := os.Rename(tmpPath, finalPath); err != nil {+ os.Remove(tmpPath)+ return err+ }++ httpx.JSON(w, 200, map[string]any{+ "path": finalPath,+ "entries": len(entries),+ "format": formatName,+ })+ return nil+}++// HandleNetworkExportStream streams network entries to a file as they arrive.+//+// @Endpoint GET /network/export/stream+// @Description Live capture: write entries to file as they are captured+//+// @Param tabId string query Tab ID+// @Param format string query Export format (default: har)+// @Param path string query Output filename (required)+// @Param body string query "true" to include response bodies+// @Param filter... (same as HandleNetworkExport)+//+// @Response 200 text/event-stream SSE progress events+func (h *Handlers) HandleNetworkExportStream(w http.ResponseWriter, r *http.Request) {+ if err := h.ensureChrome(); err != nil {+ httpx.Error(w, 500, fmt.Errorf("chrome initialization: %w", err))+ return+ }++ userPath := r.URL.Query().Get("path")+ if userPath == "" {+ httpx.Error(w, 400, fmt.Errorf("path required for streaming export"))+ return+ }++ tabID := r.URL.Query().Get("tabId")+ tabCtx, resolvedTabID, err := h.tabContext(r, tabID)+ if err != nil {+ httpx.Error(w, 404, err)+ return+ }+ if _, ok := h.enforceCurrentTabDomainPolicy(w, r, tabCtx, resolvedTabID); !ok {+ return+ }++ formatName := r.URL.Query().Get("format")+ if formatName == "" {+ formatName = "har"+ }+ factory := observe.GetFormat(formatName)+ if factory == nil {+ httpx.JSON(w, 400, map[string]any{+ "code": "unknown_format",+ "error": fmt.Sprintf("unknown format %q", formatName),+ "available": observe.ListFormats(),+ })+ return+ }++ nm := h.Bridge.NetworkMonitor()+ if nm == nil {+ httpx.Error(w, 500, fmt.Errorf("network monitor not available"))+ return+ }++ bufferSize := parseBufferSize(r)+ buf := nm.GetBuffer(resolvedTabID)+ if buf == nil {+ if err := nm.StartCaptureWithSize(tabCtx, resolvedTabID, bufferSize); err != nil {+ httpx.Error(w, 500, fmt.Errorf("start capture: %w", err))+ return+ }+ buf = nm.GetBuffer(resolvedTabID)+ }++ includeBody := r.URL.Query().Get("body") == "true"+ filter := parseNetworkFilter(r)++ // Open file+ dir := filepath.Join(h.Config.StateDir, "exports")+ if err := os.MkdirAll(dir, 0750); err != nil {+ httpx.Error(w, 500, fmt.Errorf("create dir: %w", err))+ return+ }+ finalPath := filepath.Join(dir, filepath.Base(userPath))+ f, err := os.OpenFile(finalPath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0600)+ if err != nil {+ httpx.Error(w, 500, fmt.Errorf("create file: %w", err))+ return+ }++ enc := factory("PinchTab", h.version())+ if err := enc.Start(f); err != nil {+ f.Close()+ httpx.Error(w, 500, fmt.Errorf("start encoder: %w", err))+ return+ }++ // Subscribe for live entries+ subID, ch := buf.Subscribe()+ defer buf.Unsubscribe(subID)++ // SSE setup+ flusher, ok := w.(http.Flusher)+ if !ok {+ enc.Finish()+ f.Close()+ httpx.Error(w, 500, fmt.Errorf("streaming not supported"))+ return+ }++ w.Header().Set("Content-Type", "text/event-stream")+ w.Header().Set("Cache-Control", "no-cache")+ w.Header().Set("Connection", "keep-alive")+ w.Header().Set("X-Accel-Buffering", "no")++ count := 0+ keepalive := time.NewTicker(15 * time.Second)+ defer keepalive.Stop()++ for {+ select {+ case <-r.Context().Done():+ _ = enc.Finish()+ f.Close()+ fmt.Fprintf(w, "event: done\ndata: {\"entries\":%d,\"path\":%q}\n\n", count, finalPath)+ flusher.Flush()+ return++ case entry, ok := <-ch:+ if !ok {+ _ = enc.Finish()+ f.Close()+ fmt.Fprintf(w, "event: done\ndata: {\"entries\":%d,\"path\":%q}\n\n", count, finalPath)+ flusher.Flush()+ return+ }+ if !filter.Match(entry) {+ continue+ }+ var body string+ var b64 bool+ if includeBody && entry.Finished && !entry.Failed {+ body, b64, _ = nm.GetResponseBody(tabCtx, entry.RequestID)+ }+ export := observe.NetworkEntryToExport(entry, body, b64)+ if err := enc.Encode(export); err != nil {+ _ = enc.Finish()+ f.Close()+ return+ }+ count+++ fmt.Fprintf(w, "event: export\ndata: {\"entries\":%d,\"url\":%q}\n\n", count, truncateURL(entry.URL))+ flusher.Flush()++ case <-keepalive.C:+ fmt.Fprintf(w, ": keepalive\n\n")+ flusher.Flush()+ }+ }+}++// HandleTabNetworkExport handles GET /tabs/{id}/network/export.+func (h *Handlers) HandleTabNetworkExport(w http.ResponseWriter, r *http.Request) {+ tabID := r.PathValue("id")+ if tabID == "" {+ httpx.Error(w, 400, fmt.Errorf("tab id required"))+ return+ }+ q := r.URL.Query()+ q.Set("tabId", tabID)+ req := r.Clone(r.Context())+ u := *r.URL+ u.RawQuery = q.Encode()+ req.URL = &u+ h.HandleNetworkExport(w, req)+}++// HandleTabNetworkExportStream handles GET /tabs/{id}/network/export/stream.+func (h *Handlers) HandleTabNetworkExportStream(w http.ResponseWriter, r *http.Request) {+ tabID := r.PathValue("id")+ if tabID == "" {+ httpx.Error(w, 400, fmt.Errorf("tab id required"))+ return+ }+ q := r.URL.Query()+ q.Set("tabId", tabID)+ req := r.Clone(r.Context())+ u := *r.URL+ u.RawQuery = q.Encode()+ req.URL = &u+ h.HandleNetworkExportStream(w, req)+}++// ---------------------------------------------------------------------------+// Helpers+// ---------------------------------------------------------------------------++func parseNetworkFilter(r *http.Request) bridge.NetworkFilter {+ f := bridge.NetworkFilter{+ URLPattern: r.URL.Query().Get("filter"),+ Method: r.URL.Query().Get("method"),+ StatusRange: r.URL.Query().Get("status"),+ ResourceType: r.URL.Query().Get("type"),+ }+ if v := r.URL.Query().Get("limit"); v != "" {+ if n, err := strconv.Atoi(v); err == nil && n > 0 {+ f.Limit = n+ }+ }+ return f+}++func (h *Handlers) version() string {+ return "dev"+}++func truncateURL(u string) string {+ if len(u) > 120 {+ return u[:120]+ }+ return u+}diff --git a/internal/orchestrator/handlers.go b/internal/orchestrator/handlers.goindex 93fbbe56..77bb846c 100644--- a/internal/orchestrator/handlers.go+++ b/internal/orchestrator/handlers.go@@ -80,6 +80,8 @@ func (o *Orchestrator) registerHandlers(mux *http.ServeMux, skipLaunch bool) {"POST /tabs/{id}/wait","POST /tabs/{id}/solve","POST /tabs/{id}/solve/{name}",+ "GET /tabs/{id}/network/export",+ "GET /tabs/{id}/network/export/stream",} {mux.HandleFunc(route, o.proxyTabRequest)}diff --git a/internal/strategy/autorestart/autorestart.go b/internal/strategy/autorestart/autorestart.goindex 62af7a9b..81bcd532 100644--- a/internal/strategy/autorestart/autorestart.go+++ b/internal/strategy/autorestart/autorestart.go@@ -179,7 +179,7 @@ func (s *Strategy) RegisterRoutes(mux *http.ServeMux) {"GET /console", "POST /console/clear","GET /errors", "POST /errors/clear","GET /clipboard/read", "POST /clipboard/write", "POST /clipboard/copy", "GET /clipboard/paste",- "GET /network", "GET /network/stream", "GET /network/{requestId}", "POST /network/clear",+ "GET /network", "GET /network/stream", "GET /network/export", "GET /network/export/stream", "GET /network/{requestId}", "POST /network/clear","POST /navigate", "POST /back", "POST /forward", "POST /reload","POST /action", "POST /actions","POST /dialog",diff --git a/internal/strategy/explicit/explicit.go b/internal/strategy/explicit/explicit.goindex fe277b5f..b50a8761 100644--- a/internal/strategy/explicit/explicit.go+++ b/internal/strategy/explicit/explicit.go@@ -53,6 +53,7 @@ func (s *Strategy) RegisterRoutes(mux *http.ServeMux) {"POST /find","GET /solvers","POST /solve", "POST /solve/{name}",+ "GET /network", "GET /network/stream", "GET /network/export", "GET /network/export/stream", "POST /network/clear",}for _, route := range shorthandRoutes {mux.HandleFunc(route, s.proxyToFirst)diff --git a/internal/strategy/simple/simple.go b/internal/strategy/simple/simple.goindex 188577bc..8439a0df 100644--- a/internal/strategy/simple/simple.go+++ b/internal/strategy/simple/simple.go@@ -53,7 +53,7 @@ func (s *Strategy) RegisterRoutes(mux *http.ServeMux) {"GET /console", "POST /console/clear","GET /errors", "POST /errors/clear","GET /clipboard/read", "POST /clipboard/write", "POST /clipboard/copy", "GET /clipboard/paste",- "GET /network", "GET /network/stream", "GET /network/{requestId}", "POST /network/clear",+ "GET /network", "GET /network/stream", "GET /network/export", "GET /network/export/stream", "GET /network/{requestId}", "POST /network/clear","POST /navigate", "POST /back", "POST /forward", "POST /reload","POST /action", "POST /actions","POST /dialog",