High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
feat: add pluggable network export format registry
Ariac061ca7b31f0
diff --git a/internal/bridge/observe/format_har.go b/internal/bridge/observe/format_har.gonew file mode 100644index 00000000..5574da98--- /dev/null+++ b/internal/bridge/observe/format_har.go@@ -0,0 +1,68 @@+package observe++import (+ "encoding/json"+ "fmt"+ "io"+)++func init() {+ RegisterFormat("har", func(creatorName, creatorVersion string) ExportEncoder {+ return &harEncoder{+ creator: harCreator{Name: creatorName, Version: creatorVersion},+ }+ })+}++// harEncoder writes HAR 1.2 JSON incrementally.+type harEncoder struct {+ creator harCreator+ w io.Writer+ entryCount int+}++type harCreator struct {+ Name string `json:"name"`+ Version string `json:"version"`+}++func (e *harEncoder) ContentType() string { return "application/har+json" }+func (e *harEncoder) FileExtension() string { return ".har" }++func (e *harEncoder) Start(w io.Writer) error {+ e.w = w+ header := fmt.Sprintf(+ `{"log":{"version":"1.2","creator":%s,"entries":[`,+ mustMarshal(e.creator),+ )+ _, err := io.WriteString(e.w, header)+ return err+}++func (e *harEncoder) Encode(entry ExportEntry) error {+ if e.entryCount > 0 {+ if _, err := io.WriteString(e.w, ","); err != nil {+ return err+ }+ }+ data, err := json.Marshal(entry)+ if err != nil {+ return err+ }+ _, err = e.w.Write(data)+ e.entryCount+++ return err+}++func (e *harEncoder) Finish() error {+ _, err := io.WriteString(e.w, "]}}\n")+ return err+}++func mustMarshal(v any) string {+ b, err := json.Marshal(v)+ if err != nil {+ return "{}"+ }+ return string(b)+}diff --git a/internal/bridge/observe/format_ndjson.go b/internal/bridge/observe/format_ndjson.gonew file mode 100644index 00000000..d05ef5ed--- /dev/null+++ b/internal/bridge/observe/format_ndjson.go@@ -0,0 +1,34 @@+package observe++import (+ "encoding/json"+ "io"+)++func init() {+ RegisterFormat("ndjson", func(_, _ string) ExportEncoder {+ return &ndjsonEncoder{}+ })+}++// ndjsonEncoder writes one JSON object per line (Newline Delimited JSON).+type ndjsonEncoder struct {+ enc *json.Encoder+}++func (e *ndjsonEncoder) ContentType() string { return "application/x-ndjson" }+func (e *ndjsonEncoder) FileExtension() string { return ".ndjson" }++func (e *ndjsonEncoder) Start(w io.Writer) error {+ e.enc = json.NewEncoder(w)+ e.enc.SetEscapeHTML(false)+ return nil+}++func (e *ndjsonEncoder) Encode(entry ExportEntry) error {+ return e.enc.Encode(entry)+}++func (e *ndjsonEncoder) Finish() error {+ return nil+}diff --git a/internal/bridge/observe/network_export.go b/internal/bridge/observe/network_export.gonew file mode 100644index 00000000..81c3edc9--- /dev/null+++ b/internal/bridge/observe/network_export.go@@ -0,0 +1,245 @@+package observe++import (+ "io"+ "net/url"+ "sort"+ "strings"+ "sync"+)++// ---------------------------------------------------------------------------+// ExportEncoder — pluggable format interface+// ---------------------------------------------------------------------------++// ExportEncoder streams network entries to an io.Writer in a specific format.+// Implementations are registered via RegisterFormat and created per-export.+// Not concurrent-safe; callers serialize Start → Encode* → Finish.+type ExportEncoder interface {+ // ContentType returns the MIME type for HTTP Content-Type header.+ ContentType() string++ // FileExtension returns the default extension including dot (e.g. ".har").+ FileExtension() string++ // Start writes any format preamble to w. Called once before the first Encode.+ Start(w io.Writer) error++ // Encode writes a single entry. Called zero or more times between Start and Finish.+ Encode(entry ExportEntry) error++ // Finish writes any format trailer. Called once after the last Encode.+ // Must produce valid output even if Encode was never called.+ Finish() error+}++// ExportEncoderFactory creates a new encoder for a single export session.+type ExportEncoderFactory func(creatorName, creatorVersion string) ExportEncoder++// ---------------------------------------------------------------------------+// Format registry+// ---------------------------------------------------------------------------++var (+ formatsMu sync.RWMutex+ formats = map[string]ExportEncoderFactory{}+)++// RegisterFormat registers an export format. Called from init() in format files.+func RegisterFormat(name string, factory ExportEncoderFactory) {+ formatsMu.Lock()+ defer formatsMu.Unlock()+ formats[name] = factory+}++// GetFormat returns the factory for a named format, or nil if not found.+func GetFormat(name string) ExportEncoderFactory {+ formatsMu.RLock()+ defer formatsMu.RUnlock()+ return formats[strings.ToLower(name)]+}++// ListFormats returns all registered format names, sorted.+func ListFormats() []string {+ formatsMu.RLock()+ defer formatsMu.RUnlock()+ names := make([]string, 0, len(formats))+ for name := range formats {+ names = append(names, name)+ }+ sort.Strings(names)+ return names+}++// ---------------------------------------------------------------------------+// ExportEntry — canonical intermediate for all formats+// ---------------------------------------------------------------------------++// ExportEntry is the normalized representation of a network request/response.+// All export formats encode from this struct.+type ExportEntry struct {+ StartedDateTime string `json:"startedDateTime"`+ Time float64 `json:"time"`+ Request ExportRequest `json:"request"`+ Response ExportResponse `json:"response"`+ Timings ExportTimings `json:"timings"`+}++// ExportRequest holds the request portion of an entry.+type ExportRequest struct {+ Method string `json:"method"`+ URL string `json:"url"`+ HTTPVersion string `json:"httpVersion"`+ Headers []NameValuePair `json:"headers"`+ QueryString []NameValuePair `json:"queryString"`+ PostData *ExportPostData `json:"postData,omitempty"`+ HeadersSize int `json:"headersSize"`+ BodySize int `json:"bodySize"`+}++// ExportResponse holds the response portion of an entry.+type ExportResponse struct {+ Status int `json:"status"`+ StatusText string `json:"statusText"`+ HTTPVersion string `json:"httpVersion"`+ Headers []NameValuePair `json:"headers"`+ Content ExportContent `json:"content"`+ HeadersSize int `json:"headersSize"`+ BodySize int64 `json:"bodySize"`+}++// ExportContent holds the response body metadata and optional text.+type ExportContent struct {+ Size int64 `json:"size"`+ MimeType string `json:"mimeType"`+ Text string `json:"text,omitempty"`+ Encoding string `json:"encoding,omitempty"`+}++// ExportTimings holds timing breakdown. Values are in milliseconds.+// -1 means not applicable. send, wait, receive are always >= 0.+type ExportTimings struct {+ Send float64 `json:"send"`+ Wait float64 `json:"wait"`+ Receive float64 `json:"receive"`+}++// ExportPostData holds request body information.+type ExportPostData struct {+ MimeType string `json:"mimeType"`+ Text string `json:"text"`+}++// NameValuePair is a generic name-value pair used for headers and query params.+type NameValuePair struct {+ Name string `json:"name"`+ Value string `json:"value"`+}++// ---------------------------------------------------------------------------+// Conversion: NetworkEntry → ExportEntry+// ---------------------------------------------------------------------------++// NetworkEntryToExport converts a captured NetworkEntry to the canonical ExportEntry.+// body and base64Encoded are provided when response bodies are fetched on demand.+func NetworkEntryToExport(entry NetworkEntry, body string, base64Encoded bool) ExportEntry {+ e := ExportEntry{+ StartedDateTime: entry.StartTime.UTC().Format("2006-01-02T15:04:05.000Z"),+ Time: entry.Duration,+ Request: ExportRequest{+ Method: entry.Method,+ URL: entry.URL,+ HTTPVersion: "HTTP/1.1",+ Headers: mapToNameValuePairs(entry.RequestHeaders),+ QueryString: parseQueryString(entry.URL),+ HeadersSize: -1,+ BodySize: len(entry.PostData),+ },+ Response: ExportResponse{+ Status: entry.Status,+ StatusText: entry.StatusText,+ HTTPVersion: "HTTP/1.1",+ Headers: mapToNameValuePairs(entry.ResponseHeaders),+ Content: ExportContent{+ Size: entry.Size,+ MimeType: entry.MimeType,+ },+ HeadersSize: -1,+ BodySize: entry.Size,+ },+ Timings: computeTimings(entry),+ }++ if entry.PostData != "" {+ e.Request.PostData = &ExportPostData{+ MimeType: contentTypeFromHeaders(entry.RequestHeaders),+ Text: entry.PostData,+ }+ }++ if body != "" {+ e.Response.Content.Text = body+ if base64Encoded {+ e.Response.Content.Encoding = "base64"+ }+ }++ return e+}++func mapToNameValuePairs(headers map[string]string) []NameValuePair {+ if len(headers) == 0 {+ return []NameValuePair{}+ }+ pairs := make([]NameValuePair, 0, len(headers))+ for k, v := range headers {+ pairs = append(pairs, NameValuePair{Name: k, Value: v})+ }+ sort.Slice(pairs, func(i, j int) bool { return pairs[i].Name < pairs[j].Name })+ return pairs+}++func parseQueryString(rawURL string) []NameValuePair {+ u, err := url.Parse(rawURL)+ if err != nil || u.RawQuery == "" {+ return []NameValuePair{}+ }+ params := u.Query()+ pairs := make([]NameValuePair, 0, len(params))+ for k, vals := range params {+ for _, v := range vals {+ pairs = append(pairs, NameValuePair{Name: k, Value: v})+ }+ }+ sort.Slice(pairs, func(i, j int) bool { return pairs[i].Name < pairs[j].Name })+ return pairs+}++func computeTimings(entry NetworkEntry) ExportTimings {+ total := entry.Duration+ if total <= 0 {+ return ExportTimings{Send: 0, Wait: 0, Receive: 0}+ }+ // Approximate breakdown: most time is wait (TTFB).+ // Without CDP-level timing detail, allocate 1ms send, 1ms receive, rest is wait.+ send := 1.0+ receive := 1.0+ if total < 3 {+ send = 0+ receive = 0+ }+ wait := total - send - receive+ if wait < 0 {+ wait = 0+ }+ return ExportTimings{Send: send, Wait: wait, Receive: receive}+}++func contentTypeFromHeaders(headers map[string]string) string {+ for k, v := range headers {+ if strings.EqualFold(k, "content-type") {+ return v+ }+ }+ return ""+}diff --git a/internal/bridge/observe/network_export_test.go b/internal/bridge/observe/network_export_test.gonew file mode 100644index 00000000..42353ebb--- /dev/null+++ b/internal/bridge/observe/network_export_test.go@@ -0,0 +1,188 @@+package observe++import (+ "bytes"+ "encoding/json"+ "strings"+ "testing"+ "time"+)++func TestNetworkEntryToExport(t *testing.T) {+ entry := NetworkEntry{+ RequestID: "req1",+ URL: "https://example.com/api?key=val&page=1",+ Method: "POST",+ Status: 201,+ StatusText: "Created",+ MimeType: "application/json",+ ResourceType: "XHR",+ StartTime: time.Date(2026, 3, 28, 12, 0, 0, 0, time.UTC),+ EndTime: time.Date(2026, 3, 28, 12, 0, 0, 150_000_000, time.UTC),+ Duration: 150,+ Size: 512,+ Finished: true,+ PostData: `{"name":"test"}`,+ RequestHeaders: map[string]string{"Content-Type": "application/json", "Accept": "application/json"},+ ResponseHeaders: map[string]string{"Content-Type": "application/json", "X-Request-Id": "abc"},+ }++ e := NetworkEntryToExport(entry, `{"id":1}`, false)++ if e.StartedDateTime != "2026-03-28T12:00:00.000Z" {+ t.Errorf("startedDateTime = %q", e.StartedDateTime)+ }+ if e.Time != 150 {+ t.Errorf("time = %v", e.Time)+ }+ if e.Request.Method != "POST" {+ t.Errorf("method = %q", e.Request.Method)+ }+ if len(e.Request.QueryString) != 2 {+ t.Errorf("queryString len = %d", len(e.Request.QueryString))+ }+ if e.Request.PostData == nil || e.Request.PostData.Text != `{"name":"test"}` {+ t.Errorf("postData = %v", e.Request.PostData)+ }+ if e.Response.Status != 201 {+ t.Errorf("status = %d", e.Response.Status)+ }+ if e.Response.Content.Text != `{"id":1}` {+ t.Errorf("content.text = %q", e.Response.Content.Text)+ }+ if e.Timings.Wait < 0 {+ t.Errorf("wait < 0: %v", e.Timings.Wait)+ }+}++func TestHAREncoder(t *testing.T) {+ factory := GetFormat("har")+ if factory == nil {+ t.Fatal("har format not registered")+ }++ entry := makeTestExportEntry()+ var buf bytes.Buffer+ enc := factory("TestApp", "1.0")+ if err := enc.Start(&buf); err != nil {+ t.Fatal(err)+ }+ if err := enc.Encode(entry); err != nil {+ t.Fatal(err)+ }+ if err := enc.Encode(entry); err != nil {+ t.Fatal(err)+ }+ if err := enc.Finish(); err != nil {+ t.Fatal(err)+ }++ // Verify valid JSON+ var har struct {+ Log struct {+ Version string `json:"version"`+ Creator struct{ Name string } `json:"creator"`+ Entries []json.RawMessage `json:"entries"`+ } `json:"log"`+ }+ if err := json.Unmarshal(buf.Bytes(), &har); err != nil {+ t.Fatalf("invalid HAR JSON: %v\nraw: %s", err, buf.String()[:200])+ }+ if har.Log.Version != "1.2" {+ t.Errorf("version = %q", har.Log.Version)+ }+ if har.Log.Creator.Name != "TestApp" {+ t.Errorf("creator = %q", har.Log.Creator.Name)+ }+ if len(har.Log.Entries) != 2 {+ t.Errorf("entries = %d", len(har.Log.Entries))+ }+}++func TestHAREncoder_Empty(t *testing.T) {+ factory := GetFormat("har")+ var buf bytes.Buffer+ enc := factory("Test", "0")+ enc.Start(&buf)+ enc.Finish()++ var har struct{ Log struct{ Entries []json.RawMessage } }+ if err := json.Unmarshal(buf.Bytes(), &har); err != nil {+ t.Fatalf("invalid empty HAR: %v", err)+ }+ if len(har.Log.Entries) != 0 {+ t.Errorf("expected 0 entries, got %d", len(har.Log.Entries))+ }+}++func TestNDJSONEncoder(t *testing.T) {+ factory := GetFormat("ndjson")+ if factory == nil {+ t.Fatal("ndjson format not registered")+ }++ entry := makeTestExportEntry()+ var buf bytes.Buffer+ enc := factory("", "")+ enc.Start(&buf)+ enc.Encode(entry)+ enc.Encode(entry)+ enc.Finish()++ lines := strings.Split(strings.TrimSpace(buf.String()), "\n")+ if len(lines) != 2 {+ t.Errorf("expected 2 lines, got %d", len(lines))+ }+ for i, line := range lines {+ var e ExportEntry+ if err := json.Unmarshal([]byte(line), &e); err != nil {+ t.Errorf("line %d invalid JSON: %v", i, err)+ }+ }+}++func TestListFormats(t *testing.T) {+ formats := ListFormats()+ if len(formats) < 2 {+ t.Errorf("expected at least 2 formats, got %v", formats)+ }+ found := map[string]bool{}+ for _, f := range formats {+ found[f] = true+ }+ if !found["har"] || !found["ndjson"] {+ t.Errorf("missing format: %v", formats)+ }+}++func TestGetFormat_Unknown(t *testing.T) {+ if GetFormat("xml") != nil {+ t.Error("expected nil for unknown format")+ }+}++func makeTestExportEntry() ExportEntry {+ return ExportEntry{+ StartedDateTime: "2026-03-28T12:00:00.000Z",+ Time: 100,+ Request: ExportRequest{+ Method: "GET",+ URL: "https://example.com/test",+ HTTPVersion: "HTTP/1.1",+ Headers: []NameValuePair{{Name: "Accept", Value: "*/*"}},+ QueryString: []NameValuePair{},+ HeadersSize: -1,+ BodySize: 0,+ },+ Response: ExportResponse{+ Status: 200,+ StatusText: "OK",+ HTTPVersion: "HTTP/1.1",+ Headers: []NameValuePair{{Name: "Content-Type", Value: "text/html"}},+ Content: ExportContent{Size: 100, MimeType: "text/html"},+ HeadersSize: -1,+ BodySize: 100,+ },+ Timings: ExportTimings{Send: 1, Wait: 98, Receive: 1},+ }+}