Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

18.2 KB
Raw Blame
package observe

import (
	"context"
	"encoding/base64"
	"encoding/json"
	"fmt"
	"log/slog"
	"strings"
	"sync"
	"time"
	"unicode/utf8"

	"github.com/chromedp/cdproto/network"
	"github.com/chromedp/chromedp"
	"github.com/pinchtab/pinchtab/internal/config"
	"github.com/pinchtab/pinchtab/internal/sanitize"
)

const defaultRetainBodyMaxBytesPerTab = 4 << 20
const defaultRetainBodyConcurrency = 4

const (
	maxNetworkURLBytes          = 8 * 1024
	maxNetworkMethodBytes       = 32
	maxNetworkResourceTypeBytes = 64
	maxNetworkStatusTextBytes   = 512
	maxNetworkMimeTypeBytes     = 512
	maxNetworkErrorBytes        = 2 * 1024
	maxNetworkPostDataBytes     = 64 * 1024
	maxNetworkHeaderKeyBytes    = 256
	maxNetworkHeaderValueBytes  = 4 * 1024
	maxNetworkHeaderTotalBytes  = 32 * 1024
)

type NetworkMonitor struct {
	mu                  sync.RWMutex
	buffers             map[string]*NetworkBuffer
	listeners           map[string]context.CancelFunc
	bufSize             int
	retainBodies        bool
	retainBodyMaxBytes  int
	retainBodyMaxPerTab int64
	retainBodySemaphore chan struct{}
}

func NewNetworkMonitor(bufferSize int) *NetworkMonitor {
	bufferSize = config.ClampNetworkBufferSize(bufferSize)
	if bufferSize <= 0 {
		bufferSize = DefaultNetworkBufferSize
	}
	return &NetworkMonitor{
		buffers:             make(map[string]*NetworkBuffer),
		listeners:           make(map[string]context.CancelFunc),
		bufSize:             bufferSize,
		retainBodies:        false,
		retainBodyMaxBytes:  0,
		retainBodyMaxPerTab: defaultRetainBodyMaxBytesPerTab,
		retainBodySemaphore: make(chan struct{}, defaultRetainBodyConcurrency),
	}
}

func (nm *NetworkMonitor) ConfigureBodyRetention(enabled bool, maxBytes int) {
	nm.mu.Lock()
	defer nm.mu.Unlock()
	nm.retainBodies = enabled
	if maxBytes < 0 {
		maxBytes = 0
	}
	nm.retainBodyMaxBytes = maxBytes
}

func (nm *NetworkMonitor) getOrCreateBuffer(tabID string) *NetworkBuffer {
	return nm.getOrCreateBufferWithSize(tabID, 0)
}

func (nm *NetworkMonitor) getOrCreateBufferWithSize(tabID string, size int) *NetworkBuffer {
	nm.mu.Lock()
	defer nm.mu.Unlock()
	buf, ok := nm.buffers[tabID]
	if !ok {
		if size <= 0 {
			size = nm.bufSize
		}
		buf = NewNetworkBuffer(size)
		nm.buffers[tabID] = buf
	}
	return buf
}

func (nm *NetworkMonitor) GetOrCreateBufferForTest(tabID string) *NetworkBuffer {
	return nm.getOrCreateBuffer(tabID)
}

func (nm *NetworkMonitor) GetOrCreateBufferWithSizeForTest(tabID string, size int) *NetworkBuffer {
	return nm.getOrCreateBufferWithSize(tabID, size)
}

func (nm *NetworkMonitor) GetBuffer(tabID string) *NetworkBuffer {
	nm.mu.RLock()
	defer nm.mu.RUnlock()
	return nm.buffers[tabID]
}

func (nm *NetworkMonitor) BufferSizeForTest() int {
	nm.mu.RLock()
	defer nm.mu.RUnlock()
	return nm.bufSize
}

func (nm *NetworkMonitor) StartCapture(tabCtx context.Context, tabID string) error {
	return nm.StartCaptureWithSize(tabCtx, tabID, 0)
}

func (nm *NetworkMonitor) StartCaptureWithSize(tabCtx context.Context, tabID string, bufferSize int) error {
	buf := nm.getOrCreateBufferWithSize(tabID, bufferSize)

	listenerCtx, _, alreadyActive := nm.reserveCaptureListener(tabID, tabCtx)
	if alreadyActive {
		// Capture is already running for this tab (the buffer exists above). Do
		// NOT stack another ListenTarget callback — that would double-record events.
		return nil
	}

	if err := chromedp.Run(tabCtx, chromedp.ActionFunc(func(ctx context.Context) error {
		return network.Enable().Do(ctx)
	})); err != nil {
		nm.releaseCaptureListener(tabID)
		return fmt.Errorf("network enable: %w", err)
	}

	chromedp.ListenTarget(listenerCtx, func(ev interface{}) {
		switch e := ev.(type) {
		case *network.EventRequestWillBeSent:
			buf.Add(requestEntryFromEvent(e))
			buf.MarkRequestStart(string(e.RequestID))

		case *network.EventResponseReceived:
			buf.Update(string(e.RequestID), func(entry *NetworkEntry) {
				entry.Status = int(e.Response.Status)
				entry.StatusText = e.Response.StatusText
				entry.MimeType = e.Response.MimeType
				if e.Response.Headers != nil {
					respHeaders := make(map[string]string)
					for k, v := range e.Response.Headers {
						if s, ok := v.(string); ok {
							respHeaders[k] = s
						}
					}
					entry.ResponseHeaders = respHeaders
				}
				if e.Response.EncodedDataLength > 0 {
					entry.Size = int64(e.Response.EncodedDataLength)
				}
			})

		case *network.EventLoadingFinished:
			buf.Update(string(e.RequestID), func(entry *NetworkEntry) {
				entry.Finished = true
				entry.EndTime = time.Now()
				if !entry.StartTime.IsZero() {
					entry.Duration = float64(entry.EndTime.Sub(entry.StartTime).Milliseconds())
				}
				if e.EncodedDataLength > 0 {
					entry.Size = int64(e.EncodedDataLength)
				}
				if nm.bodyRetentionEnabled() {
					entry.BodyPending = true
					entry.BodySkipped = false
					entry.BodySkipReason = ""
					entry.BodyError = ""
				}
			})
			buf.MarkRequestEnd(string(e.RequestID))
			go nm.maybeRetainBody(tabCtx, buf, string(e.RequestID))

		case *network.EventLoadingFailed:
			buf.Update(string(e.RequestID), func(entry *NetworkEntry) {
				entry.Failed = true
				entry.Finished = true
				entry.EndTime = time.Now()
				if !entry.StartTime.IsZero() {
					entry.Duration = float64(entry.EndTime.Sub(entry.StartTime).Milliseconds())
				}
				entry.Error = e.ErrorText
			})
			buf.MarkRequestEnd(string(e.RequestID))
		}
	})

	// Self-heal the listeners map when the listener ends (StopCapture cancel or
	// tab close via tabCtx), so a later capture for a reused tabID re-registers.
	go func() {
		<-listenerCtx.Done()
		nm.mu.Lock()
		delete(nm.listeners, tabID)
		nm.mu.Unlock()
	}()

	slog.Debug("network capture started", "tabId", tabID)
	return nil
}

// reserveCaptureListener reserves the per-tab listener slot. If capture is
// already active for tabID it returns alreadyActive=true (with a nil cancel);
// otherwise it stores a fresh cancel derived from tabCtx and returns it.
func (nm *NetworkMonitor) reserveCaptureListener(tabID string, tabCtx context.Context) (context.Context, context.CancelFunc, bool) {
	nm.mu.Lock()
	defer nm.mu.Unlock()
	if _, exists := nm.listeners[tabID]; exists {
		return nil, nil, true
	}
	listenerCtx, cancel := context.WithCancel(tabCtx)
	nm.listeners[tabID] = cancel
	return listenerCtx, cancel, false
}

// releaseCaptureListener cancels and removes the per-tab capture listener, if any.
func (nm *NetworkMonitor) releaseCaptureListener(tabID string) {
	nm.mu.Lock()
	cancel, ok := nm.listeners[tabID]
	delete(nm.listeners, tabID)
	nm.mu.Unlock()
	if ok {
		cancel()
	}
}

func (nm *NetworkMonitor) StopCapture(tabID string) {
	nm.releaseCaptureListener(tabID)
	nm.mu.Lock()
	delete(nm.buffers, tabID)
	nm.mu.Unlock()
}

func (nm *NetworkMonitor) ClearTab(tabID string) {
	nm.mu.RLock()
	buf := nm.buffers[tabID]
	nm.mu.RUnlock()
	if buf != nil {
		buf.Clear()
	}
}

func (nm *NetworkMonitor) ClearAll() {
	nm.mu.RLock()
	defer nm.mu.RUnlock()
	for _, buf := range nm.buffers {
		buf.Clear()
	}
}

func (nm *NetworkMonitor) bodyRetentionEnabled() bool {
	nm.mu.RLock()
	defer nm.mu.RUnlock()
	return nm.retainBodies
}

// fetchResponseBody reads the body from the browser. Indirected so retention
// policy can be exercised without a live CDP target.
var fetchResponseBody = GetResponseBody

func skipRetainedBody(buf *NetworkBuffer, requestID, reason string) {
	buf.Update(requestID, func(entry *NetworkEntry) {
		entry.BodyPending = false
		entry.BodySkipped = true
		entry.BodySkipReason = reason
		entry.BodyError = ""
	})
}

// Operator-facing names of the two budgets a retained body is cut against, used
// to build the skip reason so a dropped body says which one dropped it.
const (
	retentionLimitScope  = "retention limit"
	retentionBudgetScope = "retention budget"
	postDataLimitScope   = "request body limit"
)

// clampRetainedBody applies a byte budget to a retained payload — a response body,
// or the request body in PostData. One rule: a retained payload is a byte-exact
// prefix of what crossed the wire, or there is no retained payload. Text is cut on a rune boundary with the suffix-free helper — the display
// variant appends "..." inside the budget, and a machine-read body must not carry
// characters the payload never contained when the truncated flag already says it
// was cut. A base64 body cannot be cut at all (the encoding's length and padding make
// a fragment undecodable in whole, not only at the tail), and a budget smaller
// than the body's first character leaves no whole rune to keep; both are dropped
// with a reason rather than returned corrupt or returned empty-but-retained.
func clampRetainedBody(body string, base64Encoded bool, limit int, scope string) (clamped string, truncated bool, dropReason string) {
	if limit <= 0 || len(body) <= limit {
		return body, false, ""
	}
	if base64Encoded {
		return "", false, "base64 body exceeds " + scope
	}
	prefix := sanitize.TruncateUTF8BytesExact(body, limit)
	if prefix == "" {
		return "", false, scope + " is smaller than the body's first character"
	}
	return prefix, true, ""
}

func (nm *NetworkMonitor) maybeRetainBody(tabCtx context.Context, buf *NetworkBuffer, requestID string) {
	// Every return path below resolves BodyPending via buf.Update; signal once on
	// exit so retained-body readers wake instead of polling.
	defer buf.SignalBodyChange()

	nm.mu.RLock()
	enabled := nm.retainBodies
	maxBytes := nm.retainBodyMaxBytes
	nm.mu.RUnlock()
	if !enabled {
		skipRetainedBody(buf, requestID, "retention disabled")
		return
	}
	if buf.RetainedBytes() >= nm.retainBodyMaxPerTab {
		skipRetainedBody(buf, requestID, "retention budget exceeded")
		return
	}
	select {
	case nm.retainBodySemaphore <- struct{}{}:
		defer func() { <-nm.retainBodySemaphore }()
	default:
		skipRetainedBody(buf, requestID, "retention concurrency limit reached")
		return
	}
	body, base64Encoded, err := fetchResponseBody(tabCtx, requestID)
	if err != nil {
		buf.Update(requestID, func(entry *NetworkEntry) {
			entry.BodyPending = false
			entry.BodySkipped = false
			entry.BodySkipReason = ""
			entry.BodyError = err.Error()
		})
		return
	}
	body, truncated, dropReason := clampRetainedBody(body, base64Encoded, maxBytes, retentionLimitScope)
	if dropReason != "" {
		skipRetainedBody(buf, requestID, dropReason)
		return
	}
	remainingBudget := int(nm.retainBodyMaxPerTab - buf.RetainedBytes())
	if remainingBudget <= 0 {
		skipRetainedBody(buf, requestID, "retention budget exceeded")
		return
	}
	body, cutForBudget, dropReason := clampRetainedBody(body, base64Encoded, remainingBudget, retentionBudgetScope)
	if dropReason != "" {
		skipRetainedBody(buf, requestID, dropReason)
		return
	}
	truncated = truncated || cutForBudget
	buf.Update(requestID, func(entry *NetworkEntry) {
		entry.ResponseBody = body
		entry.Base64Encoded = base64Encoded
		entry.BodyRetained = true
		entry.BodyPending = false
		entry.BodySkipped = false
		entry.BodySkipReason = ""
		entry.BodyTruncated = truncated
		entry.BodyError = ""
	})
}

func (nm *NetworkMonitor) IsTabIdle(tabID string) (bool, bool) {
	nm.mu.RLock()
	buf, ok := nm.buffers[tabID]
	nm.mu.RUnlock()
	if !ok || buf == nil {
		return false, false
	}
	count, _ := buf.InflightStatus()
	return count == 0, true
}

// GetResponseBody is the only response-body fetcher. It reads Network.getResponseBody
// as raw JSON so the body and the base64Encoded flag describing it come from the same
// call and travel together.
//
// It deliberately does NOT use cdproto's typed constructor plus Do: that helper
// base64-decodes the payload inside the dependency and returns []byte, so by the time
// it returns the flag is gone and the string holds raw bytes. A second fetcher built
// that way reported base64Encoded=false for every response, which meant a binary body
// was retained as a string of U+FFFD once JSON-encoded — and made clampRetainedBody's
// drop-and-mark branch unreachable in production, since nothing ever set the flag it
// tests.
func GetResponseBody(ctx context.Context, requestID string) (string, bool, error) {
	var body string
	var base64Encoded bool

	err := chromedp.Run(ctx, chromedp.ActionFunc(func(ctx context.Context) error {
		executor := chromedp.FromContext(ctx).Target
		if executor == nil {
			return fmt.Errorf("no CDP executor available")
		}
		var result json.RawMessage
		if err := executor.Execute(ctx, "Network.getResponseBody", map[string]any{
			"requestId": requestID,
		}, &result); err != nil {
			return err
		}
		var resp struct {
			Body          string `json:"body"`
			Base64Encoded bool   `json:"base64Encoded"`
		}
		if err := json.Unmarshal(result, &resp); err != nil {
			return err
		}
		body = resp.Body
		base64Encoded = resp.Base64Encoded
		return nil
	}))

	return body, base64Encoded, err
}

// decodePostData joins a request's body entries into the bytes the page sent. CDP declares
// PostDataEntry.Bytes as a protocol binary field, so each entry arrives base64-encoded, and
// joining the entries as strings is wrong twice over: the published value is an encoded blob
// in a field named after the request body, and base64(a)+base64(b) is not base64(a+b) once an
// entry's length is not a multiple of three, so any multi-entry body arrives corrupt.
//
// Anything that cannot be published as the text it claims to be publishes nothing, and says
// why: an entry that is not base64, or a body that is not valid UTF-8 once decoded, such as a
// file part in a multipart POST. The alternative is mojibake, or a blob in a field with no
// encoding signal, or — worse — an empty string that reads as a request sent with no body.
// The second return is that reason, empty when there is nothing to explain.
func decodePostData(entries []*network.PostDataEntry) (string, string) {
	var decoded []byte
	for _, entry := range entries {
		if entry == nil {
			continue
		}
		chunk, err := base64.StdEncoding.DecodeString(entry.Bytes)
		if err != nil {
			return "", "request body entry is not base64"
		}
		decoded = append(decoded, chunk...)
	}
	if !utf8.Valid(decoded) {
		return "", "request body is not valid UTF-8"
	}
	return string(decoded), ""
}

// requestEntryFromEvent builds the entry a started request is recorded as. It is the one
// place a refused request body turns into a stated reason on the entry, so an absent
// postData is never mistaken for a request sent without one.
func requestEntryFromEvent(e *network.EventRequestWillBeSent) NetworkEntry {
	headers := make(map[string]string)
	if e.Request.Headers != nil {
		for k, v := range e.Request.Headers {
			if s, ok := v.(string); ok {
				headers[k] = s
			}
		}
	}
	postData, postDataSkipReason := decodePostData(e.Request.PostDataEntries)
	return NetworkEntry{
		RequestID:          string(e.RequestID),
		URL:                e.Request.URL,
		Method:             e.Request.Method,
		ResourceType:       e.Type.String(),
		RequestHeaders:     headers,
		PostData:           postData,
		PostDataSkipped:    postDataSkipReason != "",
		PostDataSkipReason: postDataSkipReason,
		StartTime:          time.Now(),
	}
}

func normalizeNetworkEntry(entry NetworkEntry) NetworkEntry {
	entry.URL = sanitize.TruncateUTF8BytesWithEllipsis(entry.URL, maxNetworkURLBytes)
	entry.Method = sanitize.TruncateUTF8BytesWithEllipsis(entry.Method, maxNetworkMethodBytes)
	entry.ResourceType = sanitize.TruncateUTF8BytesWithEllipsis(entry.ResourceType, maxNetworkResourceTypeBytes)
	entry.StatusText = sanitize.TruncateUTF8BytesWithEllipsis(entry.StatusText, maxNetworkStatusTextBytes)
	entry.MimeType = sanitize.TruncateUTF8BytesWithEllipsis(entry.MimeType, maxNetworkMimeTypeBytes)
	entry.Error = sanitize.TruncateUTF8BytesWithEllipsis(entry.Error, maxNetworkErrorBytes)
	// The request body is clamped by the one owner of that rule, so it inherits the
	// prefix-or-nothing policy the response body already states. The budget measures the
	// decoded body, which is what PostData holds, so the constant describes the request
	// content rather than its encoded length. Flags are only ever set here, never cleared:
	// an entry is re-normalised on every update, and a second pass sees a value already
	// within budget.
	clampedPostData, postDataTruncated, postDataDropReason := clampRetainedBody(entry.PostData, false, maxNetworkPostDataBytes, postDataLimitScope)
	entry.PostData = clampedPostData
	if postDataTruncated {
		entry.PostDataTruncated = true
	}
	if postDataDropReason != "" {
		entry.PostDataSkipped = true
		entry.PostDataSkipReason = postDataDropReason
	}
	entry.RequestHeaders = normalizeNetworkHeaders(entry.RequestHeaders)
	entry.ResponseHeaders = normalizeNetworkHeaders(entry.ResponseHeaders)
	return entry
}

func normalizeNetworkHeaders(headers map[string]string) map[string]string {
	if len(headers) == 0 {
		return nil
	}

	remaining := maxNetworkHeaderTotalBytes
	normalized := make(map[string]string, len(headers))
	for key, value := range headers {
		if remaining <= 0 {
			break
		}

		key = sanitize.TruncateUTF8BytesWithEllipsis(key, maxNetworkHeaderKeyBytes)
		if key == "" {
			continue
		}

		valueLimit := maxNetworkHeaderValueBytes
		if max := remaining - len(key); max < valueLimit {
			valueLimit = max
		}
		if valueLimit <= 0 {
			break
		}

		value = sanitize.TruncateUTF8BytesWithEllipsis(value, valueLimit)
		entryBytes := len(key) + len(value)
		if entryBytes <= 0 {
			continue
		}

		normalized[key] = value
		remaining -= entryBytes
	}

	if len(normalized) == 0 {
		return nil
	}
	return normalized
}

// BrokenAsset is a subresource that failed to load: an HTTP error response
// (status >= 400) or a request that failed outright (network error, abort).
type BrokenAsset struct {
	URL          string `json:"url"`
	ResourceType string `json:"resourceType"`
	StatusCode   int    `json:"statusCode"`
	Error        string `json:"error,omitempty"`
}

// IsBrokenAsset reports whether entry represents a failed load: a response
// with status >= 400, or a failed request. In-flight requests are not broken.
func IsBrokenAsset(entry NetworkEntry) bool {
	return entry.Status >= 400 || entry.Failed
}

// BrokenAssets classifies the broken loads in entries. Resource types are
// the CDP categories lowercased (image, script, stylesheet, font, xhr,
// fetch, document, ...).
func BrokenAssets(entries []NetworkEntry) []BrokenAsset {
	broken := []BrokenAsset{}
	for _, entry := range entries {
		if !IsBrokenAsset(entry) {
			continue
		}
		broken = append(broken, BrokenAsset{
			URL:          entry.URL,
			ResourceType: strings.ToLower(entry.ResourceType),
			StatusCode:   entry.Status,
			Error:        entry.Error,
		})
	}
	return broken
}