High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
package handlers
import (
"context"
"encoding/json"
"errors"
"fmt"
"net/http"
"time"
"github.com/pinchtab/pinchtab/internal/activity"
"github.com/pinchtab/pinchtab/internal/bridge"
"github.com/pinchtab/pinchtab/internal/extract"
"github.com/pinchtab/pinchtab/internal/httpx"
)
type extractRequest struct {
TabID string `json:"tabId,omitempty"`
Schema json.RawMessage `json:"schema"`
Scope string `json:"scope,omitempty"`
Threshold float64 `json:"threshold,omitempty"`
MaxItems int `json:"maxItems,omitempty"`
}
type extractResponse struct {
Data map[string]any `json:"data"`
Fields map[string]extract.FieldResult `json:"fields"`
Missing []string `json:"missing"`
Truncated bool `json:"truncated"`
LatencyMs int64 `json:"latency_ms"`
ElementCount int `json:"element_count"`
VocabularyToken string `json:"vocabularyToken,omitempty"`
IDPIWarning string `json:"idpiWarning,omitempty"`
}
// @Endpoint POST /extract
// @Description Extract schema-typed data from the page
//
// @Param schema object body JSON schema (object with properties; array-of-object properties allowed) (required)
// @Param tabId string body Tab ID (optional, defaults to active tab)
// @Param scope string body Confine every field to the subtree of one element: a ref, role:, text: or plain query; CSS and XPath are refused (optional)
// @Param threshold float body Minimum match score per field (optional, default: 0.3)
// @Param maxItems int body Cap on array items (optional, default: 100)
//
// @Response 200 application/json Typed data plus per-field ref, score and confidence, the vocabularyToken for follow-up ref actions (also in X-PinchTab-Vocab), and X-PinchTab-Tab-Id naming the resolved tab
// @Response 400 application/json Missing or unsupported schema, naming the offending path
// @Response 403 application/json IDPI strict mode blocked injected content on the page or in the extracted values
// @Response 404 application/json Tab not found
// @Response 409 application/json A JavaScript dialog is blocking the tab
// @Response 500 application/json Snapshot error
func (h *Handlers) HandleExtract(w http.ResponseWriter, r *http.Request) {
if err := h.ensureBrowser(h.Config); err != nil {
if h.writeBridgeUnavailable(w, err) {
return
}
httpx.Error(w, 500, fmt.Errorf("browser initialization: %w", err))
return
}
req, schema, err := parseExtractRequest(w, r)
if err != nil {
httpx.Error(w, 400, err)
return
}
ctxTab, resolvedTabID, ok := h.guardedTabContextWithHeader(w, r, req.TabID, guardDialogBlocked|guardDomainPolicy)
if !ok {
return
}
tCtx, cancel := context.WithTimeout(ctxTab, h.Config.ActionTimeout)
defer cancel()
go httpx.CancelOnClientDone(r.Context(), cancel)
nodes, vocab, serr := h.acquireExtractNodes(w, tCtx, resolvedTabID)
if serr != nil {
httpx.Error(w, serr.status, serr.err)
return
}
start := time.Now()
result := extract.ResolveContext(tCtx, schema, nodes, extract.Options{Threshold: req.Threshold, MaxItems: req.MaxItems, Matcher: h.Matcher})
idpiWarning, blocked := h.scanFindCorpusForIDPI(w, tCtx, nodes, extractedStrings(result.Data)...)
if blocked {
return
}
h.recordActivity(r, activity.Update{Action: "extract"})
resp := buildExtractResponse(result, len(nodes), idpiWarning, start)
resp.VocabularyToken = vocab
httpx.JSON(w, 200, resp)
}
func (h *Handlers) acquireExtractNodes(w http.ResponseWriter, ctx context.Context, tabID string) ([]bridge.A11yNode, string, *statusError) {
result, err := h.Bridge.Snapshot(ctx, tabID, "", bridge.ContentParams{})
if err != nil {
return nil, "", &statusError{500, fmt.Errorf("snapshot: %w", err)}
}
if len(result.Nodes) == 0 {
return nil, "", &statusError{500, fmt.Errorf("no elements found in snapshot for tab %s — navigate first", tabID)}
}
cache := bridge.EpochRefs(h.Bridge.GetRefCache(tabID), result.Nodes)
h.Bridge.SetRefCache(tabID, cache)
publishVocab(w, tabID, cache.DomEpoch)
return result.Nodes, cache.DomEpoch, nil
}
func extractedStrings(v any) []string {
switch t := v.(type) {
case string:
return []string{t}
case map[string]any:
var out []string
for _, e := range t {
out = append(out, extractedStrings(e)...)
}
return out
case []map[string]any:
var out []string
for _, e := range t {
out = append(out, extractedStrings(e)...)
}
return out
case []any:
var out []string
for _, e := range t {
out = append(out, extractedStrings(e)...)
}
return out
}
return nil
}
func (h *Handlers) HandleTabExtract(w http.ResponseWriter, r *http.Request) {
h.withPathTabIDBody(w, r, h.HandleExtract)
}
func parseExtractRequest(w http.ResponseWriter, r *http.Request) (extractRequest, extract.Schema, error) {
var req extractRequest
if err := json.NewDecoder(http.MaxBytesReader(w, r.Body, maxBodySize)).Decode(&req); err != nil {
return extractRequest{}, extract.Schema{}, fmt.Errorf("decode: %w", err)
}
if len(req.Schema) == 0 {
return extractRequest{}, extract.Schema{}, fmt.Errorf("missing required field 'schema'")
}
var quoted string
if json.Unmarshal(req.Schema, "ed) == nil {
req.Schema = json.RawMessage(quoted)
}
schema, err := extract.ParseSchema(req.Schema)
if err != nil {
var unsupported *extract.UnsupportedError
if errors.As(err, &unsupported) {
return extractRequest{}, extract.Schema{}, fmt.Errorf("schema %s", err)
}
return extractRequest{}, extract.Schema{}, fmt.Errorf("schema: %w", err)
}
if schema, err = schema.WithScope(req.Scope); err != nil {
return extractRequest{}, extract.Schema{}, err
}
return req, schema, nil
}
func buildExtractResponse(result extract.Result, elementCount int, idpiWarning string, start time.Time) extractResponse {
resp := extractResponse{
Data: result.Data,
Fields: result.Fields,
Missing: result.Missing,
LatencyMs: time.Since(start).Milliseconds(),
ElementCount: elementCount,
IDPIWarning: idpiWarning,
}
if resp.Missing == nil {
resp.Missing = []string{}
}
for _, fr := range result.Fields {
if fr.Truncated {
resp.Truncated = true
}
}
return resp
}