High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix(snapshot): charge maxTokens what the formatter emits instead of a hand-modelled estimate
Luigi Agostidc2802be429a
diff --git a/docs/reference/snapshot.md b/docs/reference/snapshot.mdindex 4e2e4a81..e7efdab2 100644--- a/docs/reference/snapshot.md+++ b/docs/reference/snapshot.md@@ -53,6 +53,25 @@ pinchtab snap --max-tokens 2000 # Limit output size| `maxTokens` | Token budget limit || `depth` | Tree depth limit |+## What `maxTokens` guarantees++`maxTokens` is a ceiling, not a hint. The nodes returned are the longest prefix whose+rendered output fits the budget in the format you asked for, so the response never exceeds+what you asked for and stops one node short of it at worst. Measured across `compact`,+`text`, `json` and `yaml` on a page of realistic interactive nodes, a budget that actually+constrains the result delivers 87–100% of it.++The cost is measured, not modelled: each node is charged the bytes its own format emits —+rendered for `compact` and `text`, marshalled for `json` and `yaml` — so a change to a+formatter changes the budget with it. Tokens are estimated at four bytes each.++Formats are not interchangeable for a given budget. `yaml` is roughly three times the size+of `json` for the same nodes, because the node struct carries JSON field tags and no YAML+ones, so YAML emits every field including the empty ones. The same `maxTokens` therefore+returns far fewer nodes in `yaml` than in `json` — which is the budget working, not a+regression. Prefer `compact` when the budget is tight: it fits several times more nodes+into the same tokens than either structured format.+## Control state on a nodeA snapshot reports the state of a control, not just its identity, so an agent candiff --git a/internal/bridge/observe/snapshot_budget_test.go b/internal/bridge/observe/snapshot_budget_test.gonew file mode 100644index 00000000..8ade5f52--- /dev/null+++ b/internal/bridge/observe/snapshot_budget_test.go@@ -0,0 +1,269 @@+package observe++import (+ "encoding/json"+ "fmt"+ "os"+ "path/filepath"+ "strings"+ "testing"++ "github.com/pinchtab/pinchtab/internal/srccensus"+ "gopkg.in/yaml.v3"+)++// minObserveSourceFiles is the vacuity floor: the package already has more non-test+// sources than this, so a scan that silently stops seeing most of it fails instead of+// passing for the wrong reason.+const minObserveSourceFiles = 8++// budgetProbeNodes is a page's worth of realistic interactive nodes: the mix of roles,+// names, values and states a real form or app shell produces, at varying depth. The budget+// is only meaningful against content like this — a fixture of empty nodes would let almost+// any estimator look correct.+func budgetProbeNodes(count int) []A11yNode {+ roles := []string{"button", "textbox", "link", "checkbox", "combobox", "heading", "listitem", "tab"}+ names := []string{+ "Submit order",+ "Email address",+ "Continue to payment",+ "Remember this device",+ "Country or region",+ "Shipping information",+ "Standard delivery, 3-5 business days",+ "Account",+ }+ values := []string{"", "[email protected]", "", "", "United Kingdom", "", "", ""}++ nodes := make([]A11yNode, 0, count)+ for i := 0; i < count; i++ {+ nodes = append(nodes, A11yNode{+ Ref: fmt.Sprintf("e%d", i+1),+ Role: roles[i%len(roles)],+ Name: names[i%len(names)],+ Value: values[i%len(values)],+ Depth: i % 5,+ Focused: i%9 == 0,+ Disabled: i%11 == 0,+ Checked: []CheckedState{"", CheckedTrue, CheckedFalse, CheckedMixed}[i%4],+ })+ }+ return nodes+}++// renderedTokens is the ground truth the card measured against: the tokens the caller+// actually receives for the nodes the truncator kept, counted from the bytes the matching+// formatter emits.+func renderedTokens(t *testing.T, nodes []A11yNode, format string) int {+ t.Helper()+ switch format {+ case "compact":+ return estimateTokens(len(FormatSnapshotCompact(nodes)))+ case "text":+ return estimateTokens(len(FormatSnapshotText(nodes)))+ case "yaml":+ out, err := yaml.Marshal(nodes)+ if err != nil {+ t.Fatalf("marshal yaml: %v", err)+ }+ return estimateTokens(len(out))+ default:+ out, err := json.Marshal(nodes)+ if err != nil {+ t.Fatalf("marshal json: %v", err)+ }+ return estimateTokens(len(out))+ }+}++// The contract maxTokens now carries, stated as the two properties that make a budget a+// budget rather than a hint:+//+// 1. the output never exceeds it, and+// 2. nothing is left on the table — the first node that was dropped would not have fit.+//+// Property 2 is what a percentage floor cannot express. A shortfall is only legitimate+// when the next node was too big for the remainder, and that is checkable exactly, so+// there is no tolerance here to quietly widen until the current constants pass.+func TestTruncateToTokensDeliversTheBudgetItWasAsked(t *testing.T) {+ nodes := budgetProbeNodes(40)++ for _, format := range []string{"compact", "text", "json", "yaml"} {+ for _, budget := range []int{100, 300, 1000} {+ t.Run(fmt.Sprintf("%s/%d", format, budget), func(t *testing.T) {+ kept, truncated := TruncateToTokens(nodes, budget, format)+ actual := renderedTokens(t, kept, format)++ if actual > budget {+ t.Errorf("%s budget=%d kept %d/%d nodes but renders ~%d tokens (%+d%%): the budget is a ceiling and it was exceeded",+ format, budget, len(kept), len(nodes), actual, percentOff(actual, budget))+ }++ if truncated != (len(kept) < len(nodes)) {+ t.Errorf("%s budget=%d reported truncated=%v while keeping %d of %d nodes",+ format, budget, truncated, len(kept), len(nodes))+ }++ if truncated {+ withNext := renderedTokens(t, nodes[:len(kept)+1], format)+ if withNext <= budget {+ t.Errorf("%s budget=%d stopped at %d/%d nodes (~%d tokens) although the next node would have fit in ~%d: the caller was short-changed",+ format, budget, len(kept), len(nodes), actual, withNext)+ }+ }+ })+ }+ }+}++// The band the two properties above actually produce, recorded so the number in+// docs/reference/snapshot.md is measured rather than asserted. A format whose nodes are+// large relative to the budget lands lower in the band; that is the one-node gap, not+// slack in the estimator.+func TestBudgetDeliveryBandIsWorthDocumenting(t *testing.T) {+ nodes := budgetProbeNodes(40)+ worst := 100+ for _, format := range []string{"compact", "text", "json", "yaml"} {+ for _, budget := range []int{100, 300, 1000} {+ kept, truncated := TruncateToTokens(nodes, budget, format)+ got := renderedTokens(t, kept, format) * 100 / budget+ if !truncated {+ // Everything fit, so the budget was never the constraint and the+ // percentage says nothing about the allocator.+ t.Logf("%-8s budget=%4d -> kept all %d nodes (~%d%% of budget, not constrained)", format, budget, len(kept), got)+ continue+ }+ t.Logf("%-8s budget=%4d -> kept %2d/%d, ~%d%% of budget", format, budget, len(kept), len(nodes), got)+ if got < worst {+ worst = got+ }+ }+ }+ if worst < 80 {+ t.Errorf("worst delivery was %d%% of budget; docs claim at least 80%%", worst)+ }+}++func percentOff(actual, budget int) int {+ if budget == 0 {+ return 0+ }+ return (actual - budget) * 100 / budget+}++// The estimate cannot drift from the output because it IS the output: what the truncator+// charges for a node must be the bytes that node costs once rendered. Change a formatter+// without changing the charge and this reds — which is the link the old estimator did not+// have, and the reason maxTokens had already drifted from all four formats.+func TestWhatTheTruncatorChargesIsWhatTheFormatterEmits(t *testing.T) {+ nodes := budgetProbeNodes(40)++ for _, tc := range []struct {+ format string+ // slack is the array or document framing that belongs to the collection rather+ // than to any node: the bytes no per-node charge can be attributed.+ slack int+ }{+ {format: "compact", slack: 0},+ {format: "text", slack: 0},+ {format: "yaml", slack: 0},+ {format: "json", slack: 1},+ } {+ t.Run(tc.format, func(t *testing.T) {+ cost := nodeCost(tc.format)+ charged := 0+ for _, n := range nodes {+ charged += cost(n)+ }+ emitted := renderedBytes(t, nodes, tc.format)++ if diff := emitted - charged; diff != tc.slack {+ t.Errorf("%s: charged %d bytes for %d nodes but the formatter emits %d (off by %d, want %d): the cost model has drifted from the layout",+ tc.format, charged, len(nodes), emitted, diff, tc.slack)+ }+ })+ }+}++// A node's charge must also track its own shape, not just aggregate correctly: an empty+// name, a value, a focus marker and a checked state each change the bytes emitted, and a+// charge that ignores any of them under-bills that node shape specifically.+func TestEveryRenderedFieldIsChargedForCompactAndText(t *testing.T) {+ base := A11yNode{Ref: "e1", Role: "button"}+ for _, tc := range []struct {+ name string+ node A11yNode+ }{+ {name: "bare", node: base},+ {name: "named", node: A11yNode{Ref: "e1", Role: "button", Name: "Submit order"}},+ {name: "valued", node: A11yNode{Ref: "e1", Role: "textbox", Value: "[email protected]"}},+ {name: "focused", node: A11yNode{Ref: "e1", Role: "button", Focused: true}},+ {name: "disabled", node: A11yNode{Ref: "e1", Role: "button", Disabled: true}},+ {name: "hidden", node: A11yNode{Ref: "e1", Role: "button", Hidden: true}},+ {name: "checked", node: A11yNode{Ref: "e1", Role: "checkbox", Checked: CheckedTrue}},+ {name: "unchecked", node: A11yNode{Ref: "e1", Role: "checkbox", Checked: CheckedFalse}},+ {name: "mixed", node: A11yNode{Ref: "e1", Role: "checkbox", Checked: CheckedMixed}},+ {name: "deep", node: A11yNode{Ref: "e1", Role: "listitem", Depth: 4}},+ {name: "everything", node: A11yNode{Ref: "e12", Role: "checkbox", Name: "Remember me", Value: "on", Depth: 3, Focused: true, Disabled: true, Hidden: true, Checked: CheckedMixed}},+ } {+ t.Run(tc.name, func(t *testing.T) {+ one := []A11yNode{tc.node}+ if got, want := nodeCost("compact")(tc.node), len(FormatSnapshotCompact(one)); got != want {+ t.Errorf("compact charged %d bytes, formatter emits %d", got, want)+ }+ if got, want := nodeCost("text")(tc.node), len(FormatSnapshotText(one)); got != want {+ t.Errorf("text charged %d bytes, formatter emits %d", got, want)+ }+ })+ }+}++func renderedBytes(t *testing.T, nodes []A11yNode, format string) int {+ t.Helper()+ switch format {+ case "compact":+ return len(FormatSnapshotCompact(nodes))+ case "text":+ return len(FormatSnapshotText(nodes))+ case "yaml":+ out, err := yaml.Marshal(nodes)+ if err != nil {+ t.Fatalf("marshal yaml: %v", err)+ }+ return len(out)+ default:+ out, err := json.Marshal(nodes)+ if err != nil {+ t.Fatalf("marshal json: %v", err)+ }+ return len(out)+ }+}++// The behavioural guards above compare a charge to an output, which cannot catch a FIFTH+// renderer that emits the layout inline and is never costed at all. This one can: the+// distinctive layout literals belong to appendNode and nowhere else.+func TestNodeLayoutIsWrittenDownExactlyOnce(t *testing.T) {+ pkg := srccensus.Load(t, ".", minObserveSourceFiles)++ for _, literal := range []string{` val="`, ` [focused]`, ` [disabled]`} {+ t.Run(strings.TrimSpace(literal), func(t *testing.T) {+ var sites []string+ for _, name := range pkg.Files() {+ body, err := os.ReadFile(filepath.Join(pkg.Dir(), name))+ if err != nil {+ t.Fatal(err)+ }+ for i, line := range strings.Split(string(body), "\n") {+ if strings.Contains(line, literal) {+ sites = append(sites, fmt.Sprintf("%s:%d: %s", name, i+1, strings.TrimSpace(line)))+ }+ }+ }+ if len(sites) != 1 {+ t.Errorf("%q appears %d times, want exactly 1 — a node's layout written down twice is what let maxTokens drift from every format; emit it through appendNode, or if the layout genuinely moved, re-point this guard at its new single home rather than deleting it:\n%s",+ literal, len(sites), strings.Join(sites, "\n"))+ }+ })+ }+}diff --git a/internal/bridge/observe/snapshot_format.go b/internal/bridge/observe/snapshot_format.goindex e1a21812..d4f77906 100644--- a/internal/bridge/observe/snapshot_format.go+++ b/internal/bridge/observe/snapshot_format.go@@ -1,6 +1,11 @@package observe-import "strings"+import (+ "encoding/json"+ "strings"++ "gopkg.in/yaml.v3"+)// All THREE states are annotated, not just the checked one: absent means the control// has no checkedness and false means it is off, so rendering only [checked] would make@@ -19,75 +24,92 @@ var checkedCompactAnnotations = map[CheckedState]string{CheckedMixed: " [/]",}-func FormatSnapshotText(nodes []A11yNode) string {- var b strings.Builder- for _, n := range nodes {- for i := 0; i < n.Depth; i++ {- b.WriteString(" ")- }- b.WriteString(n.Ref)- b.WriteByte(' ')- b.WriteString(n.Role)- if n.Name != "" {- b.WriteString(` "`)- b.WriteString(n.Name)- b.WriteByte('"')- }- if n.Value != "" {- b.WriteString(` val="`)- b.WriteString(n.Value)- b.WriteByte('"')- }- if n.Focused {- b.WriteString(" [focused]")- }- if annotation, ok := checkedAnnotations[n.Checked]; ok {- b.WriteString(annotation)- }- if n.Disabled {- b.WriteString(" [disabled]")- }- if n.Hidden {- b.WriteString(" [hidden]")- }- b.WriteByte('\n')+// nodeStyle is everything that differs between the text and compact layouts. The field+// ORDER and the quoting are not in here because they do not differ — they live once, in+// appendNode. A style that could reorder fields would be a second layout model, which is+// the thing this file used to have three of.+type nodeStyle struct {+ indent string+ refSep string+ focused string+ disabled string+ hidden string+ checked map[CheckedState]string+}++var textNodeStyle = nodeStyle{+ indent: " ",+ refSep: " ",+ focused: " [focused]",+ disabled: " [disabled]",+ hidden: " [hidden]",+ checked: checkedAnnotations,+}++var compactNodeStyle = nodeStyle{+ refSep: ":",+ focused: " *",+ disabled: " -",+ hidden: " [hidden]",+ checked: checkedCompactAnnotations,+}++// appendNode writes one node exactly as the caller's format emits it. It is the single+// source of truth for what a node costs, because the truncator charges the caller for+// what this writes rather than for a second hand-maintained model of it.+func appendNode(b *strings.Builder, n A11yNode, style nodeStyle, marker string) {+ for i := 0; i < n.Depth; i++ {+ b.WriteString(style.indent)}- return b.String()+ b.WriteString(n.Ref)+ b.WriteString(style.refSep)+ b.WriteString(n.Role)+ if n.Name != "" {+ b.WriteString(` "`)+ b.WriteString(n.Name)+ b.WriteByte('"')+ }+ if n.Value != "" {+ b.WriteString(` val="`)+ b.WriteString(n.Value)+ b.WriteByte('"')+ }+ if n.Focused {+ b.WriteString(style.focused)+ }+ if annotation, ok := style.checked[n.Checked]; ok {+ b.WriteString(annotation)+ }+ if n.Disabled {+ b.WriteString(style.disabled)+ }+ if n.Hidden {+ b.WriteString(style.hidden)+ }+ b.WriteString(marker)+ b.WriteByte('\n')}-func FormatSnapshotCompact(nodes []A11yNode) string {+func formatNodes(nodes []A11yNode, style nodeStyle, marker func(A11yNode) string) string {var b strings.Builderfor _, n := range nodes {- b.WriteString(n.Ref)- b.WriteByte(':')- b.WriteString(n.Role)- if n.Name != "" {- b.WriteString(` "`)- b.WriteString(n.Name)- b.WriteByte('"')- }- if n.Value != "" {- b.WriteString(` val="`)- b.WriteString(n.Value)- b.WriteByte('"')- }- if n.Focused {- b.WriteString(" *")- }- if annotation, ok := checkedCompactAnnotations[n.Checked]; ok {- b.WriteString(annotation)- }- if n.Disabled {- b.WriteString(" -")- }- if n.Hidden {- b.WriteString(" [hidden]")+ suffix := ""+ if marker != nil {+ suffix = marker(n)}- b.WriteByte('\n')+ appendNode(&b, n, style, suffix)}return b.String()}+func FormatSnapshotText(nodes []A11yNode) string {+ return formatNodes(nodes, textNodeStyle, nil)+}++func FormatSnapshotCompact(nodes []A11yNode) string {+ return formatNodes(nodes, compactNodeStyle, nil)+}+// FormatSnapshotCompactDiff outputs all current nodes in compact format with// change markers: [+] for added, [~] for changed. Removed refs are listed at// the end as [- ref]. This gives agents the full valid ref set plus change info.@@ -102,39 +124,16 @@ func FormatSnapshotCompactDiff(nodes []A11yNode, added, changed, removed []A11yN}var b strings.Builder- for _, n := range nodes {- b.WriteString(n.Ref)- b.WriteByte(':')- b.WriteString(n.Role)- if n.Name != "" {- b.WriteString(` "`)- b.WriteString(n.Name)- b.WriteByte('"')- }- if n.Value != "" {- b.WriteString(` val="`)- b.WriteString(n.Value)- b.WriteByte('"')- }- if n.Focused {- b.WriteString(" *")- }- if annotation, ok := checkedCompactAnnotations[n.Checked]; ok {- b.WriteString(annotation)- }- if n.Disabled {- b.WriteString(" -")- }- if n.Hidden {- b.WriteString(" [hidden]")- }- if addedRefs[n.Ref] {- b.WriteString(" [+]")- } else if changedRefs[n.Ref] {- b.WriteString(" [~]")+ b.WriteString(formatNodes(nodes, compactNodeStyle, func(n A11yNode) string {+ switch {+ case addedRefs[n.Ref]:+ return " [+]"+ case changedRefs[n.Ref]:+ return " [~]"+ default:+ return ""}- b.WriteByte('\n')- }+ }))if len(removed) > 0 {b.WriteString("# removed:")@@ -148,26 +147,63 @@ func FormatSnapshotCompactDiff(nodes []A11yNode, added, changed, removed []A11yNreturn b.String()}+// estimateTokens is the one place bytes become tokens. Four bytes per token is the+// approximation the whole budget rests on, so it lives here rather than being spelled+// out per format — a format that divides by its own constant is a second model of the+// same thing, which is how the old estimator drifted.+func estimateTokens(bytes int) int {+ return bytes / 4+}++// nodeCost reports what one node costs the caller in the format it asked for. Every+// branch MEASURES: the text layouts by rendering through appendNode, the structured ones+// by marshalling the node with the same encoder the handler uses. Nothing here models a+// layout, so nothing here can drift away from one.+func nodeCost(format string) func(A11yNode) int {+ switch format {+ case "compact":+ return renderedNodeCost(compactNodeStyle)+ case "text":+ return renderedNodeCost(textNodeStyle)+ case "yaml":+ return func(n A11yNode) int {+ out, err := yaml.Marshal([]A11yNode{n})+ if err != nil {+ return 0+ }+ return len(out)+ }+ default:+ return func(n A11yNode) int {+ out, err := json.Marshal(n)+ if err != nil {+ return 0+ }+ // +1 for the comma or closing bracket this node brings with it once it is+ // an element of the nodes array rather than a value on its own.+ return len(out) + 1+ }+ }+}++func renderedNodeCost(style nodeStyle) func(A11yNode) int {+ var b strings.Builder+ return func(n A11yNode) int {+ b.Reset()+ appendNode(&b, n, style, "")+ return b.Len()+ }+}++// TruncateToTokens keeps the longest prefix of nodes whose rendered output fits in+// maxTokens. The budget is a ceiling, never a target to overshoot: it stops at the last+// node that fits, so the only shortfall possible is the one node that did not.func TruncateToTokens(nodes []A11yNode, maxTokens int, format string) ([]A11yNode, bool) {- tokensUsed := 0+ cost := nodeCost(format)+ bytesUsed := 0for i, n := range nodes {- var nodeTokens int- switch format {- case "compact":- size := len(n.Ref) + 1 + len(n.Role) + len(n.Name) + len(n.Value) + 8 + len(checkedCompactAnnotations[n.Checked])- nodeTokens = size / 4- case "text":- size := n.Depth*2 + len(n.Ref) + 1 + len(n.Role) + len(n.Name) + len(n.Value) + 8 + len(checkedAnnotations[n.Checked])- nodeTokens = size / 4- default:- size := len(n.Ref) + len(n.Role) + len(n.Name) + len(n.Value) + 60- nodeTokens = size / 3- }- if nodeTokens < 1 {- nodeTokens = 1- }- tokensUsed += nodeTokens- if tokensUsed > maxTokens {+ bytesUsed += cost(n)+ if estimateTokens(bytesUsed) > maxTokens {return nodes[:i], true}}