Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix(snapshot): charge maxTokens what the formatter emits instead of a hand-modelled estimate

Luigi Agostidc2802be429a
diff --git a/docs/reference/snapshot.md b/docs/reference/snapshot.md
index 4e2e4a81..e7efdab2 100644
--- a/docs/reference/snapshot.md
+++ b/docs/reference/snapshot.md
@@ -53,6 +53,25 @@ pinchtab snap --max-tokens 2000 # Limit output size
| `maxTokens` | Token budget limit |
| `depth` | Tree depth limit |
+## What `maxTokens` guarantees
+
+`maxTokens` is a ceiling, not a hint. The nodes returned are the longest prefix whose
+rendered output fits the budget in the format you asked for, so the response never exceeds
+what you asked for and stops one node short of it at worst. Measured across `compact`,
+`text`, `json` and `yaml` on a page of realistic interactive nodes, a budget that actually
+constrains the result delivers 87–100% of it.
+
+The cost is measured, not modelled: each node is charged the bytes its own format emits —
+rendered for `compact` and `text`, marshalled for `json` and `yaml` — so a change to a
+formatter changes the budget with it. Tokens are estimated at four bytes each.
+
+Formats are not interchangeable for a given budget. `yaml` is roughly three times the size
+of `json` for the same nodes, because the node struct carries JSON field tags and no YAML
+ones, so YAML emits every field including the empty ones. The same `maxTokens` therefore
+returns far fewer nodes in `yaml` than in `json` — which is the budget working, not a
+regression. Prefer `compact` when the budget is tight: it fits several times more nodes
+into the same tokens than either structured format.
+
## Control state on a node
A snapshot reports the state of a control, not just its identity, so an agent can
diff --git a/internal/bridge/observe/snapshot_budget_test.go b/internal/bridge/observe/snapshot_budget_test.go
new file mode 100644
index 00000000..8ade5f52
--- /dev/null
+++ b/internal/bridge/observe/snapshot_budget_test.go
@@ -0,0 +1,269 @@
+package observe
+
+import (
+ "encoding/json"
+ "fmt"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/pinchtab/pinchtab/internal/srccensus"
+ "gopkg.in/yaml.v3"
+)
+
+// minObserveSourceFiles is the vacuity floor: the package already has more non-test
+// sources than this, so a scan that silently stops seeing most of it fails instead of
+// passing for the wrong reason.
+const minObserveSourceFiles = 8
+
+// budgetProbeNodes is a page's worth of realistic interactive nodes: the mix of roles,
+// names, values and states a real form or app shell produces, at varying depth. The budget
+// is only meaningful against content like this — a fixture of empty nodes would let almost
+// any estimator look correct.
+func budgetProbeNodes(count int) []A11yNode {
+ roles := []string{"button", "textbox", "link", "checkbox", "combobox", "heading", "listitem", "tab"}
+ names := []string{
+ "Submit order",
+ "Email address",
+ "Continue to payment",
+ "Remember this device",
+ "Country or region",
+ "Shipping information",
+ "Standard delivery, 3-5 business days",
+ "Account",
+ }
+ values := []string{"", "[email protected]", "", "", "United Kingdom", "", "", ""}
+
+ nodes := make([]A11yNode, 0, count)
+ for i := 0; i < count; i++ {
+ nodes = append(nodes, A11yNode{
+ Ref: fmt.Sprintf("e%d", i+1),
+ Role: roles[i%len(roles)],
+ Name: names[i%len(names)],
+ Value: values[i%len(values)],
+ Depth: i % 5,
+ Focused: i%9 == 0,
+ Disabled: i%11 == 0,
+ Checked: []CheckedState{"", CheckedTrue, CheckedFalse, CheckedMixed}[i%4],
+ })
+ }
+ return nodes
+}
+
+// renderedTokens is the ground truth the card measured against: the tokens the caller
+// actually receives for the nodes the truncator kept, counted from the bytes the matching
+// formatter emits.
+func renderedTokens(t *testing.T, nodes []A11yNode, format string) int {
+ t.Helper()
+ switch format {
+ case "compact":
+ return estimateTokens(len(FormatSnapshotCompact(nodes)))
+ case "text":
+ return estimateTokens(len(FormatSnapshotText(nodes)))
+ case "yaml":
+ out, err := yaml.Marshal(nodes)
+ if err != nil {
+ t.Fatalf("marshal yaml: %v", err)
+ }
+ return estimateTokens(len(out))
+ default:
+ out, err := json.Marshal(nodes)
+ if err != nil {
+ t.Fatalf("marshal json: %v", err)
+ }
+ return estimateTokens(len(out))
+ }
+}
+
+// The contract maxTokens now carries, stated as the two properties that make a budget a
+// budget rather than a hint:
+//
+// 1. the output never exceeds it, and
+// 2. nothing is left on the table — the first node that was dropped would not have fit.
+//
+// Property 2 is what a percentage floor cannot express. A shortfall is only legitimate
+// when the next node was too big for the remainder, and that is checkable exactly, so
+// there is no tolerance here to quietly widen until the current constants pass.
+func TestTruncateToTokensDeliversTheBudgetItWasAsked(t *testing.T) {
+ nodes := budgetProbeNodes(40)
+
+ for _, format := range []string{"compact", "text", "json", "yaml"} {
+ for _, budget := range []int{100, 300, 1000} {
+ t.Run(fmt.Sprintf("%s/%d", format, budget), func(t *testing.T) {
+ kept, truncated := TruncateToTokens(nodes, budget, format)
+ actual := renderedTokens(t, kept, format)
+
+ if actual > budget {
+ t.Errorf("%s budget=%d kept %d/%d nodes but renders ~%d tokens (%+d%%): the budget is a ceiling and it was exceeded",
+ format, budget, len(kept), len(nodes), actual, percentOff(actual, budget))
+ }
+
+ if truncated != (len(kept) < len(nodes)) {
+ t.Errorf("%s budget=%d reported truncated=%v while keeping %d of %d nodes",
+ format, budget, truncated, len(kept), len(nodes))
+ }
+
+ if truncated {
+ withNext := renderedTokens(t, nodes[:len(kept)+1], format)
+ if withNext <= budget {
+ t.Errorf("%s budget=%d stopped at %d/%d nodes (~%d tokens) although the next node would have fit in ~%d: the caller was short-changed",
+ format, budget, len(kept), len(nodes), actual, withNext)
+ }
+ }
+ })
+ }
+ }
+}
+
+// The band the two properties above actually produce, recorded so the number in
+// docs/reference/snapshot.md is measured rather than asserted. A format whose nodes are
+// large relative to the budget lands lower in the band; that is the one-node gap, not
+// slack in the estimator.
+func TestBudgetDeliveryBandIsWorthDocumenting(t *testing.T) {
+ nodes := budgetProbeNodes(40)
+ worst := 100
+ for _, format := range []string{"compact", "text", "json", "yaml"} {
+ for _, budget := range []int{100, 300, 1000} {
+ kept, truncated := TruncateToTokens(nodes, budget, format)
+ got := renderedTokens(t, kept, format) * 100 / budget
+ if !truncated {
+ // Everything fit, so the budget was never the constraint and the
+ // percentage says nothing about the allocator.
+ t.Logf("%-8s budget=%4d -> kept all %d nodes (~%d%% of budget, not constrained)", format, budget, len(kept), got)
+ continue
+ }
+ t.Logf("%-8s budget=%4d -> kept %2d/%d, ~%d%% of budget", format, budget, len(kept), len(nodes), got)
+ if got < worst {
+ worst = got
+ }
+ }
+ }
+ if worst < 80 {
+ t.Errorf("worst delivery was %d%% of budget; docs claim at least 80%%", worst)
+ }
+}
+
+func percentOff(actual, budget int) int {
+ if budget == 0 {
+ return 0
+ }
+ return (actual - budget) * 100 / budget
+}
+
+// The estimate cannot drift from the output because it IS the output: what the truncator
+// charges for a node must be the bytes that node costs once rendered. Change a formatter
+// without changing the charge and this reds — which is the link the old estimator did not
+// have, and the reason maxTokens had already drifted from all four formats.
+func TestWhatTheTruncatorChargesIsWhatTheFormatterEmits(t *testing.T) {
+ nodes := budgetProbeNodes(40)
+
+ for _, tc := range []struct {
+ format string
+ // slack is the array or document framing that belongs to the collection rather
+ // than to any node: the bytes no per-node charge can be attributed.
+ slack int
+ }{
+ {format: "compact", slack: 0},
+ {format: "text", slack: 0},
+ {format: "yaml", slack: 0},
+ {format: "json", slack: 1},
+ } {
+ t.Run(tc.format, func(t *testing.T) {
+ cost := nodeCost(tc.format)
+ charged := 0
+ for _, n := range nodes {
+ charged += cost(n)
+ }
+ emitted := renderedBytes(t, nodes, tc.format)
+
+ if diff := emitted - charged; diff != tc.slack {
+ t.Errorf("%s: charged %d bytes for %d nodes but the formatter emits %d (off by %d, want %d): the cost model has drifted from the layout",
+ tc.format, charged, len(nodes), emitted, diff, tc.slack)
+ }
+ })
+ }
+}
+
+// A node's charge must also track its own shape, not just aggregate correctly: an empty
+// name, a value, a focus marker and a checked state each change the bytes emitted, and a
+// charge that ignores any of them under-bills that node shape specifically.
+func TestEveryRenderedFieldIsChargedForCompactAndText(t *testing.T) {
+ base := A11yNode{Ref: "e1", Role: "button"}
+ for _, tc := range []struct {
+ name string
+ node A11yNode
+ }{
+ {name: "bare", node: base},
+ {name: "named", node: A11yNode{Ref: "e1", Role: "button", Name: "Submit order"}},
+ {name: "valued", node: A11yNode{Ref: "e1", Role: "textbox", Value: "[email protected]"}},
+ {name: "focused", node: A11yNode{Ref: "e1", Role: "button", Focused: true}},
+ {name: "disabled", node: A11yNode{Ref: "e1", Role: "button", Disabled: true}},
+ {name: "hidden", node: A11yNode{Ref: "e1", Role: "button", Hidden: true}},
+ {name: "checked", node: A11yNode{Ref: "e1", Role: "checkbox", Checked: CheckedTrue}},
+ {name: "unchecked", node: A11yNode{Ref: "e1", Role: "checkbox", Checked: CheckedFalse}},
+ {name: "mixed", node: A11yNode{Ref: "e1", Role: "checkbox", Checked: CheckedMixed}},
+ {name: "deep", node: A11yNode{Ref: "e1", Role: "listitem", Depth: 4}},
+ {name: "everything", node: A11yNode{Ref: "e12", Role: "checkbox", Name: "Remember me", Value: "on", Depth: 3, Focused: true, Disabled: true, Hidden: true, Checked: CheckedMixed}},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ one := []A11yNode{tc.node}
+ if got, want := nodeCost("compact")(tc.node), len(FormatSnapshotCompact(one)); got != want {
+ t.Errorf("compact charged %d bytes, formatter emits %d", got, want)
+ }
+ if got, want := nodeCost("text")(tc.node), len(FormatSnapshotText(one)); got != want {
+ t.Errorf("text charged %d bytes, formatter emits %d", got, want)
+ }
+ })
+ }
+}
+
+func renderedBytes(t *testing.T, nodes []A11yNode, format string) int {
+ t.Helper()
+ switch format {
+ case "compact":
+ return len(FormatSnapshotCompact(nodes))
+ case "text":
+ return len(FormatSnapshotText(nodes))
+ case "yaml":
+ out, err := yaml.Marshal(nodes)
+ if err != nil {
+ t.Fatalf("marshal yaml: %v", err)
+ }
+ return len(out)
+ default:
+ out, err := json.Marshal(nodes)
+ if err != nil {
+ t.Fatalf("marshal json: %v", err)
+ }
+ return len(out)
+ }
+}
+
+// The behavioural guards above compare a charge to an output, which cannot catch a FIFTH
+// renderer that emits the layout inline and is never costed at all. This one can: the
+// distinctive layout literals belong to appendNode and nowhere else.
+func TestNodeLayoutIsWrittenDownExactlyOnce(t *testing.T) {
+ pkg := srccensus.Load(t, ".", minObserveSourceFiles)
+
+ for _, literal := range []string{` val="`, ` [focused]`, ` [disabled]`} {
+ t.Run(strings.TrimSpace(literal), func(t *testing.T) {
+ var sites []string
+ for _, name := range pkg.Files() {
+ body, err := os.ReadFile(filepath.Join(pkg.Dir(), name))
+ if err != nil {
+ t.Fatal(err)
+ }
+ for i, line := range strings.Split(string(body), "\n") {
+ if strings.Contains(line, literal) {
+ sites = append(sites, fmt.Sprintf("%s:%d: %s", name, i+1, strings.TrimSpace(line)))
+ }
+ }
+ }
+ if len(sites) != 1 {
+ t.Errorf("%q appears %d times, want exactly 1 — a node's layout written down twice is what let maxTokens drift from every format; emit it through appendNode, or if the layout genuinely moved, re-point this guard at its new single home rather than deleting it:\n%s",
+ literal, len(sites), strings.Join(sites, "\n"))
+ }
+ })
+ }
+}
diff --git a/internal/bridge/observe/snapshot_format.go b/internal/bridge/observe/snapshot_format.go
index e1a21812..d4f77906 100644
--- a/internal/bridge/observe/snapshot_format.go
+++ b/internal/bridge/observe/snapshot_format.go
@@ -1,6 +1,11 @@
package observe
-import "strings"
+import (
+ "encoding/json"
+ "strings"
+
+ "gopkg.in/yaml.v3"
+)
// All THREE states are annotated, not just the checked one: absent means the control
// has no checkedness and false means it is off, so rendering only [checked] would make
@@ -19,75 +24,92 @@ var checkedCompactAnnotations = map[CheckedState]string{
CheckedMixed: " [/]",
}
-func FormatSnapshotText(nodes []A11yNode) string {
- var b strings.Builder
- for _, n := range nodes {
- for i := 0; i < n.Depth; i++ {
- b.WriteString(" ")
- }
- b.WriteString(n.Ref)
- b.WriteByte(' ')
- b.WriteString(n.Role)
- if n.Name != "" {
- b.WriteString(` "`)
- b.WriteString(n.Name)
- b.WriteByte('"')
- }
- if n.Value != "" {
- b.WriteString(` val="`)
- b.WriteString(n.Value)
- b.WriteByte('"')
- }
- if n.Focused {
- b.WriteString(" [focused]")
- }
- if annotation, ok := checkedAnnotations[n.Checked]; ok {
- b.WriteString(annotation)
- }
- if n.Disabled {
- b.WriteString(" [disabled]")
- }
- if n.Hidden {
- b.WriteString(" [hidden]")
- }
- b.WriteByte('\n')
+// nodeStyle is everything that differs between the text and compact layouts. The field
+// ORDER and the quoting are not in here because they do not differ — they live once, in
+// appendNode. A style that could reorder fields would be a second layout model, which is
+// the thing this file used to have three of.
+type nodeStyle struct {
+ indent string
+ refSep string
+ focused string
+ disabled string
+ hidden string
+ checked map[CheckedState]string
+}
+
+var textNodeStyle = nodeStyle{
+ indent: " ",
+ refSep: " ",
+ focused: " [focused]",
+ disabled: " [disabled]",
+ hidden: " [hidden]",
+ checked: checkedAnnotations,
+}
+
+var compactNodeStyle = nodeStyle{
+ refSep: ":",
+ focused: " *",
+ disabled: " -",
+ hidden: " [hidden]",
+ checked: checkedCompactAnnotations,
+}
+
+// appendNode writes one node exactly as the caller's format emits it. It is the single
+// source of truth for what a node costs, because the truncator charges the caller for
+// what this writes rather than for a second hand-maintained model of it.
+func appendNode(b *strings.Builder, n A11yNode, style nodeStyle, marker string) {
+ for i := 0; i < n.Depth; i++ {
+ b.WriteString(style.indent)
}
- return b.String()
+ b.WriteString(n.Ref)
+ b.WriteString(style.refSep)
+ b.WriteString(n.Role)
+ if n.Name != "" {
+ b.WriteString(` "`)
+ b.WriteString(n.Name)
+ b.WriteByte('"')
+ }
+ if n.Value != "" {
+ b.WriteString(` val="`)
+ b.WriteString(n.Value)
+ b.WriteByte('"')
+ }
+ if n.Focused {
+ b.WriteString(style.focused)
+ }
+ if annotation, ok := style.checked[n.Checked]; ok {
+ b.WriteString(annotation)
+ }
+ if n.Disabled {
+ b.WriteString(style.disabled)
+ }
+ if n.Hidden {
+ b.WriteString(style.hidden)
+ }
+ b.WriteString(marker)
+ b.WriteByte('\n')
}
-func FormatSnapshotCompact(nodes []A11yNode) string {
+func formatNodes(nodes []A11yNode, style nodeStyle, marker func(A11yNode) string) string {
var b strings.Builder
for _, n := range nodes {
- b.WriteString(n.Ref)
- b.WriteByte(':')
- b.WriteString(n.Role)
- if n.Name != "" {
- b.WriteString(` "`)
- b.WriteString(n.Name)
- b.WriteByte('"')
- }
- if n.Value != "" {
- b.WriteString(` val="`)
- b.WriteString(n.Value)
- b.WriteByte('"')
- }
- if n.Focused {
- b.WriteString(" *")
- }
- if annotation, ok := checkedCompactAnnotations[n.Checked]; ok {
- b.WriteString(annotation)
- }
- if n.Disabled {
- b.WriteString(" -")
- }
- if n.Hidden {
- b.WriteString(" [hidden]")
+ suffix := ""
+ if marker != nil {
+ suffix = marker(n)
}
- b.WriteByte('\n')
+ appendNode(&b, n, style, suffix)
}
return b.String()
}
+func FormatSnapshotText(nodes []A11yNode) string {
+ return formatNodes(nodes, textNodeStyle, nil)
+}
+
+func FormatSnapshotCompact(nodes []A11yNode) string {
+ return formatNodes(nodes, compactNodeStyle, nil)
+}
+
// FormatSnapshotCompactDiff outputs all current nodes in compact format with
// change markers: [+] for added, [~] for changed. Removed refs are listed at
// the end as [- ref]. This gives agents the full valid ref set plus change info.
@@ -102,39 +124,16 @@ func FormatSnapshotCompactDiff(nodes []A11yNode, added, changed, removed []A11yN
}
var b strings.Builder
- for _, n := range nodes {
- b.WriteString(n.Ref)
- b.WriteByte(':')
- b.WriteString(n.Role)
- if n.Name != "" {
- b.WriteString(` "`)
- b.WriteString(n.Name)
- b.WriteByte('"')
- }
- if n.Value != "" {
- b.WriteString(` val="`)
- b.WriteString(n.Value)
- b.WriteByte('"')
- }
- if n.Focused {
- b.WriteString(" *")
- }
- if annotation, ok := checkedCompactAnnotations[n.Checked]; ok {
- b.WriteString(annotation)
- }
- if n.Disabled {
- b.WriteString(" -")
- }
- if n.Hidden {
- b.WriteString(" [hidden]")
- }
- if addedRefs[n.Ref] {
- b.WriteString(" [+]")
- } else if changedRefs[n.Ref] {
- b.WriteString(" [~]")
+ b.WriteString(formatNodes(nodes, compactNodeStyle, func(n A11yNode) string {
+ switch {
+ case addedRefs[n.Ref]:
+ return " [+]"
+ case changedRefs[n.Ref]:
+ return " [~]"
+ default:
+ return ""
}
- b.WriteByte('\n')
- }
+ }))
if len(removed) > 0 {
b.WriteString("# removed:")
@@ -148,26 +147,63 @@ func FormatSnapshotCompactDiff(nodes []A11yNode, added, changed, removed []A11yN
return b.String()
}
+// estimateTokens is the one place bytes become tokens. Four bytes per token is the
+// approximation the whole budget rests on, so it lives here rather than being spelled
+// out per format — a format that divides by its own constant is a second model of the
+// same thing, which is how the old estimator drifted.
+func estimateTokens(bytes int) int {
+ return bytes / 4
+}
+
+// nodeCost reports what one node costs the caller in the format it asked for. Every
+// branch MEASURES: the text layouts by rendering through appendNode, the structured ones
+// by marshalling the node with the same encoder the handler uses. Nothing here models a
+// layout, so nothing here can drift away from one.
+func nodeCost(format string) func(A11yNode) int {
+ switch format {
+ case "compact":
+ return renderedNodeCost(compactNodeStyle)
+ case "text":
+ return renderedNodeCost(textNodeStyle)
+ case "yaml":
+ return func(n A11yNode) int {
+ out, err := yaml.Marshal([]A11yNode{n})
+ if err != nil {
+ return 0
+ }
+ return len(out)
+ }
+ default:
+ return func(n A11yNode) int {
+ out, err := json.Marshal(n)
+ if err != nil {
+ return 0
+ }
+ // +1 for the comma or closing bracket this node brings with it once it is
+ // an element of the nodes array rather than a value on its own.
+ return len(out) + 1
+ }
+ }
+}
+
+func renderedNodeCost(style nodeStyle) func(A11yNode) int {
+ var b strings.Builder
+ return func(n A11yNode) int {
+ b.Reset()
+ appendNode(&b, n, style, "")
+ return b.Len()
+ }
+}
+
+// TruncateToTokens keeps the longest prefix of nodes whose rendered output fits in
+// maxTokens. The budget is a ceiling, never a target to overshoot: it stops at the last
+// node that fits, so the only shortfall possible is the one node that did not.
func TruncateToTokens(nodes []A11yNode, maxTokens int, format string) ([]A11yNode, bool) {
- tokensUsed := 0
+ cost := nodeCost(format)
+ bytesUsed := 0
for i, n := range nodes {
- var nodeTokens int
- switch format {
- case "compact":
- size := len(n.Ref) + 1 + len(n.Role) + len(n.Name) + len(n.Value) + 8 + len(checkedCompactAnnotations[n.Checked])
- nodeTokens = size / 4
- case "text":
- size := n.Depth*2 + len(n.Ref) + 1 + len(n.Role) + len(n.Name) + len(n.Value) + 8 + len(checkedAnnotations[n.Checked])
- nodeTokens = size / 4
- default:
- size := len(n.Ref) + len(n.Role) + len(n.Name) + len(n.Value) + 60
- nodeTokens = size / 3
- }
- if nodeTokens < 1 {
- nodeTokens = 1
- }
- tokensUsed += nodeTokens
- if tokensUsed > maxTokens {
+ bytesUsed += cost(n)
+ if estimateTokens(bytesUsed) > maxTokens {
return nodes[:i], true
}
}