Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

chore: auto-wait for readyState before extraction, harden opt runner

Mario4cc5c1334bbf
diff --git a/internal/handlers/text.go b/internal/handlers/text.go
index d940183a..73684d22 100644
--- a/internal/handlers/text.go
+++ b/internal/handlers/text.go
@@ -7,6 +7,7 @@ import (
"net/http"
"strconv"
"strings"
+ "time"
"github.com/chromedp/chromedp"
"github.com/pinchtab/pinchtab/internal/assets"
@@ -85,6 +86,11 @@ func (h *Handlers) HandleText(w http.ResponseWriter, r *http.Request) {
}
}
+ // Auto-wait: if the document is still loading, wait for readyState to
+ // reach at least "interactive" before extracting text. Prevents empty or
+ // partial results when text is called before the page finishes loading.
+ waitForReadyState(tCtx)
+
// Handle element selector - extract text from specific element instead of full page
selectorParam := r.URL.Query().Get("selector")
refParam := r.URL.Query().Get("ref")
@@ -356,3 +362,29 @@ func (h *Handlers) extractElementText(ctx context.Context, tabID, selector, ref
}
return text, nil
}
+
+func waitForReadyState(ctx context.Context) {
+ var state string
+ if err := chromedp.Run(ctx, chromedp.Evaluate(`document.readyState`, &state)); err != nil {
+ return
+ }
+ if state != "loading" {
+ return
+ }
+ deadline := time.After(5 * time.Second)
+ for {
+ select {
+ case <-ctx.Done():
+ return
+ case <-deadline:
+ return
+ case <-time.After(100 * time.Millisecond):
+ if err := chromedp.Run(ctx, chromedp.Evaluate(`document.readyState`, &state)); err != nil {
+ return
+ }
+ if state != "loading" {
+ return
+ }
+ }
+ }
+}
diff --git a/skills/pinchtab-coldstart/SKILL.md b/skills/pinchtab-coldstart/SKILL.md
index c6778c6b..637c15f8 100644
--- a/skills/pinchtab-coldstart/SKILL.md
+++ b/skills/pinchtab-coldstart/SKILL.md
@@ -13,6 +13,9 @@ Before spawning the subagent, clean the environment:
```bash
PROJECT_ROOT=$(git rev-parse --show-toplevel)
+# Stop Docker benchmark services if running — they bind port 9867 and
+# the cold-start native server will collide with them.
+docker compose -f "$PROJECT_ROOT/tests/tools/docker-compose.yml" down 2>/dev/null
pkill -f 'pinchtab' 2>/dev/null
pkill -f 'Google Chrome.*pinchtab' 2>/dev/null
sleep 2
diff --git a/skills/pinchtab-opt/SKILL.md b/skills/pinchtab-opt/SKILL.md
index cdfe64d0..5322b192 100644
--- a/skills/pinchtab-opt/SKILL.md
+++ b/skills/pinchtab-opt/SKILL.md
@@ -20,7 +20,17 @@ The subagents must run with `$TOOLS_DIR` as their working directory because `./s
## Prerequisites
-Docker services must be running. Verify before spawning agents:
+Stop any native PinchTab server that might occupy port 9867, then ensure Docker services are running:
+
+```bash
+# Kill native PinchTab server if running — it binds the same port as Docker services.
+pkill -f 'pinchtab server' 2>/dev/null
+pkill -f 'pinchtab.*serve' 2>/dev/null
+lsof -ti:9867 2>/dev/null | xargs kill 2>/dev/null
+sleep 1
+```
+
+Verify Docker health:
```bash
$TOOLS_DIR/scripts/pt health
diff --git a/skills/pinchtab/SKILL.md b/skills/pinchtab/SKILL.md
index e49e93c2..69e4b8f1 100644
--- a/skills/pinchtab/SKILL.md
+++ b/skills/pinchtab/SKILL.md
@@ -224,6 +224,11 @@ pinchtab upload /absolute/path -s <css> # requires security.allowUpl
- `eval`: narrow read-only DOM inspection unless user asks for mutation. Blocked by default (`security.allowEvaluate: false`).
- `download`: prefer temp/workspace path over arbitrary filesystem. Blocked by default.
- `upload`: path must be user-provided or clearly approved. Blocked by default.
+ The file must exist inside the Docker container. Create it first, then upload:
+ ```bash
+ echo "file content" | docker exec -i tools-pinchtab-1 sh -c 'cat > /tmp/upload.txt'
+ pinchtab upload /tmp/upload.txt -s "#file-input"
+ ```
### HTTP API fallback
diff --git a/tests/optimization/group-03.md b/tests/optimization/group-03.md
index ff5b6ab2..dd68f765 100644
--- a/tests/optimization/group-03.md
+++ b/tests/optimization/group-03.md
@@ -14,8 +14,12 @@ Go to `http://fixtures/form.html` and fill out the entire form with the followin
**Verify**: The form was submitted and a confirmation appeared.
### 3.2 Reset and refill
-After submitting, check whether there is a way to reset the form or go back.
+After submitting, check whether a reset button is visible on the page. Pick one:
-**Verify**: A reset or back control is visible on the page.
+- A) `RESET_BUTTON_VISIBLE` — a reset button is present
+- B) `RESET_BUTTON_HIDDEN` — no reset button is visible
+- C) `RESET_BUTTON_ABSENT` — no reset button exists in the DOM
+
+**Verify**: Answer is one of the options above.
---
diff --git a/tests/optimization/group-15.md b/tests/optimization/group-15.md
index 897deb4a..c244c888 100644
--- a/tests/optimization/group-15.md
+++ b/tests/optimization/group-15.md
@@ -21,8 +21,13 @@ Navigate to `http://fixtures/article.html` and extract the HTML content of the m
**Verify**: The extracted HTML captures the full article content from the page.
### 15.4 Inspect a stable computed CSS property
-Navigate to `http://fixtures/pricing.html` and find the Pro plan card. Report its computed `display` CSS value.
+Navigate to `http://fixtures/pricing.html` and find the Pro plan card element itself (not its parent container). Read its computed `display` CSS value and pick one:
-**Verify**: The computed display value is reported correctly.
+- A) `COMPUTED_DISPLAY_BLOCK`
+- B) `COMPUTED_DISPLAY_FLEX`
+- C) `COMPUTED_DISPLAY_GRID`
+- D) `COMPUTED_DISPLAY_INLINE`
+
+**Verify**: Answer is one of the options above.
---
diff --git a/tests/optimization/group-41.md b/tests/optimization/group-41.md
index 0be64378..63732c5d 100644
--- a/tests/optimization/group-41.md
+++ b/tests/optimization/group-41.md
@@ -1,9 +1,14 @@
# Group 41: Autocomplete / Typeahead
### 41.1 Trigger suggestions
-Navigate to `http://fixtures/autocomplete.html`. Start typing "pyt" in the search box and report what suggestions appear.
+Navigate to `http://fixtures/autocomplete.html`. Type "pyt" in the search box and look at the status element on the page. Pick the option that matches what you see:
-**Verify**: Report lists the matching suggestions.
+- A) `SUGGESTIONS_VISIBLE_COUNT_1`
+- B) `SUGGESTIONS_VISIBLE_COUNT_2`
+- C) `SUGGESTIONS_VISIBLE_COUNT_3`
+- D) `SUGGESTIONS_VISIBLE_COUNT_0`
+
+**Verify**: Answer is one of the options above.
### 41.2 Select a suggestion
Select "Python" from the suggestions.
diff --git a/tests/optimization/subagent-context.md b/tests/optimization/subagent-context.md
index 6ff1103c..5c093444 100644
--- a/tests/optimization/subagent-context.md
+++ b/tests/optimization/subagent-context.md
@@ -47,6 +47,7 @@ For failures:
- `<step>` is the step number within the group (e.g., `1`, `2`, `3`)
- Keep answers factual — do not self-grade in the answer payload.
- **Quote actual output, don't paraphrase.** The answer field must include the literal text or marker from the tool output. For example, if the server returns `status: ok`, write `status: ok` in the answer — not "Server responded with ok". Verification patterns match against exact substrings, so paraphrasing causes false failures.
+- **Always include `UPPER_CASE_MARKER` strings verbatim.** Fixture pages embed verification markers like `SUGGESTIONS_VISIBLE_COUNT_2`, `VERIFY_HOME_LOADED_12345`, `SCROLL_MIDDLE_MARKER`, etc. When you see these in the page content or tool output, copy them exactly into your answer — they are the primary tokens that automated verification matches against.
## Execution approach
diff --git a/tests/tools/runner/internal/bench/recordstep.go b/tests/tools/runner/internal/bench/recordstep.go
index f1e04c29..a22873ca 100644
--- a/tests/tools/runner/internal/bench/recordstep.go
+++ b/tests/tools/runner/internal/bench/recordstep.go
@@ -180,6 +180,12 @@ func RunRecordStep(argv []string, stdout, stderr io.Writer) int {
timestamp := time.Now().UTC().Format(time.RFC3339)
stepID := fmt.Sprintf("%d.%d", args.Group, args.Step)
+ // Reset stale timing files when this is the first step in the report.
+ existingSteps, _ := data["steps"].([]any)
+ if len(existingSteps) == 0 {
+ resetTimingFiles(resultsDir, benchmarkType)
+ }
+
timing := computeTiming(resultsDir, benchmarkType)
dockerStats := collectDockerStats(benchmarkType)
cost := computeCost(args.InputTokens, args.OutputTokens, benchmark)
@@ -286,6 +292,12 @@ func persistTiming(resultsDir, benchmarkType string, stepEndMs int64) {
writeMsFile(lastStepEndFile, stepEndMs)
}
+func resetTimingFiles(resultsDir, benchmarkType string) {
+ key := strings.ReplaceAll(benchmarkType, "_", "-")
+ _ = os.Remove(filepath.Join(resultsDir, fmt.Sprintf("run_start_%s.ms", key)))
+ _ = os.Remove(filepath.Join(resultsDir, fmt.Sprintf("last_step_end_%s.ms", key)))
+}
+
func nowMs() int64 {
return time.Now().UnixMilli()
}
diff --git a/tests/tools/runner/internal/opt/injectusage.go b/tests/tools/runner/internal/opt/injectusage.go
index 564accfd..22a4a2ad 100644
--- a/tests/tools/runner/internal/opt/injectusage.go
+++ b/tests/tools/runner/internal/opt/injectusage.go
@@ -8,6 +8,7 @@ import (
"os"
"path/filepath"
"strings"
+ "time"
)
func RunInjectUsage(argv []string, stdout, stderr io.Writer) int {
@@ -60,6 +61,9 @@ func RunInjectUsage(argv []string, stdout, stderr io.Writer) int {
RequestCount int64 `json:"request_count"`
}
+ var agentDurations []float64
+ var wallClockMs float64
+
for _, f := range files {
fh, err := os.Open(f)
if err != nil {
@@ -68,11 +72,14 @@ func RunInjectUsage(argv []string, stdout, stderr io.Writer) int {
}
var fileRequests int64
+ var firstTS, lastTS string
scanner := bufio.NewScanner(fh)
scanner.Buffer(make([]byte, 0, 1024*1024), 10*1024*1024)
for scanner.Scan() {
+ line := scanner.Bytes()
var entry struct {
- Message struct {
+ Timestamp string `json:"timestamp"`
+ Message struct {
Role string `json:"role"`
Usage struct {
InputTokens int64 `json:"input_tokens"`
@@ -82,9 +89,15 @@ func RunInjectUsage(argv []string, stdout, stderr io.Writer) int {
} `json:"usage"`
} `json:"message"`
}
- if err := json.Unmarshal(scanner.Bytes(), &entry); err != nil {
+ if err := json.Unmarshal(line, &entry); err != nil {
continue
}
+ if entry.Timestamp != "" {
+ if firstTS == "" {
+ firstTS = entry.Timestamp
+ }
+ lastTS = entry.Timestamp
+ }
u := entry.Message.Usage
if u.InputTokens == 0 && u.CacheCreationInputTokens == 0 && u.CacheReadInputTokens == 0 && u.OutputTokens == 0 {
continue
@@ -99,6 +112,18 @@ func RunInjectUsage(argv []string, stdout, stderr io.Writer) int {
totals.RequestCount += fileRequests
_, _ = fmt.Fprintf(stdout, "Parsed %d API responses from %s\n", fileRequests, filepath.Base(f))
+
+ if firstTS != "" && lastTS != "" {
+ t0, err0 := time.Parse(time.RFC3339Nano, firstTS)
+ t1, err1 := time.Parse(time.RFC3339Nano, lastTS)
+ if err0 == nil && err1 == nil {
+ d := t1.Sub(t0).Seconds() * 1000
+ agentDurations = append(agentDurations, d)
+ if d > wallClockMs {
+ wallClockMs = d
+ }
+ }
+ }
}
totalInput := totals.InputTokens + totals.CacheCreationInputTokens + totals.CacheReadInputTokens
@@ -115,6 +140,10 @@ func RunInjectUsage(argv []string, stdout, stderr io.Writer) int {
"output_tokens": totals.OutputTokens,
"total_tokens": totalTokens,
}
+ if wallClockMs > 0 {
+ usage["wall_clock_ms"] = wallClockMs
+ usage["agent_durations_ms"] = agentDurations
+ }
_, _ = fmt.Fprintf(stdout, "\nToken usage:\n")
_, _ = fmt.Fprintf(stdout, " API requests: %d\n", totals.RequestCount)
diff --git a/tests/tools/runner/internal/opt/summarize.go b/tests/tools/runner/internal/opt/summarize.go
index 0f1989b9..ed1cee0f 100644
--- a/tests/tools/runner/internal/opt/summarize.go
+++ b/tests/tools/runner/internal/opt/summarize.go
@@ -211,19 +211,21 @@ func RunSummarize(argv []string, stdout, stderr io.Writer) int {
}
rows = append(rows, row{"Errors", "0", fmt.Sprintf("%d", failed)})
+ // Prefer transcript-based timing, then run_usage wall clock, then per-step sum.
+ var displayDurationMs float64
if totalAgentDurationMs > 0 {
- avgMs := totalAgentDurationMs / float64(blSteps)
- rows = append(rows,
- row{"", "", ""},
- row{"Avg time/step", "", fmt.Sprintf("%.1fs", avgMs/1000)},
- row{"Total time", "", fmtDuration(totalAgentDurationMs)},
- )
+ displayDurationMs = totalAgentDurationMs
+ } else if wc, ok := usage["wall_clock_ms"].(float64); ok && wc > 0 {
+ displayDurationMs = wc
} else if stepsWithDuration > 0 {
- avgMs := totalDurationMs / float64(stepsWithDuration)
+ displayDurationMs = totalDurationMs
+ }
+ if displayDurationMs > 0 {
+ avgMs := displayDurationMs / float64(blSteps)
rows = append(rows,
row{"", "", ""},
row{"Avg time/step", "", fmt.Sprintf("%.1fs", avgMs/1000)},
- row{"Total time", "", fmtDuration(totalDurationMs)},
+ row{"Total time", "", fmtDuration(displayDurationMs)},
)
}
diff --git a/tests/tools/runner/internal/opt/verifyanswers.go b/tests/tools/runner/internal/opt/verifyanswers.go
index 413a1816..d65734df 100644
--- a/tests/tools/runner/internal/opt/verifyanswers.go
+++ b/tests/tools/runner/internal/opt/verifyanswers.go
@@ -19,7 +19,7 @@ var expectPatterns = map[string]string{
"0.2": `401|missing_token|unauth|bad_token|rejected`,
"0.3": `200|authed.*200|status.*ok|^ok$|"ok"`,
"0.4": `running`,
- "0.5": `tab.*list|tabs.*returned|active tab|listed.*tab|\d+ tab.*found|tabs:|tab.*id`,
+ "0.5": `tab.*list|tabs.*returned|active tab|listed.*tab|\d+ tabs? (found|open)|tabs:|tab.*id`,
"0.6": `cleaned|closed|no stale|reused.*tab|no cleanup|single active|is empty|already empty|clean.*state|cannot close last|one tab`,
"0.7": `VERIFY_HOME_LOADED_12345|navigated to fixtures|fixtures.*home|PinchTab Benchmark|fixtures/`,
"0.8": `[0-9A-F]{32}|tab.*id.*captured`,
@@ -36,7 +36,7 @@ var expectPatterns = map[string]string{
"2.3": `Artificial Intelligence|ARTIFICIAL_INTELLIGENCE`,
// Group 3: Form
"3.1": `submitted|VERIFY_FORM_SUBMITTED_SUCCESS|FORM_SUBMITTED|SUBMISSION_DATA`,
- "3.2": `[Rr]eset.*button|#reset-btn|[Rr]eset.*present|reset-btn`,
+ "3.2": `RESET_BUTTON_VISIBLE`,
// Group 4: SPA
"4.1": `TASK_STATS_TOTAL_3|Total.*3.*Active.*2.*Done.*1|3.*2.*1`,
"4.2": `TASK_ADDED|AUTOMATE|DEPLOYMENT|high`,
@@ -76,7 +76,7 @@ var expectPatterns = map[string]string{
"15.1": `1,284,930.*384,930|PROFIT_MARGIN_CALCULATED|Revenue.*Profit`,
"15.2": `FEATURE_COUNT_6.*7.*5|Go=6.*Python=7.*Rust=5|6, 7, 5|COMPARISON_TABLE_BUILT|Go.*6.*Python.*7.*Rust.*5`,
"15.3": `VERIFY_ARTICLE_PAGE_41414|<article|<h[12]|article.*html|html.*article|extracted.*html`,
- "15.4": `flex`,
+ "15.4": `COMPUTED_DISPLAY_FLEX`,
// Group 16: Hover
"16.1": `HOVER_REVEALED_USER_1`,
"16.2": `HOVER_REVEALED_USER_2`,