Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

fix: keep last step record per id in opt merge-reports and verify-answers

Luigi Agosti6457e080eb7f
diff --git a/tests/tools/runner/internal/opt/mergereports.go b/tests/tools/runner/internal/opt/mergereports.go
index 0c0e5db1..d592a258 100644
--- a/tests/tools/runner/internal/opt/mergereports.go
+++ b/tests/tools/runner/internal/opt/mergereports.go
@@ -64,8 +64,7 @@ func RunMergeReports(argv []string, stdout, stderr io.Writer) int {
}
}
- var allSteps []map[string]any
- seen := make(map[string]bool)
+ seenSteps := make(map[string]map[string]any)
for _, f := range files {
data, err := os.ReadFile(f)
@@ -91,14 +90,16 @@ func RunMergeReports(argv []string, stdout, stderr io.Writer) int {
s, _ := step["step"].(float64)
id = fmt.Sprintf("%d.%d", int(g), int(s))
}
- if !seen[id] {
- seen[id] = true
- allSteps = append(allSteps, step)
- }
+ seenSteps[id] = step
}
_, _ = fmt.Fprintf(stdout, "Loaded %d steps from %s\n", len(steps), filepath.Base(f))
}
+ allSteps := make([]map[string]any, 0, len(seenSteps))
+ for _, step := range seenSteps {
+ allSteps = append(allSteps, step)
+ }
+
// Sort by group.step
sort.Slice(allSteps, func(i, j int) bool {
gi, _ := allSteps[i]["group"].(float64)
diff --git a/tests/tools/runner/internal/opt/mergereports_test.go b/tests/tools/runner/internal/opt/mergereports_test.go
new file mode 100644
index 00000000..f03aaa69
--- /dev/null
+++ b/tests/tools/runner/internal/opt/mergereports_test.go
@@ -0,0 +1,59 @@
+package opt
+
+import (
+ "bytes"
+ "encoding/json"
+ "os"
+ "path/filepath"
+ "testing"
+)
+
+// TestMergeReportsKeepsLastWhenDuplicateStepIDs locks in the merger's
+// "keep last" dedup semantics. step-end appends a new row per call, so an
+// initial fail followed by a successful re-record produces two rows for the
+// same id within a single report file. The merger must keep the later one
+// so downstream verify-answers / summarize see the agent's final answer.
+// Previously this used "keep first", which surfaced stale failure records.
+func TestMergeReportsKeepsLastWhenDuplicateStepIDs(t *testing.T) {
+ tmp := t.TempDir()
+ input := filepath.Join(tmp, "agentA.json")
+ output := filepath.Join(tmp, "merged.json")
+
+ report := `{
+ "steps": [
+ {"id": "7.1", "group": 7, "step": 1, "status": "fail", "answer": "Error 403: blocked"},
+ {"id": "7.1", "group": 7, "step": 1, "status": "answer", "answer": "COMMENT_POSTED_RATING_5"}
+ ]
+}`
+ if err := os.WriteFile(input, []byte(report), 0o644); err != nil {
+ t.Fatal(err)
+ }
+
+ var stdout, stderr bytes.Buffer
+ if code := RunMergeReports([]string{"-o", output, input}, &stdout, &stderr); code != 0 {
+ t.Fatalf("RunMergeReports exit=%d, stderr=%s", code, stderr.String())
+ }
+
+ data, err := os.ReadFile(output)
+ if err != nil {
+ t.Fatal(err)
+ }
+ var merged struct {
+ Steps []struct {
+ ID string `json:"id"`
+ Status string `json:"status"`
+ Answer string `json:"answer"`
+ } `json:"steps"`
+ }
+ if err := json.Unmarshal(data, &merged); err != nil {
+ t.Fatal(err)
+ }
+
+ if len(merged.Steps) != 1 {
+ t.Fatalf("expected 1 deduplicated step, got %d: %+v", len(merged.Steps), merged.Steps)
+ }
+ got := merged.Steps[0]
+ if got.Status != "answer" || got.Answer != "COMMENT_POSTED_RATING_5" {
+ t.Errorf("expected last record to win (status=answer, answer=COMMENT_...), got status=%q answer=%q", got.Status, got.Answer)
+ }
+}
diff --git a/tests/tools/runner/internal/opt/verifyanswers.go b/tests/tools/runner/internal/opt/verifyanswers.go
index 4c4e7750..ce19bb49 100644
--- a/tests/tools/runner/internal/opt/verifyanswers.go
+++ b/tests/tools/runner/internal/opt/verifyanswers.go
@@ -238,7 +238,6 @@ func RunVerifyAnswers(argv []string, stdout, stderr io.Writer) int {
}
}
- // Load all steps, deduplicate by ID (keep first seen)
type stepData struct {
ID string `json:"id"`
Answer string `json:"answer"`
@@ -259,9 +258,7 @@ func RunVerifyAnswers(argv []string, stdout, stderr io.Writer) int {
return 1
}
for _, s := range report.Steps {
- if _, exists := seen[s.ID]; !exists {
- seen[s.ID] = s
- }
+ seen[s.ID] = s
}
}
diff --git a/tests/tools/runner/internal/opt/verifyanswers_test.go b/tests/tools/runner/internal/opt/verifyanswers_test.go
new file mode 100644
index 00000000..45fb5584
--- /dev/null
+++ b/tests/tools/runner/internal/opt/verifyanswers_test.go
@@ -0,0 +1,39 @@
+package opt
+
+import (
+ "bytes"
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+)
+
+func TestVerifyAnswersKeepsLastWhenDuplicateStepIDs(t *testing.T) {
+ tmp := t.TempDir()
+ reportPath := filepath.Join(tmp, "report.json")
+ // Step 0.1 has two records: first a fail (would fail verification),
+ // then a successful re-record (matches expected pattern for 0.1).
+ report := `{
+ "steps": [
+ {"id": "0.1", "status": "fail", "answer": "Error 403: blocked"},
+ {"id": "0.1", "status": "answer", "answer": "ok"}
+ ]
+}`
+ if err := os.WriteFile(reportPath, []byte(report), 0o644); err != nil {
+ t.Fatal(err)
+ }
+
+ var stdout, stderr bytes.Buffer
+ code := RunVerifyAnswers([]string{reportPath}, &stdout, &stderr)
+ if code != 0 {
+ t.Fatalf("RunVerifyAnswers exit=%d, stderr=%s", code, stderr.String())
+ }
+
+ out := stdout.String()
+ if !strings.Contains(out, "pass: 1") {
+ t.Errorf("expected 'pass: 1' (latest record matches expected pattern), got:\n%s", out)
+ }
+ if strings.Contains(out, "fail: 1") {
+ t.Errorf("did not expect 'fail: 1' — verifier must pick the latest entry, not the first:\n%s", out)
+ }
+}