High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
fix: keep last step record per id in opt merge-reports and verify-answers
Luigi Agosti6457e080eb7f
diff --git a/tests/tools/runner/internal/opt/mergereports.go b/tests/tools/runner/internal/opt/mergereports.goindex 0c0e5db1..d592a258 100644--- a/tests/tools/runner/internal/opt/mergereports.go+++ b/tests/tools/runner/internal/opt/mergereports.go@@ -64,8 +64,7 @@ func RunMergeReports(argv []string, stdout, stderr io.Writer) int {}}- var allSteps []map[string]any- seen := make(map[string]bool)+ seenSteps := make(map[string]map[string]any)for _, f := range files {data, err := os.ReadFile(f)@@ -91,14 +90,16 @@ func RunMergeReports(argv []string, stdout, stderr io.Writer) int {s, _ := step["step"].(float64)id = fmt.Sprintf("%d.%d", int(g), int(s))}- if !seen[id] {- seen[id] = true- allSteps = append(allSteps, step)- }+ seenSteps[id] = step}_, _ = fmt.Fprintf(stdout, "Loaded %d steps from %s\n", len(steps), filepath.Base(f))}+ allSteps := make([]map[string]any, 0, len(seenSteps))+ for _, step := range seenSteps {+ allSteps = append(allSteps, step)+ }+// Sort by group.stepsort.Slice(allSteps, func(i, j int) bool {gi, _ := allSteps[i]["group"].(float64)diff --git a/tests/tools/runner/internal/opt/mergereports_test.go b/tests/tools/runner/internal/opt/mergereports_test.gonew file mode 100644index 00000000..f03aaa69--- /dev/null+++ b/tests/tools/runner/internal/opt/mergereports_test.go@@ -0,0 +1,59 @@+package opt++import (+ "bytes"+ "encoding/json"+ "os"+ "path/filepath"+ "testing"+)++// TestMergeReportsKeepsLastWhenDuplicateStepIDs locks in the merger's+// "keep last" dedup semantics. step-end appends a new row per call, so an+// initial fail followed by a successful re-record produces two rows for the+// same id within a single report file. The merger must keep the later one+// so downstream verify-answers / summarize see the agent's final answer.+// Previously this used "keep first", which surfaced stale failure records.+func TestMergeReportsKeepsLastWhenDuplicateStepIDs(t *testing.T) {+ tmp := t.TempDir()+ input := filepath.Join(tmp, "agentA.json")+ output := filepath.Join(tmp, "merged.json")++ report := `{+ "steps": [+ {"id": "7.1", "group": 7, "step": 1, "status": "fail", "answer": "Error 403: blocked"},+ {"id": "7.1", "group": 7, "step": 1, "status": "answer", "answer": "COMMENT_POSTED_RATING_5"}+ ]+}`+ if err := os.WriteFile(input, []byte(report), 0o644); err != nil {+ t.Fatal(err)+ }++ var stdout, stderr bytes.Buffer+ if code := RunMergeReports([]string{"-o", output, input}, &stdout, &stderr); code != 0 {+ t.Fatalf("RunMergeReports exit=%d, stderr=%s", code, stderr.String())+ }++ data, err := os.ReadFile(output)+ if err != nil {+ t.Fatal(err)+ }+ var merged struct {+ Steps []struct {+ ID string `json:"id"`+ Status string `json:"status"`+ Answer string `json:"answer"`+ } `json:"steps"`+ }+ if err := json.Unmarshal(data, &merged); err != nil {+ t.Fatal(err)+ }++ if len(merged.Steps) != 1 {+ t.Fatalf("expected 1 deduplicated step, got %d: %+v", len(merged.Steps), merged.Steps)+ }+ got := merged.Steps[0]+ if got.Status != "answer" || got.Answer != "COMMENT_POSTED_RATING_5" {+ t.Errorf("expected last record to win (status=answer, answer=COMMENT_...), got status=%q answer=%q", got.Status, got.Answer)+ }+}diff --git a/tests/tools/runner/internal/opt/verifyanswers.go b/tests/tools/runner/internal/opt/verifyanswers.goindex 4c4e7750..ce19bb49 100644--- a/tests/tools/runner/internal/opt/verifyanswers.go+++ b/tests/tools/runner/internal/opt/verifyanswers.go@@ -238,7 +238,6 @@ func RunVerifyAnswers(argv []string, stdout, stderr io.Writer) int {}}- // Load all steps, deduplicate by ID (keep first seen)type stepData struct {ID string `json:"id"`Answer string `json:"answer"`@@ -259,9 +258,7 @@ func RunVerifyAnswers(argv []string, stdout, stderr io.Writer) int {return 1}for _, s := range report.Steps {- if _, exists := seen[s.ID]; !exists {- seen[s.ID] = s- }+ seen[s.ID] = s}}diff --git a/tests/tools/runner/internal/opt/verifyanswers_test.go b/tests/tools/runner/internal/opt/verifyanswers_test.gonew file mode 100644index 00000000..45fb5584--- /dev/null+++ b/tests/tools/runner/internal/opt/verifyanswers_test.go@@ -0,0 +1,39 @@+package opt++import (+ "bytes"+ "os"+ "path/filepath"+ "strings"+ "testing"+)++func TestVerifyAnswersKeepsLastWhenDuplicateStepIDs(t *testing.T) {+ tmp := t.TempDir()+ reportPath := filepath.Join(tmp, "report.json")+ // Step 0.1 has two records: first a fail (would fail verification),+ // then a successful re-record (matches expected pattern for 0.1).+ report := `{+ "steps": [+ {"id": "0.1", "status": "fail", "answer": "Error 403: blocked"},+ {"id": "0.1", "status": "answer", "answer": "ok"}+ ]+}`+ if err := os.WriteFile(reportPath, []byte(report), 0o644); err != nil {+ t.Fatal(err)+ }++ var stdout, stderr bytes.Buffer+ code := RunVerifyAnswers([]string{reportPath}, &stdout, &stderr)+ if code != 0 {+ t.Fatalf("RunVerifyAnswers exit=%d, stderr=%s", code, stderr.String())+ }++ out := stdout.String()+ if !strings.Contains(out, "pass: 1") {+ t.Errorf("expected 'pass: 1' (latest record matches expected pattern), got:\n%s", out)+ }+ if strings.Contains(out, "fail: 1") {+ t.Errorf("did not expect 'fail: 1' — verifier must pick the latest entry, not the first:\n%s", out)+ }+}