High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
feat(e2e-runner): sample docker stats for pinchtab containers and record peak/final memory in timings (PIN-382)
Luigi Agosti5da622920e43
diff --git a/tests/tools/runner/internal/e2e/runner.go b/tests/tools/runner/internal/e2e/runner.goindex b43337de..12e60564 100644--- a/tests/tools/runner/internal/e2e/runner.go+++ b/tests/tools/runner/internal/e2e/runner.go@@ -31,6 +31,7 @@ type Runner struct {logsMode stringoverall overallReportDataoverrides *providerOverrides+ mem *suiteMemory}type overallReportData struct {@@ -532,9 +533,17 @@ func (r *Runner) runSinglePlanWithCompose(plan suitePlan, composeFile string) inreturn 1}+ var probe *memoryProbe+ if !r.args.DryRun {+ probe = r.startMemorySampler()+ }code := r.runLoggedCommand("running "+def.Name+" suite", def.Output, command)+ if probe != nil {+ r.applyMemoryResult(probe.finish())+ }duration := time.Since(started)summary := r.writeSuiteReports(def, duration, code)+ r.mem = nilr.recordOverallSummary(summary)r.printSuiteSummary(def, summary, duration)if code != 0 {@@ -1021,6 +1030,103 @@ func (r *Runner) readLastRunningName(outputFile string) string {return last}+func (r *Runner) provider() string {+ if r.args.Provider != "" {+ return r.args.Provider+ }+ return defaultProvider+}++type memoryResult struct {+ containers []containerMemory+ note string+}++type memoryProbe struct {+ stop chan struct{}+ result chan memoryResult+}++func (r *Runner) startMemorySampler() *memoryProbe {+ probe := &memoryProbe{stop: make(chan struct{}), result: make(chan memoryResult, 1)}+ go func() {+ acc := newMemoryAccumulator()+ firstErr := ""+ sampleOnce := func() {+ out, err := r.dockerStatsSnapshot()+ if err != nil {+ if firstErr == "" {+ firstErr = err.Error()+ }+ return+ }+ for _, line := range strings.Split(strings.TrimSpace(out), "\n") {+ if s, ok := parseDockerStatsLine(line); ok && isPinchtabBrowserContainer(s.Name) {+ acc.add(s)+ }+ }+ }++ sampleOnce()+ ticker := time.NewTicker(3 * time.Second)+ defer ticker.Stop()+ for {+ select {+ case <-probe.stop:+ sampleOnce()+ containers := acc.reduce()+ note := ""+ if len(containers) == 0 {+ note = "no pinchtab container memory sampled"+ if firstErr != "" {+ note += " (" + firstErr + ")"+ }+ }+ probe.result <- memoryResult{containers: containers, note: note}+ return+ case <-ticker.C:+ sampleOnce()+ }+ }+ }()+ return probe+}++func (p *memoryProbe) finish() memoryResult {+ close(p.stop)+ return <-p.result+}++func (r *Runner) applyMemoryResult(result memoryResult) {+ if len(result.containers) == 0 {+ r.mem = nil+ if result.note != "" {+ _, _ = fmt.Fprintf(r.stdout, " memory: %s\n", result.note)+ }+ return+ }+ r.mem = &suiteMemory{Provider: r.provider(), Containers: result.containers}+}++func (r *Runner) dockerStatsSnapshot() (string, error) {+ cmd := exec.Command("docker", "stats", "--no-stream", "--format", "{{.Name}},{{.MemUsage}},{{.PIDs}}") // #nosec G204 -- fixed docker stats invocation+ cmd.Dir = r.repoRoot+ cmd.Env = os.Environ()+ var buf bytes.Buffer+ cmd.Stdout = &buf+ cmd.Stderr = &buf+ if err := cmd.Run(); err != nil {+ return "", fmt.Errorf("docker stats unavailable: %v: %s", err, strings.TrimSpace(buf.String()))+ }+ return buf.String(), nil+}++func isPinchtabBrowserContainer(name string) bool {+ return strings.Contains(name, "pinchtab") &&+ !strings.Contains(name, "fixtures") &&+ !strings.Contains(name, "runner")+}+func resolveCompose(dryRun bool) ([]string, error) {if custom := strings.TrimSpace(os.Getenv("PINCHTAB_COMPOSE")); custom != "" {return strings.Fields(custom), nildiff --git a/tests/tools/runner/internal/e2e/timings.go b/tests/tools/runner/internal/e2e/timings.goindex 4145be40..c9b39682 100644--- a/tests/tools/runner/internal/e2e/timings.go+++ b/tests/tools/runner/internal/e2e/timings.go@@ -4,9 +4,11 @@ import ("encoding/json""fmt""io"+ "math""os""path/filepath""sort"+ "strconv""strings")@@ -40,10 +42,128 @@ type suiteTimings struct {ComparisonFloorMs int64 `json:"comparisonFloorMs"`Tests int `json:"tests"`TotalMs int64 `json:"totalMs"`+ Memory *suiteMemory `json:"memory,omitempty"`Scenarios []scenarioTotal `json:"scenarios"`Records []timingRecord `json:"records"`}+type suiteMemory struct {+ Provider string `json:"provider"`+ Containers []containerMemory `json:"containers"`+}++type containerMemory struct {+ Container string `json:"container"`+ PeakMiB float64 `json:"peakMiB"`+ FinalMiB float64 `json:"finalMiB"`+ PeakPids int `json:"peakPids"`+}++type memorySample struct {+ Name string+ MiB float64+ Pids int+}++type memoryAccumulator struct {+ order []string+ seen map[string]bool+ peak map[string]float64+ final map[string]float64+ pids map[string]int+}++func newMemoryAccumulator() *memoryAccumulator {+ return &memoryAccumulator{+ seen: map[string]bool{},+ peak: map[string]float64{},+ final: map[string]float64{},+ pids: map[string]int{},+ }+}++func (a *memoryAccumulator) add(s memorySample) {+ if !a.seen[s.Name] {+ a.seen[s.Name] = true+ a.order = append(a.order, s.Name)+ }+ if s.MiB > a.peak[s.Name] {+ a.peak[s.Name] = s.MiB+ }+ a.final[s.Name] = s.MiB+ if s.Pids > a.pids[s.Name] {+ a.pids[s.Name] = s.Pids+ }+}++func (a *memoryAccumulator) reduce() []containerMemory {+ out := make([]containerMemory, 0, len(a.order))+ for _, name := range a.order {+ out = append(out, containerMemory{+ Container: name,+ PeakMiB: round1(a.peak[name]),+ FinalMiB: round1(a.final[name]),+ PeakPids: a.pids[name],+ })+ }+ return out+}++func round1(v float64) float64 {+ return math.Round(v*10) / 10+}++func parseDockerStatsLine(line string) (memorySample, bool) {+ fields := strings.Split(strings.TrimSpace(line), ",")+ if len(fields) != 3 {+ return memorySample{}, false+ }+ name := strings.TrimSpace(fields[0])+ mib, ok := parseMemUsageMiB(fields[1])+ if name == "" || !ok {+ return memorySample{}, false+ }+ pids, err := strconv.Atoi(strings.TrimSpace(fields[2]))+ if err != nil {+ pids = 0+ }+ return memorySample{Name: name, MiB: mib, Pids: pids}, true+}++func parseMemUsageMiB(field string) (float64, bool) {+ used := field+ if idx := strings.Index(field, "/"); idx >= 0 {+ used = field[:idx]+ }+ used = strings.TrimSpace(used)++ split := 0+ for split < len(used) && (used[split] == '.' || (used[split] >= '0' && used[split] <= '9')) {+ split+++ }+ if split == 0 {+ return 0, false+ }+ value, err := strconv.ParseFloat(used[:split], 64)+ if err != nil {+ return 0, false+ }+ switch strings.TrimSpace(used[split:]) {+ case "B", "":+ return value / (1024 * 1024), true+ case "KiB":+ return value / 1024, true+ case "MiB":+ return value, true+ case "GiB":+ return value * 1024, true+ case "TiB":+ return value * 1024 * 1024, true+ default:+ return 0, false+ }+}+// stackLabel names the compose file the suite ran under. Run-to-run comparisons// that mix stacks are measuring contention, not the change under test.func stackLabel(compose string) string {@@ -120,11 +240,8 @@ func buildSuiteTimings(suite, stack, provider, timestamp string, results []suite}func (r *Runner) writeSuiteTimings(def suiteDef, data suiteReportData, timestamp string) {- provider := r.args.Provider- if provider == "" {- provider = defaultProvider- }- timings := buildSuiteTimings(def.Name, stackLabel(def.Compose), provider, timestamp, data.Results)+ timings := buildSuiteTimings(def.Name, stackLabel(def.Compose), r.provider(), timestamp, data.Results)+ timings.Memory = r.memencoded, err := json.MarshalIndent(timings, "", " ")if err != nil {@@ -174,6 +291,13 @@ func renderSlowest(timings suiteTimings, top int) string {fmt.Fprintf(&out, " %9s %-32s %s\n", formatMs(scenario.Ms), scenario.Scenario, pluralTests(scenario.Tests))}+ if timings.Memory != nil && len(timings.Memory.Containers) > 0 {+ fmt.Fprintf(&out, "\ncontainer memory (browser: %s)\n", timings.Memory.Provider)+ for _, c := range timings.Memory.Containers {+ fmt.Fprintf(&out, " peak %7.0f MiB final %7.0f MiB peak pids %3d %s\n", c.PeakMiB, c.FinalMiB, c.PeakPids, c.Container)+ }+ }+fmt.Fprintf(&out, "\nGate on the totals above, not on per-test ratios: run-to-run swing is ~16%%.\n")fmt.Fprintf(&out, "Exclude tests under %dms from any ratio — at that size the ratio is noise.\n", timings.ComparisonFloorMs)return out.String()diff --git a/tests/tools/runner/internal/e2e/timings_test.go b/tests/tools/runner/internal/e2e/timings_test.goindex cf357209..cc5ac053 100644--- a/tests/tools/runner/internal/e2e/timings_test.go+++ b/tests/tools/runner/internal/e2e/timings_test.go@@ -5,6 +5,7 @@ import ("encoding/json""os""path/filepath"+ "reflect""strings""testing")@@ -201,6 +202,136 @@ func TestParseArgsValidatesSlowest(t *testing.T) {}}+func TestParseDockerStatsLine(t *testing.T) {+ for _, tc := range []struct {+ name string+ line string+ wantOK bool+ wantName string+ wantMiB float64+ wantPids int+ }{+ {"chrome pinchtab", "pinchtab-pinchtab-1,484MiB / 7.667GiB,12", true, "pinchtab-pinchtab-1", 484, 12},+ {"gib used", "pinchtab-pinchtab-secure-1,1.5GiB / 7.667GiB,30", true, "pinchtab-pinchtab-secure-1", 1536, 30},+ {"non-numeric pids keeps the memory sample", "pinchtab-pinchtab-1,200MiB / 8GiB,--", true, "pinchtab-pinchtab-1", 200, 0},+ {"too few fields", "only-two,200MiB", false, "", 0, 0},+ {"unparseable memory", "pinchtab-pinchtab-1,notmem,3", false, "", 0, 0},+ {"blank", "", false, "", 0, 0},+ } {+ t.Run(tc.name, func(t *testing.T) {+ got, ok := parseDockerStatsLine(tc.line)+ if ok != tc.wantOK {+ t.Fatalf("parseDockerStatsLine(%q) ok = %v, want %v", tc.line, ok, tc.wantOK)+ }+ if !ok {+ return+ }+ if got.Name != tc.wantName || got.MiB != tc.wantMiB || got.Pids != tc.wantPids {+ t.Errorf("parseDockerStatsLine(%q) = %+v, want name %q mib %v pids %d", tc.line, got, tc.wantName, tc.wantMiB, tc.wantPids)+ }+ })+ }+}++func TestParseMemUsageMiB(t *testing.T) {+ for _, tc := range []struct {+ field string+ want float64+ wantOK bool+ }{+ {"484MiB / 7.667GiB", 484, true},+ {"1GiB / 8GiB", 1024, true},+ {"512KiB / 8GiB", 0.5, true},+ {"0B / 8GiB", 0, true},+ {"1TiB", 1024 * 1024, true},+ {"nonsense", 0, false},+ } {+ t.Run(tc.field, func(t *testing.T) {+ got, ok := parseMemUsageMiB(tc.field)+ if ok != tc.wantOK || (ok && got != tc.want) {+ t.Errorf("parseMemUsageMiB(%q) = (%v, %v), want (%v, %v)", tc.field, got, ok, tc.want, tc.wantOK)+ }+ })+ }+}++func TestMemoryAccumulatorTracksPeakFinalAndPeakPids(t *testing.T) {+ acc := newMemoryAccumulator()+ for _, s := range []memorySample{+ {Name: "pinchtab-pinchtab-1", MiB: 300, Pids: 4},+ {Name: "pinchtab-pinchtab-secure-1", MiB: 120, Pids: 2},+ {Name: "pinchtab-pinchtab-1", MiB: 670, Pids: 12},+ {Name: "pinchtab-pinchtab-1", MiB: 450, Pids: 9},+ } {+ acc.add(s)+ }++ got := acc.reduce()+ want := []containerMemory{+ {Container: "pinchtab-pinchtab-1", PeakMiB: 670, FinalMiB: 450, PeakPids: 12},+ {Container: "pinchtab-pinchtab-secure-1", PeakMiB: 120, FinalMiB: 120, PeakPids: 2},+ }+ if !reflect.DeepEqual(got, want) {+ t.Errorf("reduce() = %+v, want %+v", got, want)+ }+}++func TestRenderSlowestPrintsContainerMemoryAfterTheScenarioTotals(t *testing.T) {+ timings := buildSuiteTimings("api", "singleCompose", "chrome", "2026-01-01T00:00:00Z", sampleResults())+ timings.Memory = &suiteMemory{+ Provider: "chrome",+ Containers: []containerMemory{+ {Container: "pinchtab-pinchtab-1", PeakMiB: 670, FinalMiB: 450, PeakPids: 12},+ },+ }+ out := renderSlowest(timings, 2)++ totalsIdx := strings.Index(out, "per-scenario totals")+ memIdx := strings.Index(out, "container memory (browser: chrome)")+ if memIdx < 0 {+ t.Fatalf("memory section missing:\n%s", out)+ }+ if memIdx < totalsIdx {+ t.Errorf("memory section must come after the per-scenario totals:\n%s", out)+ }+ if !strings.Contains(out, "pinchtab-pinchtab-1") || !strings.Contains(out, "peak pids 12") {+ t.Errorf("memory line missing peak/pid figures:\n%s", out)+ }++ timings.Memory = nil+ if strings.Contains(renderSlowest(timings, 2), "container memory") {+ t.Error("a run with no sampled memory must not print an empty memory section")+ }+}++func TestSuiteTimingsOmitsMemoryWhenNoneSampled(t *testing.T) {+ timings := buildSuiteTimings("api", "singleCompose", "chrome", "", sampleResults())+ encoded, err := json.Marshal(timings)+ if err != nil {+ t.Fatal(err)+ }+ if strings.Contains(string(encoded), "\"memory\"") {+ t.Errorf("timings JSON carries a memory key with nothing sampled:\n%s", encoded)+ }+}++func TestIsPinchtabBrowserContainerExcludesFixturesAndRunners(t *testing.T) {+ for _, tc := range []struct {+ name string+ want bool+ }{+ {"pinchtab-pinchtab-1", true},+ {"pinchtab-pinchtab-secure-1", true},+ {"pinchtab-fixtures-1", false},+ {"pinchtab-runner-api-run-abc", false},+ {"some-other-service-1", false},+ } {+ if got := isPinchtabBrowserContainer(tc.name); got != tc.want {+ t.Errorf("isPinchtabBrowserContainer(%q) = %v, want %v", tc.name, got, tc.want)+ }+ }+}+func TestPrepareSuiteResultsClearsStaleTimings(t *testing.T) {root := t.TempDir()def := suiteDescriptor{Name: "api-extended", Compose: multiCompose}.build()