Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

feat(e2e-runner): sample docker stats for pinchtab containers and record peak/final memory in timings (PIN-382)

Luigi Agosti5da622920e43
diff --git a/tests/tools/runner/internal/e2e/runner.go b/tests/tools/runner/internal/e2e/runner.go
index b43337de..12e60564 100644
--- a/tests/tools/runner/internal/e2e/runner.go
+++ b/tests/tools/runner/internal/e2e/runner.go
@@ -31,6 +31,7 @@ type Runner struct {
logsMode string
overall overallReportData
overrides *providerOverrides
+ mem *suiteMemory
}
type overallReportData struct {
@@ -532,9 +533,17 @@ func (r *Runner) runSinglePlanWithCompose(plan suitePlan, composeFile string) in
return 1
}
+ var probe *memoryProbe
+ if !r.args.DryRun {
+ probe = r.startMemorySampler()
+ }
code := r.runLoggedCommand("running "+def.Name+" suite", def.Output, command)
+ if probe != nil {
+ r.applyMemoryResult(probe.finish())
+ }
duration := time.Since(started)
summary := r.writeSuiteReports(def, duration, code)
+ r.mem = nil
r.recordOverallSummary(summary)
r.printSuiteSummary(def, summary, duration)
if code != 0 {
@@ -1021,6 +1030,103 @@ func (r *Runner) readLastRunningName(outputFile string) string {
return last
}
+func (r *Runner) provider() string {
+ if r.args.Provider != "" {
+ return r.args.Provider
+ }
+ return defaultProvider
+}
+
+type memoryResult struct {
+ containers []containerMemory
+ note string
+}
+
+type memoryProbe struct {
+ stop chan struct{}
+ result chan memoryResult
+}
+
+func (r *Runner) startMemorySampler() *memoryProbe {
+ probe := &memoryProbe{stop: make(chan struct{}), result: make(chan memoryResult, 1)}
+ go func() {
+ acc := newMemoryAccumulator()
+ firstErr := ""
+ sampleOnce := func() {
+ out, err := r.dockerStatsSnapshot()
+ if err != nil {
+ if firstErr == "" {
+ firstErr = err.Error()
+ }
+ return
+ }
+ for _, line := range strings.Split(strings.TrimSpace(out), "\n") {
+ if s, ok := parseDockerStatsLine(line); ok && isPinchtabBrowserContainer(s.Name) {
+ acc.add(s)
+ }
+ }
+ }
+
+ sampleOnce()
+ ticker := time.NewTicker(3 * time.Second)
+ defer ticker.Stop()
+ for {
+ select {
+ case <-probe.stop:
+ sampleOnce()
+ containers := acc.reduce()
+ note := ""
+ if len(containers) == 0 {
+ note = "no pinchtab container memory sampled"
+ if firstErr != "" {
+ note += " (" + firstErr + ")"
+ }
+ }
+ probe.result <- memoryResult{containers: containers, note: note}
+ return
+ case <-ticker.C:
+ sampleOnce()
+ }
+ }
+ }()
+ return probe
+}
+
+func (p *memoryProbe) finish() memoryResult {
+ close(p.stop)
+ return <-p.result
+}
+
+func (r *Runner) applyMemoryResult(result memoryResult) {
+ if len(result.containers) == 0 {
+ r.mem = nil
+ if result.note != "" {
+ _, _ = fmt.Fprintf(r.stdout, " memory: %s\n", result.note)
+ }
+ return
+ }
+ r.mem = &suiteMemory{Provider: r.provider(), Containers: result.containers}
+}
+
+func (r *Runner) dockerStatsSnapshot() (string, error) {
+ cmd := exec.Command("docker", "stats", "--no-stream", "--format", "{{.Name}},{{.MemUsage}},{{.PIDs}}") // #nosec G204 -- fixed docker stats invocation
+ cmd.Dir = r.repoRoot
+ cmd.Env = os.Environ()
+ var buf bytes.Buffer
+ cmd.Stdout = &buf
+ cmd.Stderr = &buf
+ if err := cmd.Run(); err != nil {
+ return "", fmt.Errorf("docker stats unavailable: %v: %s", err, strings.TrimSpace(buf.String()))
+ }
+ return buf.String(), nil
+}
+
+func isPinchtabBrowserContainer(name string) bool {
+ return strings.Contains(name, "pinchtab") &&
+ !strings.Contains(name, "fixtures") &&
+ !strings.Contains(name, "runner")
+}
+
func resolveCompose(dryRun bool) ([]string, error) {
if custom := strings.TrimSpace(os.Getenv("PINCHTAB_COMPOSE")); custom != "" {
return strings.Fields(custom), nil
diff --git a/tests/tools/runner/internal/e2e/timings.go b/tests/tools/runner/internal/e2e/timings.go
index 4145be40..c9b39682 100644
--- a/tests/tools/runner/internal/e2e/timings.go
+++ b/tests/tools/runner/internal/e2e/timings.go
@@ -4,9 +4,11 @@ import (
"encoding/json"
"fmt"
"io"
+ "math"
"os"
"path/filepath"
"sort"
+ "strconv"
"strings"
)
@@ -40,10 +42,128 @@ type suiteTimings struct {
ComparisonFloorMs int64 `json:"comparisonFloorMs"`
Tests int `json:"tests"`
TotalMs int64 `json:"totalMs"`
+ Memory *suiteMemory `json:"memory,omitempty"`
Scenarios []scenarioTotal `json:"scenarios"`
Records []timingRecord `json:"records"`
}
+type suiteMemory struct {
+ Provider string `json:"provider"`
+ Containers []containerMemory `json:"containers"`
+}
+
+type containerMemory struct {
+ Container string `json:"container"`
+ PeakMiB float64 `json:"peakMiB"`
+ FinalMiB float64 `json:"finalMiB"`
+ PeakPids int `json:"peakPids"`
+}
+
+type memorySample struct {
+ Name string
+ MiB float64
+ Pids int
+}
+
+type memoryAccumulator struct {
+ order []string
+ seen map[string]bool
+ peak map[string]float64
+ final map[string]float64
+ pids map[string]int
+}
+
+func newMemoryAccumulator() *memoryAccumulator {
+ return &memoryAccumulator{
+ seen: map[string]bool{},
+ peak: map[string]float64{},
+ final: map[string]float64{},
+ pids: map[string]int{},
+ }
+}
+
+func (a *memoryAccumulator) add(s memorySample) {
+ if !a.seen[s.Name] {
+ a.seen[s.Name] = true
+ a.order = append(a.order, s.Name)
+ }
+ if s.MiB > a.peak[s.Name] {
+ a.peak[s.Name] = s.MiB
+ }
+ a.final[s.Name] = s.MiB
+ if s.Pids > a.pids[s.Name] {
+ a.pids[s.Name] = s.Pids
+ }
+}
+
+func (a *memoryAccumulator) reduce() []containerMemory {
+ out := make([]containerMemory, 0, len(a.order))
+ for _, name := range a.order {
+ out = append(out, containerMemory{
+ Container: name,
+ PeakMiB: round1(a.peak[name]),
+ FinalMiB: round1(a.final[name]),
+ PeakPids: a.pids[name],
+ })
+ }
+ return out
+}
+
+func round1(v float64) float64 {
+ return math.Round(v*10) / 10
+}
+
+func parseDockerStatsLine(line string) (memorySample, bool) {
+ fields := strings.Split(strings.TrimSpace(line), ",")
+ if len(fields) != 3 {
+ return memorySample{}, false
+ }
+ name := strings.TrimSpace(fields[0])
+ mib, ok := parseMemUsageMiB(fields[1])
+ if name == "" || !ok {
+ return memorySample{}, false
+ }
+ pids, err := strconv.Atoi(strings.TrimSpace(fields[2]))
+ if err != nil {
+ pids = 0
+ }
+ return memorySample{Name: name, MiB: mib, Pids: pids}, true
+}
+
+func parseMemUsageMiB(field string) (float64, bool) {
+ used := field
+ if idx := strings.Index(field, "/"); idx >= 0 {
+ used = field[:idx]
+ }
+ used = strings.TrimSpace(used)
+
+ split := 0
+ for split < len(used) && (used[split] == '.' || (used[split] >= '0' && used[split] <= '9')) {
+ split++
+ }
+ if split == 0 {
+ return 0, false
+ }
+ value, err := strconv.ParseFloat(used[:split], 64)
+ if err != nil {
+ return 0, false
+ }
+ switch strings.TrimSpace(used[split:]) {
+ case "B", "":
+ return value / (1024 * 1024), true
+ case "KiB":
+ return value / 1024, true
+ case "MiB":
+ return value, true
+ case "GiB":
+ return value * 1024, true
+ case "TiB":
+ return value * 1024 * 1024, true
+ default:
+ return 0, false
+ }
+}
+
// stackLabel names the compose file the suite ran under. Run-to-run comparisons
// that mix stacks are measuring contention, not the change under test.
func stackLabel(compose string) string {
@@ -120,11 +240,8 @@ func buildSuiteTimings(suite, stack, provider, timestamp string, results []suite
}
func (r *Runner) writeSuiteTimings(def suiteDef, data suiteReportData, timestamp string) {
- provider := r.args.Provider
- if provider == "" {
- provider = defaultProvider
- }
- timings := buildSuiteTimings(def.Name, stackLabel(def.Compose), provider, timestamp, data.Results)
+ timings := buildSuiteTimings(def.Name, stackLabel(def.Compose), r.provider(), timestamp, data.Results)
+ timings.Memory = r.mem
encoded, err := json.MarshalIndent(timings, "", " ")
if err != nil {
@@ -174,6 +291,13 @@ func renderSlowest(timings suiteTimings, top int) string {
fmt.Fprintf(&out, " %9s %-32s %s\n", formatMs(scenario.Ms), scenario.Scenario, pluralTests(scenario.Tests))
}
+ if timings.Memory != nil && len(timings.Memory.Containers) > 0 {
+ fmt.Fprintf(&out, "\ncontainer memory (browser: %s)\n", timings.Memory.Provider)
+ for _, c := range timings.Memory.Containers {
+ fmt.Fprintf(&out, " peak %7.0f MiB final %7.0f MiB peak pids %3d %s\n", c.PeakMiB, c.FinalMiB, c.PeakPids, c.Container)
+ }
+ }
+
fmt.Fprintf(&out, "\nGate on the totals above, not on per-test ratios: run-to-run swing is ~16%%.\n")
fmt.Fprintf(&out, "Exclude tests under %dms from any ratio — at that size the ratio is noise.\n", timings.ComparisonFloorMs)
return out.String()
diff --git a/tests/tools/runner/internal/e2e/timings_test.go b/tests/tools/runner/internal/e2e/timings_test.go
index cf357209..cc5ac053 100644
--- a/tests/tools/runner/internal/e2e/timings_test.go
+++ b/tests/tools/runner/internal/e2e/timings_test.go
@@ -5,6 +5,7 @@ import (
"encoding/json"
"os"
"path/filepath"
+ "reflect"
"strings"
"testing"
)
@@ -201,6 +202,136 @@ func TestParseArgsValidatesSlowest(t *testing.T) {
}
}
+func TestParseDockerStatsLine(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ line string
+ wantOK bool
+ wantName string
+ wantMiB float64
+ wantPids int
+ }{
+ {"chrome pinchtab", "pinchtab-pinchtab-1,484MiB / 7.667GiB,12", true, "pinchtab-pinchtab-1", 484, 12},
+ {"gib used", "pinchtab-pinchtab-secure-1,1.5GiB / 7.667GiB,30", true, "pinchtab-pinchtab-secure-1", 1536, 30},
+ {"non-numeric pids keeps the memory sample", "pinchtab-pinchtab-1,200MiB / 8GiB,--", true, "pinchtab-pinchtab-1", 200, 0},
+ {"too few fields", "only-two,200MiB", false, "", 0, 0},
+ {"unparseable memory", "pinchtab-pinchtab-1,notmem,3", false, "", 0, 0},
+ {"blank", "", false, "", 0, 0},
+ } {
+ t.Run(tc.name, func(t *testing.T) {
+ got, ok := parseDockerStatsLine(tc.line)
+ if ok != tc.wantOK {
+ t.Fatalf("parseDockerStatsLine(%q) ok = %v, want %v", tc.line, ok, tc.wantOK)
+ }
+ if !ok {
+ return
+ }
+ if got.Name != tc.wantName || got.MiB != tc.wantMiB || got.Pids != tc.wantPids {
+ t.Errorf("parseDockerStatsLine(%q) = %+v, want name %q mib %v pids %d", tc.line, got, tc.wantName, tc.wantMiB, tc.wantPids)
+ }
+ })
+ }
+}
+
+func TestParseMemUsageMiB(t *testing.T) {
+ for _, tc := range []struct {
+ field string
+ want float64
+ wantOK bool
+ }{
+ {"484MiB / 7.667GiB", 484, true},
+ {"1GiB / 8GiB", 1024, true},
+ {"512KiB / 8GiB", 0.5, true},
+ {"0B / 8GiB", 0, true},
+ {"1TiB", 1024 * 1024, true},
+ {"nonsense", 0, false},
+ } {
+ t.Run(tc.field, func(t *testing.T) {
+ got, ok := parseMemUsageMiB(tc.field)
+ if ok != tc.wantOK || (ok && got != tc.want) {
+ t.Errorf("parseMemUsageMiB(%q) = (%v, %v), want (%v, %v)", tc.field, got, ok, tc.want, tc.wantOK)
+ }
+ })
+ }
+}
+
+func TestMemoryAccumulatorTracksPeakFinalAndPeakPids(t *testing.T) {
+ acc := newMemoryAccumulator()
+ for _, s := range []memorySample{
+ {Name: "pinchtab-pinchtab-1", MiB: 300, Pids: 4},
+ {Name: "pinchtab-pinchtab-secure-1", MiB: 120, Pids: 2},
+ {Name: "pinchtab-pinchtab-1", MiB: 670, Pids: 12},
+ {Name: "pinchtab-pinchtab-1", MiB: 450, Pids: 9},
+ } {
+ acc.add(s)
+ }
+
+ got := acc.reduce()
+ want := []containerMemory{
+ {Container: "pinchtab-pinchtab-1", PeakMiB: 670, FinalMiB: 450, PeakPids: 12},
+ {Container: "pinchtab-pinchtab-secure-1", PeakMiB: 120, FinalMiB: 120, PeakPids: 2},
+ }
+ if !reflect.DeepEqual(got, want) {
+ t.Errorf("reduce() = %+v, want %+v", got, want)
+ }
+}
+
+func TestRenderSlowestPrintsContainerMemoryAfterTheScenarioTotals(t *testing.T) {
+ timings := buildSuiteTimings("api", "singleCompose", "chrome", "2026-01-01T00:00:00Z", sampleResults())
+ timings.Memory = &suiteMemory{
+ Provider: "chrome",
+ Containers: []containerMemory{
+ {Container: "pinchtab-pinchtab-1", PeakMiB: 670, FinalMiB: 450, PeakPids: 12},
+ },
+ }
+ out := renderSlowest(timings, 2)
+
+ totalsIdx := strings.Index(out, "per-scenario totals")
+ memIdx := strings.Index(out, "container memory (browser: chrome)")
+ if memIdx < 0 {
+ t.Fatalf("memory section missing:\n%s", out)
+ }
+ if memIdx < totalsIdx {
+ t.Errorf("memory section must come after the per-scenario totals:\n%s", out)
+ }
+ if !strings.Contains(out, "pinchtab-pinchtab-1") || !strings.Contains(out, "peak pids 12") {
+ t.Errorf("memory line missing peak/pid figures:\n%s", out)
+ }
+
+ timings.Memory = nil
+ if strings.Contains(renderSlowest(timings, 2), "container memory") {
+ t.Error("a run with no sampled memory must not print an empty memory section")
+ }
+}
+
+func TestSuiteTimingsOmitsMemoryWhenNoneSampled(t *testing.T) {
+ timings := buildSuiteTimings("api", "singleCompose", "chrome", "", sampleResults())
+ encoded, err := json.Marshal(timings)
+ if err != nil {
+ t.Fatal(err)
+ }
+ if strings.Contains(string(encoded), "\"memory\"") {
+ t.Errorf("timings JSON carries a memory key with nothing sampled:\n%s", encoded)
+ }
+}
+
+func TestIsPinchtabBrowserContainerExcludesFixturesAndRunners(t *testing.T) {
+ for _, tc := range []struct {
+ name string
+ want bool
+ }{
+ {"pinchtab-pinchtab-1", true},
+ {"pinchtab-pinchtab-secure-1", true},
+ {"pinchtab-fixtures-1", false},
+ {"pinchtab-runner-api-run-abc", false},
+ {"some-other-service-1", false},
+ } {
+ if got := isPinchtabBrowserContainer(tc.name); got != tc.want {
+ t.Errorf("isPinchtabBrowserContainer(%q) = %v, want %v", tc.name, got, tc.want)
+ }
+ }
+}
+
func TestPrepareSuiteResultsClearsStaleTimings(t *testing.T) {
root := t.TempDir()
def := suiteDescriptor{Name: "api-extended", Compose: multiCompose}.build()