High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.
test: name the dead container, the OOM kill and the full Docker disk when an e2e stack starves (PIN-028)
Luigi Agosti139531d311eb
diff --git a/tests/tools/runner/internal/e2e/runner.go b/tests/tools/runner/internal/e2e/runner.goindex e75be08c..f07200eb 100644--- a/tests/tools/runner/internal/e2e/runner.go+++ b/tests/tools/runner/internal/e2e/runner.go@@ -587,6 +587,7 @@ func (r *Runner) runSinglePlanWithCompose(plan suitePlan, composeFile string) inr.printSuiteSummary(def, summary, duration)if code != 0 {r.dumpComposeFailure(composeFile, def)+ r.reportServiceDeaths(composeFile, def.LogServices)r.showFailureArtifacts(def, duration)}return code@@ -639,10 +640,12 @@ func (r *Runner) bringUpSharedStack(composeFile string, services []string) int {}args = append(args, services...)if code := r.runLoggedCommand("starting shared stack", stackOutput, r.composeArgs(composeFile, args...)); code != 0 {+ r.reportServiceDeaths(composeFile, services)return code}if err := r.assertStealthStatus(composeFile); err != nil {_, _ = fmt.Fprintf(r.stderr, "e2e: pre-suite stealth assertion failed: %v\n", err)+ r.reportServiceDeaths(composeFile, services)return 1}return 0@@ -658,6 +661,10 @@ func (r *Runner) buildSharedStackWithOverrides(composeFile string, includeOverriif code == 0 {return 0}+ if r.stackOutputIsOutOfDisk() {+ _, _ = fmt.Fprintln(r.stdout, outOfDiskRemedy)+ return code+ }if !r.stackOutputHasBuildKitCacheFailure() {return code}@@ -685,11 +692,19 @@ func needsStockPinchtabImage(services []string) bool {}func (r *Runner) stackOutputHasBuildKitCacheFailure() bool {+ return isBuildKitCacheFailureLog(r.stackOutputLog())+}++func (r *Runner) stackOutputIsOutOfDisk() bool {+ return isOutOfDiskLog(r.stackOutputLog())+}++func (r *Runner) stackOutputLog() string {data, err := os.ReadFile(filepath.Join(r.repoRoot, stackOutput))if err != nil {- return false+ return ""}- return isBuildKitCacheFailureLog(string(data))+ return string(data)}func isBuildKitCacheFailureLog(log string) bool {diff --git a/tests/tools/runner/internal/e2e/stack_diagnostics.go b/tests/tools/runner/internal/e2e/stack_diagnostics.gonew file mode 100644index 00000000..ab660ee9--- /dev/null+++ b/tests/tools/runner/internal/e2e/stack_diagnostics.go@@ -0,0 +1,147 @@+package e2e++import (+ "encoding/json"+ "fmt"+ "path/filepath"+ "strings"+)++type serviceState struct {+ ID string `json:"ID"`+ Service string `json:"Service"`+ State string `json:"State"`+ ExitCode int `json:"ExitCode"`++ oomKilled bool+}++// parseComposePS reads `compose ps --format json`, which emits one object per+// line on modern Compose and a single array on older versions.+func parseComposePS(raw []byte) []serviceState {+ trimmed := strings.TrimSpace(string(raw))+ if trimmed == "" {+ return nil+ }+ if strings.HasPrefix(trimmed, "[") {+ var states []serviceState+ if err := json.Unmarshal([]byte(trimmed), &states); err != nil {+ return nil+ }+ return states+ }+ var states []serviceState+ for _, line := range strings.Split(trimmed, "\n") {+ line = strings.TrimSpace(line)+ if line == "" {+ continue+ }+ var state serviceState+ if err := json.Unmarshal([]byte(line), &state); err != nil {+ continue+ }+ states = append(states, state)+ }+ return states+}++// describeServiceDeath reports the loud line for a service that is no longer+// running, and "" for one that is. Exit 137 is SIGKILL, which is what a starved+// container looks like from the outside, so the verdict is stated here rather+// than left for a reader to infer from a client-side EOF.+func describeServiceDeath(state serviceState) string {+ if strings.EqualFold(state.State, "running") {+ return ""+ }+ line := fmt.Sprintf(" container: %s is %s (exit %d)", state.Service, strings.ToLower(state.State), state.ExitCode)+ switch {+ case state.oomKilled:+ return line + " — OOM-KILLED: the container ran out of memory. Raise the service limit or the Docker VM's memory."+ case state.ExitCode == 137:+ return line + " — SIGKILL: killed from outside the process; the kernel recorded no OOM, so suspect an external stop or a starved Docker VM."+ case state.ExitCode != 0:+ return line + " — the process exited on its own; read its captured log for the reason."+ }+ return line+}++func serviceDeathReport(states []serviceState) []string {+ var lines []string+ for _, state := range states {+ if line := describeServiceDeath(state); line != "" {+ lines = append(lines, line)+ }+ }+ return lines+}++// reportServiceDeaths prints the state of every named service, so a run that+// failed because a container died says so instead of leaving the reader with a+// bare connection error. Both the suite-failure and the stack-bring-up paths+// call it: a starved container dies during bring-up too, where the only symptom+// is a refused connection on the readiness probe.+func (r *Runner) reportServiceDeaths(composeFile string, services []string) {+ if r.args.DryRun || len(services) == 0 {+ return+ }+ args := append([]string{"ps", "-a", "--format", "json"}, services...)+ raw, err := commandOutput(r.repoRoot, r.composeArgs(composeFile, args...))+ if len(raw) == 0 {+ if err != nil {+ _, _ = fmt.Fprintf(r.stderr, "e2e: failed to read container state: %v\n", err)+ }+ return+ }+ states := parseComposePS(raw)+ for i := range states {+ states[i].oomKilled = r.containerWasOOMKilled(states[i].ID)+ }+ for _, state := range states {+ line := describeServiceDeath(state)+ if line == "" {+ continue+ }+ _, _ = fmt.Fprintln(r.stdout, line)+ _, _ = fmt.Fprintf(r.stdout, " log: %s\n", r.captureServiceLog(composeFile, state.Service))+ }+}++// captureServiceLog writes the dead service's own log where the reader can find+// it, since the stack-bring-up path has no suite artifacts to fall back on.+func (r *Runner) captureServiceLog(composeFile, service string) string {+ rel := filepath.Join(resultsDir, "logs-stack-"+service+".log")+ if err := writeCommandOutput(filepath.Join(r.repoRoot, rel), r.repoRoot, r.composeArgs(composeFile, "logs", "--no-color", service)); err != nil {+ return "unavailable: " + err.Error()+ }+ return rel+}++func (r *Runner) containerWasOOMKilled(containerID string) bool {+ if strings.TrimSpace(containerID) == "" {+ return false+ }+ out, err := commandOutput(r.repoRoot, []string{dockerBinary(r.compose), "inspect", containerID, "--format", "{{.State.OOMKilled}}"})+ if err != nil {+ return false+ }+ return strings.TrimSpace(string(out)) == "true"+}++// isOutOfDiskLog spots the Docker VM running out of disk, which reaches the+// reader as an ordinary build failure with the reason buried in the build log.+func isOutOfDiskLog(log string) bool {+ return strings.Contains(strings.ToLower(log), "no space left on device")+}++const outOfDiskRemedy = " the Docker VM is OUT OF DISK — reclaim space before re-running: docker system df, then docker builder prune -f (and docker image prune -a)"++func commandOutput(dir string, command []string) ([]byte, error) {+ return execCommand(command, dir).Output()+}++func dockerBinary(compose []string) string {+ if len(compose) > 0 && filepath.Base(compose[0]) == "docker" {+ return compose[0]+ }+ return "docker"+}diff --git a/tests/tools/runner/internal/e2e/stack_diagnostics_test.go b/tests/tools/runner/internal/e2e/stack_diagnostics_test.gonew file mode 100644index 00000000..3b8d7822--- /dev/null+++ b/tests/tools/runner/internal/e2e/stack_diagnostics_test.go@@ -0,0 +1,114 @@+package e2e++import (+ "strings"+ "testing"+)++func TestParseComposePSReadsBothComposeFormats(t *testing.T) {+ ndjson := []byte(`{"ID":"abc","Service":"pinchtab","State":"exited","ExitCode":137}+{"ID":"def","Service":"fixtures","State":"running","ExitCode":0}`)+ states := parseComposePS(ndjson)+ if len(states) != 2 {+ t.Fatalf("ndjson states = %d, want 2: %+v", len(states), states)+ }+ if states[0].Service != "pinchtab" || states[0].ExitCode != 137 || states[0].ID != "abc" {+ t.Fatalf("first ndjson state = %+v", states[0])+ }++ array := []byte(`[{"ID":"abc","Service":"pinchtab","State":"exited","ExitCode":1}]`)+ states = parseComposePS(array)+ if len(states) != 1 || states[0].ExitCode != 1 {+ t.Fatalf("array states = %+v", states)+ }++ if got := parseComposePS([]byte(" ")); got != nil {+ t.Fatalf("blank output states = %+v, want none", got)+ }+ if got := parseComposePS([]byte("not json")); got != nil {+ t.Fatalf("garbage output states = %+v, want none", got)+ }+}++// The failure this exists for reached the reader as `Post /navigate: EOF` plus a+// `connection refused` on the next call, with nothing naming the dead container.+// A service that is gone must be reported as gone, and an out-of-memory kill+// must be named as one rather than left to be inferred.+func TestServiceDeathReportNamesTheDeadContainerAndOOM(t *testing.T) {+ oom := serviceState{Service: "pinchtab", State: "exited", ExitCode: 137}+ oom.oomKilled = true+ lines := serviceDeathReport([]serviceState{+ {Service: "fixtures", State: "running"},+ oom,+ })+ if len(lines) != 1 {+ t.Fatalf("report lines = %v, want only the dead service", lines)+ }+ if !strings.Contains(lines[0], "pinchtab") {+ t.Fatalf("line does not name the service: %q", lines[0])+ }+ if !strings.Contains(lines[0], "OOM-KILLED") {+ t.Fatalf("an out-of-memory kill must be named: %q", lines[0])+ }++ if got := describeServiceDeath(serviceState{Service: "pinchtab", State: "running"}); got != "" {+ t.Fatalf("a running service must produce no line, got %q", got)+ }++ sigkill := describeServiceDeath(serviceState{Service: "pinchtab", State: "exited", ExitCode: 137})+ if strings.Contains(sigkill, "OOM-KILLED") {+ t.Fatalf("exit 137 alone must not claim an OOM kill: %q", sigkill)+ }+ if !strings.Contains(sigkill, "SIGKILL") {+ t.Fatalf("exit 137 must name the signal: %q", sigkill)+ }++ ownExit := describeServiceDeath(serviceState{Service: "pinchtab", State: "exited", ExitCode: 2})+ if !strings.Contains(ownExit, "exit 2") || strings.Contains(ownExit, "SIGKILL") {+ t.Fatalf("a self-inflicted exit must report its own code: %q", ownExit)+ }+}++// A full Docker VM reaches the reader as an ordinary build failure. It must be+// named, and it must NOT be mistaken for the stale-cache condition, whose+// response is a --no-cache rebuild — the worst possible move on a full disk.+func TestOutOfDiskIsRecognisedAndNotTreatedAsStaleCache(t *testing.T) {+ diskLog := `#45 7.627 compile: writing output: write $WORK/b194/_pkg_.a: no space left on device+#45 ERROR: process "/bin/sh -c go build" did not complete successfully: exit code: 1`+ if !isOutOfDiskLog(diskLog) {+ t.Fatal("a build that ran out of disk was not recognised")+ }+ if isBuildKitCacheFailureLog(diskLog) {+ t.Fatal("an out-of-disk build must not be retried as a stale-cache build")+ }++ cacheLog := "failed to stat active key during commit"+ if isOutOfDiskLog(cacheLog) {+ t.Fatal("a stale-cache failure must not be reported as out of disk")+ }+ if !isBuildKitCacheFailureLog(cacheLog) {+ t.Fatal("the stale-cache condition stopped being recognised")+ }++ if isOutOfDiskLog("") {+ t.Fatal("an empty build log must not claim a full disk")+ }+ if !strings.Contains(outOfDiskRemedy, "docker builder prune") {+ t.Fatalf("the remedy must name a command that reclaims space: %q", outOfDiskRemedy)+ }+}++func TestDockerBinaryFollowsTheResolvedComposeCommand(t *testing.T) {+ if got := dockerBinary([]string{"docker", "compose"}); got != "docker" {+ t.Fatalf("docker binary = %q, want docker", got)+ }+ if got := dockerBinary([]string{"/usr/local/bin/docker", "compose"}); got != "/usr/local/bin/docker" {+ t.Fatalf("docker binary = %q, want the resolved path", got)+ }+ if got := dockerBinary([]string{"docker-compose"}); got != "docker" {+ t.Fatalf("docker binary = %q, want the docker fallback for compose v1", got)+ }+ if got := dockerBinary(nil); got != "docker" {+ t.Fatalf("docker binary = %q, want the docker fallback", got)+ }+}