Skip to content

High-performance browser automation bridge and multi-instance orchestrator with advanced stealth injection and real-time dashboard.

test: name the dead container, the OOM kill and the full Docker disk when an e2e stack starves (PIN-028)

Luigi Agosti139531d311eb
diff --git a/tests/tools/runner/internal/e2e/runner.go b/tests/tools/runner/internal/e2e/runner.go
index e75be08c..f07200eb 100644
--- a/tests/tools/runner/internal/e2e/runner.go
+++ b/tests/tools/runner/internal/e2e/runner.go
@@ -587,6 +587,7 @@ func (r *Runner) runSinglePlanWithCompose(plan suitePlan, composeFile string) in
r.printSuiteSummary(def, summary, duration)
if code != 0 {
r.dumpComposeFailure(composeFile, def)
+ r.reportServiceDeaths(composeFile, def.LogServices)
r.showFailureArtifacts(def, duration)
}
return code
@@ -639,10 +640,12 @@ func (r *Runner) bringUpSharedStack(composeFile string, services []string) int {
}
args = append(args, services...)
if code := r.runLoggedCommand("starting shared stack", stackOutput, r.composeArgs(composeFile, args...)); code != 0 {
+ r.reportServiceDeaths(composeFile, services)
return code
}
if err := r.assertStealthStatus(composeFile); err != nil {
_, _ = fmt.Fprintf(r.stderr, "e2e: pre-suite stealth assertion failed: %v\n", err)
+ r.reportServiceDeaths(composeFile, services)
return 1
}
return 0
@@ -658,6 +661,10 @@ func (r *Runner) buildSharedStackWithOverrides(composeFile string, includeOverri
if code == 0 {
return 0
}
+ if r.stackOutputIsOutOfDisk() {
+ _, _ = fmt.Fprintln(r.stdout, outOfDiskRemedy)
+ return code
+ }
if !r.stackOutputHasBuildKitCacheFailure() {
return code
}
@@ -685,11 +692,19 @@ func needsStockPinchtabImage(services []string) bool {
}
func (r *Runner) stackOutputHasBuildKitCacheFailure() bool {
+ return isBuildKitCacheFailureLog(r.stackOutputLog())
+}
+
+func (r *Runner) stackOutputIsOutOfDisk() bool {
+ return isOutOfDiskLog(r.stackOutputLog())
+}
+
+func (r *Runner) stackOutputLog() string {
data, err := os.ReadFile(filepath.Join(r.repoRoot, stackOutput))
if err != nil {
- return false
+ return ""
}
- return isBuildKitCacheFailureLog(string(data))
+ return string(data)
}
func isBuildKitCacheFailureLog(log string) bool {
diff --git a/tests/tools/runner/internal/e2e/stack_diagnostics.go b/tests/tools/runner/internal/e2e/stack_diagnostics.go
new file mode 100644
index 00000000..ab660ee9
--- /dev/null
+++ b/tests/tools/runner/internal/e2e/stack_diagnostics.go
@@ -0,0 +1,147 @@
+package e2e
+
+import (
+ "encoding/json"
+ "fmt"
+ "path/filepath"
+ "strings"
+)
+
+type serviceState struct {
+ ID string `json:"ID"`
+ Service string `json:"Service"`
+ State string `json:"State"`
+ ExitCode int `json:"ExitCode"`
+
+ oomKilled bool
+}
+
+// parseComposePS reads `compose ps --format json`, which emits one object per
+// line on modern Compose and a single array on older versions.
+func parseComposePS(raw []byte) []serviceState {
+ trimmed := strings.TrimSpace(string(raw))
+ if trimmed == "" {
+ return nil
+ }
+ if strings.HasPrefix(trimmed, "[") {
+ var states []serviceState
+ if err := json.Unmarshal([]byte(trimmed), &states); err != nil {
+ return nil
+ }
+ return states
+ }
+ var states []serviceState
+ for _, line := range strings.Split(trimmed, "\n") {
+ line = strings.TrimSpace(line)
+ if line == "" {
+ continue
+ }
+ var state serviceState
+ if err := json.Unmarshal([]byte(line), &state); err != nil {
+ continue
+ }
+ states = append(states, state)
+ }
+ return states
+}
+
+// describeServiceDeath reports the loud line for a service that is no longer
+// running, and "" for one that is. Exit 137 is SIGKILL, which is what a starved
+// container looks like from the outside, so the verdict is stated here rather
+// than left for a reader to infer from a client-side EOF.
+func describeServiceDeath(state serviceState) string {
+ if strings.EqualFold(state.State, "running") {
+ return ""
+ }
+ line := fmt.Sprintf(" container: %s is %s (exit %d)", state.Service, strings.ToLower(state.State), state.ExitCode)
+ switch {
+ case state.oomKilled:
+ return line + " — OOM-KILLED: the container ran out of memory. Raise the service limit or the Docker VM's memory."
+ case state.ExitCode == 137:
+ return line + " — SIGKILL: killed from outside the process; the kernel recorded no OOM, so suspect an external stop or a starved Docker VM."
+ case state.ExitCode != 0:
+ return line + " — the process exited on its own; read its captured log for the reason."
+ }
+ return line
+}
+
+func serviceDeathReport(states []serviceState) []string {
+ var lines []string
+ for _, state := range states {
+ if line := describeServiceDeath(state); line != "" {
+ lines = append(lines, line)
+ }
+ }
+ return lines
+}
+
+// reportServiceDeaths prints the state of every named service, so a run that
+// failed because a container died says so instead of leaving the reader with a
+// bare connection error. Both the suite-failure and the stack-bring-up paths
+// call it: a starved container dies during bring-up too, where the only symptom
+// is a refused connection on the readiness probe.
+func (r *Runner) reportServiceDeaths(composeFile string, services []string) {
+ if r.args.DryRun || len(services) == 0 {
+ return
+ }
+ args := append([]string{"ps", "-a", "--format", "json"}, services...)
+ raw, err := commandOutput(r.repoRoot, r.composeArgs(composeFile, args...))
+ if len(raw) == 0 {
+ if err != nil {
+ _, _ = fmt.Fprintf(r.stderr, "e2e: failed to read container state: %v\n", err)
+ }
+ return
+ }
+ states := parseComposePS(raw)
+ for i := range states {
+ states[i].oomKilled = r.containerWasOOMKilled(states[i].ID)
+ }
+ for _, state := range states {
+ line := describeServiceDeath(state)
+ if line == "" {
+ continue
+ }
+ _, _ = fmt.Fprintln(r.stdout, line)
+ _, _ = fmt.Fprintf(r.stdout, " log: %s\n", r.captureServiceLog(composeFile, state.Service))
+ }
+}
+
+// captureServiceLog writes the dead service's own log where the reader can find
+// it, since the stack-bring-up path has no suite artifacts to fall back on.
+func (r *Runner) captureServiceLog(composeFile, service string) string {
+ rel := filepath.Join(resultsDir, "logs-stack-"+service+".log")
+ if err := writeCommandOutput(filepath.Join(r.repoRoot, rel), r.repoRoot, r.composeArgs(composeFile, "logs", "--no-color", service)); err != nil {
+ return "unavailable: " + err.Error()
+ }
+ return rel
+}
+
+func (r *Runner) containerWasOOMKilled(containerID string) bool {
+ if strings.TrimSpace(containerID) == "" {
+ return false
+ }
+ out, err := commandOutput(r.repoRoot, []string{dockerBinary(r.compose), "inspect", containerID, "--format", "{{.State.OOMKilled}}"})
+ if err != nil {
+ return false
+ }
+ return strings.TrimSpace(string(out)) == "true"
+}
+
+// isOutOfDiskLog spots the Docker VM running out of disk, which reaches the
+// reader as an ordinary build failure with the reason buried in the build log.
+func isOutOfDiskLog(log string) bool {
+ return strings.Contains(strings.ToLower(log), "no space left on device")
+}
+
+const outOfDiskRemedy = " the Docker VM is OUT OF DISK — reclaim space before re-running: docker system df, then docker builder prune -f (and docker image prune -a)"
+
+func commandOutput(dir string, command []string) ([]byte, error) {
+ return execCommand(command, dir).Output()
+}
+
+func dockerBinary(compose []string) string {
+ if len(compose) > 0 && filepath.Base(compose[0]) == "docker" {
+ return compose[0]
+ }
+ return "docker"
+}
diff --git a/tests/tools/runner/internal/e2e/stack_diagnostics_test.go b/tests/tools/runner/internal/e2e/stack_diagnostics_test.go
new file mode 100644
index 00000000..3b8d7822
--- /dev/null
+++ b/tests/tools/runner/internal/e2e/stack_diagnostics_test.go
@@ -0,0 +1,114 @@
+package e2e
+
+import (
+ "strings"
+ "testing"
+)
+
+func TestParseComposePSReadsBothComposeFormats(t *testing.T) {
+ ndjson := []byte(`{"ID":"abc","Service":"pinchtab","State":"exited","ExitCode":137}
+{"ID":"def","Service":"fixtures","State":"running","ExitCode":0}`)
+ states := parseComposePS(ndjson)
+ if len(states) != 2 {
+ t.Fatalf("ndjson states = %d, want 2: %+v", len(states), states)
+ }
+ if states[0].Service != "pinchtab" || states[0].ExitCode != 137 || states[0].ID != "abc" {
+ t.Fatalf("first ndjson state = %+v", states[0])
+ }
+
+ array := []byte(`[{"ID":"abc","Service":"pinchtab","State":"exited","ExitCode":1}]`)
+ states = parseComposePS(array)
+ if len(states) != 1 || states[0].ExitCode != 1 {
+ t.Fatalf("array states = %+v", states)
+ }
+
+ if got := parseComposePS([]byte(" ")); got != nil {
+ t.Fatalf("blank output states = %+v, want none", got)
+ }
+ if got := parseComposePS([]byte("not json")); got != nil {
+ t.Fatalf("garbage output states = %+v, want none", got)
+ }
+}
+
+// The failure this exists for reached the reader as `Post /navigate: EOF` plus a
+// `connection refused` on the next call, with nothing naming the dead container.
+// A service that is gone must be reported as gone, and an out-of-memory kill
+// must be named as one rather than left to be inferred.
+func TestServiceDeathReportNamesTheDeadContainerAndOOM(t *testing.T) {
+ oom := serviceState{Service: "pinchtab", State: "exited", ExitCode: 137}
+ oom.oomKilled = true
+ lines := serviceDeathReport([]serviceState{
+ {Service: "fixtures", State: "running"},
+ oom,
+ })
+ if len(lines) != 1 {
+ t.Fatalf("report lines = %v, want only the dead service", lines)
+ }
+ if !strings.Contains(lines[0], "pinchtab") {
+ t.Fatalf("line does not name the service: %q", lines[0])
+ }
+ if !strings.Contains(lines[0], "OOM-KILLED") {
+ t.Fatalf("an out-of-memory kill must be named: %q", lines[0])
+ }
+
+ if got := describeServiceDeath(serviceState{Service: "pinchtab", State: "running"}); got != "" {
+ t.Fatalf("a running service must produce no line, got %q", got)
+ }
+
+ sigkill := describeServiceDeath(serviceState{Service: "pinchtab", State: "exited", ExitCode: 137})
+ if strings.Contains(sigkill, "OOM-KILLED") {
+ t.Fatalf("exit 137 alone must not claim an OOM kill: %q", sigkill)
+ }
+ if !strings.Contains(sigkill, "SIGKILL") {
+ t.Fatalf("exit 137 must name the signal: %q", sigkill)
+ }
+
+ ownExit := describeServiceDeath(serviceState{Service: "pinchtab", State: "exited", ExitCode: 2})
+ if !strings.Contains(ownExit, "exit 2") || strings.Contains(ownExit, "SIGKILL") {
+ t.Fatalf("a self-inflicted exit must report its own code: %q", ownExit)
+ }
+}
+
+// A full Docker VM reaches the reader as an ordinary build failure. It must be
+// named, and it must NOT be mistaken for the stale-cache condition, whose
+// response is a --no-cache rebuild — the worst possible move on a full disk.
+func TestOutOfDiskIsRecognisedAndNotTreatedAsStaleCache(t *testing.T) {
+ diskLog := `#45 7.627 compile: writing output: write $WORK/b194/_pkg_.a: no space left on device
+#45 ERROR: process "/bin/sh -c go build" did not complete successfully: exit code: 1`
+ if !isOutOfDiskLog(diskLog) {
+ t.Fatal("a build that ran out of disk was not recognised")
+ }
+ if isBuildKitCacheFailureLog(diskLog) {
+ t.Fatal("an out-of-disk build must not be retried as a stale-cache build")
+ }
+
+ cacheLog := "failed to stat active key during commit"
+ if isOutOfDiskLog(cacheLog) {
+ t.Fatal("a stale-cache failure must not be reported as out of disk")
+ }
+ if !isBuildKitCacheFailureLog(cacheLog) {
+ t.Fatal("the stale-cache condition stopped being recognised")
+ }
+
+ if isOutOfDiskLog("") {
+ t.Fatal("an empty build log must not claim a full disk")
+ }
+ if !strings.Contains(outOfDiskRemedy, "docker builder prune") {
+ t.Fatalf("the remedy must name a command that reclaims space: %q", outOfDiskRemedy)
+ }
+}
+
+func TestDockerBinaryFollowsTheResolvedComposeCommand(t *testing.T) {
+ if got := dockerBinary([]string{"docker", "compose"}); got != "docker" {
+ t.Fatalf("docker binary = %q, want docker", got)
+ }
+ if got := dockerBinary([]string{"/usr/local/bin/docker", "compose"}); got != "/usr/local/bin/docker" {
+ t.Fatalf("docker binary = %q, want the resolved path", got)
+ }
+ if got := dockerBinary([]string{"docker-compose"}); got != "docker" {
+ t.Fatalf("docker binary = %q, want the docker fallback for compose v1", got)
+ }
+ if got := dockerBinary(nil); got != "docker" {
+ t.Fatalf("docker binary = %q, want the docker fallback", got)
+ }
+}