Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package dockeragent
import (
"context"
"encoding/json"
"errors"
"io"
"net/http"
"net/http/httptest"
"net/netip"
"runtime"
"strings"
"sync"
"testing"
"time"
containertypes "github.com/moby/moby/api/types/container"
"github.com/moby/moby/api/types/network"
v1 "github.com/opencontainers/image-spec/specs-go/v1"
agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker"
"github.com/rs/zerolog"
)
func baseInspect() containertypes.InspectResponse {
state := &containertypes.State{Running: true}
hostConfig := &containertypes.HostConfig{}
return containertypes.InspectResponse{
Name: "/app",
Image: "sha256:old0000000000",
State: state,
RestartCount: 1,
HostConfig: hostConfig,
Config: &containertypes.Config{
Image: "nginx:latest",
},
NetworkSettings: &containertypes.NetworkSettings{
Networks: map[string]*network.EndpointSettings{
"net1": {Aliases: []string{"app"}},
},
},
}
}
func TestUpdateContainer_Errors(t *testing.T) {
logger := zerolog.Nop()
t.Run("inspect error", func(t *testing.T) {
agent := &Agent{
docker: &fakeDockerClient{
containerInspectFn: func(context.Context, string) (containertypes.InspectResponse, error) {
return containertypes.InspectResponse{}, errors.New("inspect failed")
},
},
logger: logger,
}
result := agent.updateContainerWithProgress(context.Background(), "container1", nil)
if result.Error == "" {
t.Fatal("expected error for inspect failure")
}
})
t.Run("pull error", func(t *testing.T) {
agent := &Agent{
docker: &fakeDockerClient{
containerInspectFn: func(context.Context, string) (containertypes.InspectResponse, error) {
return baseInspect(), nil
},
imagePullFn: func(context.Context, string, dockerImagePullOptions) (io.ReadCloser, error) {
return nil, errors.New("pull failed")
},
},
logger: logger,
}
result := agent.updateContainerWithProgress(context.Background(), "container1", nil)
if result.Error == "" {
t.Fatal("expected error for pull failure")
}
})
t.Run("stop error", func(t *testing.T) {
agent := &Agent{
docker: &fakeDockerClient{
containerInspectFn: func(context.Context, string) (containertypes.InspectResponse, error) {
return baseInspect(), nil
},
imagePullFn: func(context.Context, string, dockerImagePullOptions) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("{}")), nil
},
containerStopFn: func(context.Context, string, dockerContainerStopOptions) error {
return errors.New("stop failed")
},
},
logger: logger,
}
result := agent.updateContainerWithProgress(context.Background(), "container1", nil)
if result.Error == "" {
t.Fatal("expected error for stop failure")
}
})
t.Run("rename error", func(t *testing.T) {
startCalled := false
agent := &Agent{
docker: &fakeDockerClient{
containerInspectFn: func(context.Context, string) (containertypes.InspectResponse, error) {
return baseInspect(), nil
},
imagePullFn: func(context.Context, string, dockerImagePullOptions) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("{}")), nil
},
containerStopFn: func(context.Context, string, dockerContainerStopOptions) error {
return nil
},
containerRenameFn: func(context.Context, string, string) error {
return errors.New("rename failed")
},
containerStartFn: func(context.Context, string, dockerContainerStartOptions) error {
startCalled = true
return nil
},
},
logger: logger,
}
result := agent.updateContainerWithProgress(context.Background(), "container1", nil)
if result.Error == "" {
t.Fatal("expected error for rename failure")
}
if !startCalled {
t.Fatal("expected original container to be restarted")
}
})
t.Run("create error", func(t *testing.T) {
renameCalled := false
startCalled := false
agent := &Agent{
docker: &fakeDockerClient{
containerInspectFn: func(context.Context, string) (containertypes.InspectResponse, error) {
return baseInspect(), nil
},
imagePullFn: func(context.Context, string, dockerImagePullOptions) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("{}")), nil
},
containerStopFn: func(context.Context, string, dockerContainerStopOptions) error {
return nil
},
containerRenameFn: func(context.Context, string, string) error {
renameCalled = true
return nil
},
containerCreateFn: func(context.Context, *containertypes.Config, *containertypes.HostConfig, *network.NetworkingConfig, *v1.Platform, string) (containertypes.CreateResponse, error) {
return containertypes.CreateResponse{}, errors.New("create failed")
},
containerStartFn: func(context.Context, string, dockerContainerStartOptions) error {
startCalled = true
return nil
},
},
logger: logger,
}
result := agent.updateContainerWithProgress(context.Background(), "container1", nil)
if result.Error == "" {
t.Fatal("expected error for create failure")
}
if !renameCalled || !startCalled {
t.Fatal("expected rollback to rename and restart")
}
})
t.Run("start error", func(t *testing.T) {
removed := false
renamed := false
restarted := false
agent := &Agent{
docker: &fakeDockerClient{
containerInspectFn: func(context.Context, string) (containertypes.InspectResponse, error) {
return baseInspect(), nil
},
imagePullFn: func(context.Context, string, dockerImagePullOptions) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("{}")), nil
},
containerStopFn: func(context.Context, string, dockerContainerStopOptions) error {
return nil
},
containerCreateFn: func(context.Context, *containertypes.Config, *containertypes.HostConfig, *network.NetworkingConfig, *v1.Platform, string) (containertypes.CreateResponse, error) {
return containertypes.CreateResponse{ID: "new123"}, nil
},
containerStartFn: func(_ context.Context, id string, _ dockerContainerStartOptions) error {
if id == "new123" {
return errors.New("start failed")
}
restarted = true
return nil
},
containerRemoveFn: func(context.Context, string, dockerContainerRemoveOptions) error {
removed = true
return nil
},
containerRenameFn: func(context.Context, string, string) error {
renamed = true
return nil
},
},
logger: logger,
}
result := agent.updateContainerWithProgress(context.Background(), "container1", nil)
if result.Error == "" {
t.Fatal("expected error for start failure")
}
if !removed || !renamed || !restarted {
t.Fatal("expected rollback cleanup")
}
})
}
func TestUpdateContainer_Success(t *testing.T) {
logger := zerolog.Nop()
swap(t, &sleepFn, func(time.Duration) {})
swap(t, &nowFn, func() time.Time {
return time.Date(2024, 3, 1, 12, 0, 0, 0, time.UTC)
})
var (
mu sync.Mutex
cleanupCalls int
cleanupErr error
cleanupCh = make(chan struct{})
)
agent := &Agent{
docker: &fakeDockerClient{
containerInspectFn: func(_ context.Context, id string) (containertypes.InspectResponse, error) {
if id == "new123" {
inspect := baseInspect()
inspect.Image = "sha256:new0000000000"
return inspect, nil
}
return baseInspect(), nil
},
imagePullFn: func(context.Context, string, dockerImagePullOptions) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("{}")), nil
},
containerStopFn: func(context.Context, string, dockerContainerStopOptions) error {
return nil
},
containerRenameFn: func(context.Context, string, string) error {
return nil
},
containerCreateFn: func(context.Context, *containertypes.Config, *containertypes.HostConfig, *network.NetworkingConfig, *v1.Platform, string) (containertypes.CreateResponse, error) {
return containertypes.CreateResponse{ID: "new123"}, nil
},
containerStartFn: func(context.Context, string, dockerContainerStartOptions) error {
return nil
},
containerRemoveFn: func(context.Context, string, dockerContainerRemoveOptions) error {
mu.Lock()
cleanupCalls++
err := cleanupErr
mu.Unlock()
close(cleanupCh)
return err
},
},
logger: logger,
newTimerFn: immediateTimer,
}
result := agent.updateContainerWithProgress(context.Background(), "container1", nil)
if !result.Success {
t.Fatalf("expected success, got error %q", result.Error)
}
if !result.BackupCreated || result.BackupContainer == "" {
t.Fatalf("expected backup to be created")
}
if result.NewImageDigest == "" {
t.Fatalf("expected new image digest")
}
<-cleanupCh
mu.Lock()
if cleanupCalls != 1 {
t.Fatalf("expected cleanup to be called once, got %d", cleanupCalls)
}
mu.Unlock()
}
func TestUpdateContainer_StoppedContainerPreservesStoppedState(t *testing.T) {
logger := zerolog.Nop()
swap(t, &sleepFn, func(time.Duration) {})
swap(t, &nowFn, func() time.Time {
return time.Date(2024, 3, 1, 12, 0, 0, 0, time.UTC)
})
done := make(chan struct{})
agent := &Agent{
docker: &fakeDockerClient{
containerInspectFn: func(_ context.Context, id string) (containertypes.InspectResponse, error) {
if id == "new123" {
inspect := baseInspect()
if inspect.State != nil {
inspect.State.Running = false
}
inspect.Image = "sha256:new0000000000"
return inspect, nil
}
inspect := baseInspect()
if inspect.State != nil {
inspect.State.Running = false
}
return inspect, nil
},
imagePullFn: func(context.Context, string, dockerImagePullOptions) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("{}")), nil
},
containerStopFn: func(context.Context, string, dockerContainerStopOptions) error {
t.Fatalf("did not expect ContainerStop to be called for stopped container")
return nil
},
containerRenameFn: func(context.Context, string, string) error {
return nil
},
containerCreateFn: func(context.Context, *containertypes.Config, *containertypes.HostConfig, *network.NetworkingConfig, *v1.Platform, string) (containertypes.CreateResponse, error) {
return containertypes.CreateResponse{ID: "new123"}, nil
},
networkConnectFn: func(context.Context, string, string, *network.EndpointSettings) error {
return nil
},
containerStartFn: func(context.Context, string, dockerContainerStartOptions) error {
t.Fatalf("did not expect ContainerStart to be called for stopped container")
return nil
},
containerRemoveFn: func(context.Context, string, dockerContainerRemoveOptions) error {
close(done)
return nil
},
},
logger: logger,
newTimerFn: immediateTimer,
}
result := agent.updateContainerWithProgress(context.Background(), "container1", nil)
if !result.Success {
t.Fatalf("expected success, got error %q", result.Error)
}
if result.NewImageDigest == "" {
t.Fatalf("expected new image digest")
}
<-done
}
func TestUpdateContainer_CleanupError(t *testing.T) {
logger := zerolog.Nop()
swap(t, &sleepFn, func(time.Duration) {})
swap(t, &nowFn, func() time.Time {
return time.Date(2024, 3, 1, 12, 0, 0, 0, time.UTC)
})
cleanupErr := errors.New("cleanup failed")
done := make(chan struct{})
agent := &Agent{
docker: &fakeDockerClient{
containerInspectFn: func(_ context.Context, id string) (containertypes.InspectResponse, error) {
if id == "new123" {
inspect := baseInspect()
inspect.Image = "sha256:new0000000000"
return inspect, nil
}
return baseInspect(), nil
},
imagePullFn: func(context.Context, string, dockerImagePullOptions) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("{}")), nil
},
containerStopFn: func(context.Context, string, dockerContainerStopOptions) error {
return nil
},
containerRenameFn: func(context.Context, string, string) error {
return nil
},
containerCreateFn: func(context.Context, *containertypes.Config, *containertypes.HostConfig, *network.NetworkingConfig, *v1.Platform, string) (containertypes.CreateResponse, error) {
return containertypes.CreateResponse{ID: "new123"}, nil
},
containerStartFn: func(context.Context, string, dockerContainerStartOptions) error {
return nil
},
containerRemoveFn: func(context.Context, string, dockerContainerRemoveOptions) error {
close(done)
return cleanupErr
},
},
logger: logger,
newTimerFn: immediateTimer,
}
result := agent.updateContainerWithProgress(context.Background(), "container1", nil)
if !result.Success {
t.Fatalf("expected success, got error %q", result.Error)
}
<-done
}
func TestHandleUpdateContainerCommand(t *testing.T) {
logger := zerolog.Nop()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
agent := &Agent{
logger: logger,
hostID: "host1",
httpClients: map[bool]*http.Client{
false: server.Client(),
},
docker: &fakeDockerClient{
containerInspectFn: func(_ context.Context, id string) (containertypes.InspectResponse, error) {
inspect := baseInspect()
if id == "new123" {
inspect.Image = "sha256:new0000000000"
}
return inspect, nil
},
imagePullFn: func(context.Context, string, dockerImagePullOptions) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("{}")), nil
},
containerStopFn: func(context.Context, string, dockerContainerStopOptions) error {
return nil
},
containerRenameFn: func(context.Context, string, string) error {
return nil
},
containerCreateFn: func(context.Context, *containertypes.Config, *containertypes.HostConfig, *network.NetworkingConfig, *v1.Platform, string) (containertypes.CreateResponse, error) {
return containertypes.CreateResponse{ID: "new123"}, nil
},
containerStartFn: func(context.Context, string, dockerContainerStartOptions) error {
return nil
},
},
}
command := agentsdocker.Command{
ID: "cmd1",
Type: agentsdocker.CommandTypeUpdateContainer,
Payload: map[string]any{
"containerId": "container1",
},
}
if err := agent.handleUpdateContainerCommand(context.Background(), TargetConfig{URL: server.URL}, command); err != nil {
t.Fatalf("unexpected error: %v", err)
}
t.Run("missing container id", func(t *testing.T) {
if err := agent.handleUpdateContainerCommand(context.Background(), TargetConfig{URL: server.URL}, agentsdocker.Command{ID: "cmd2"}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
})
t.Run("missing container id ack failure", func(t *testing.T) {
if err := agent.handleUpdateContainerCommand(context.Background(), TargetConfig{URL: "http://example.com/\x7f"}, agentsdocker.Command{ID: "cmd2c"}); err != nil {
t.Fatalf("unexpected error: %v", err)
}
})
t.Run("container id wrong type", func(t *testing.T) {
cmd := agentsdocker.Command{
ID: "cmd2b",
Type: agentsdocker.CommandTypeUpdateContainer,
Payload: map[string]any{
"containerId": 123,
},
}
if err := agent.handleUpdateContainerCommand(context.Background(), TargetConfig{URL: server.URL}, cmd); err != nil {
t.Fatalf("unexpected error: %v", err)
}
})
t.Run("container id unmarshalable payload", func(t *testing.T) {
cmd := agentsdocker.Command{
ID: "cmd2d",
Type: agentsdocker.CommandTypeUpdateContainer,
Payload: map[string]any{
"containerId": make(chan int),
},
}
if err := agent.handleUpdateContainerCommand(context.Background(), TargetConfig{URL: server.URL}, cmd); err != nil {
t.Fatalf("unexpected error: %v", err)
}
})
t.Run("ack error stops early", func(t *testing.T) {
badTarget := TargetConfig{URL: "http://example.com/\x7f"}
agent.hostID = "host1"
cmd := agentsdocker.Command{
ID: "cmd3",
Type: agentsdocker.CommandTypeUpdateContainer,
Payload: map[string]any{
"containerId": "container1",
},
}
if err := agent.handleUpdateContainerCommand(context.Background(), badTarget, cmd); err != nil {
t.Fatalf("unexpected error: %v", err)
}
})
t.Run("update failure sends failed ack", func(t *testing.T) {
var ack agentsdocker.CommandAck
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
body, _ := io.ReadAll(r.Body)
_ = json.Unmarshal(body, &ack)
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
agent := &Agent{
logger: zerolog.Nop(),
hostID: "host1",
httpClients: map[bool]*http.Client{
false: server.Client(),
},
docker: &fakeDockerClient{
containerInspectFn: func(context.Context, string) (containertypes.InspectResponse, error) {
return containertypes.InspectResponse{}, errors.New("inspect failed")
},
},
}
cmd := agentsdocker.Command{
ID: "cmd4",
Type: agentsdocker.CommandTypeUpdateContainer,
Payload: map[string]any{
"containerId": "container1",
},
}
if err := agent.handleUpdateContainerCommand(context.Background(), TargetConfig{URL: server.URL, Token: "token"}, cmd); err != nil {
t.Fatalf("unexpected error: %v", err)
}
if ack.Status != agentsdocker.CommandStatusFailed {
t.Fatalf("expected failed status, got %q", ack.Status)
}
})
t.Run("completion ack error", func(t *testing.T) {
calls := 0
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
calls++
if calls == 2 {
w.WriteHeader(http.StatusInternalServerError)
return
}
w.WriteHeader(http.StatusOK)
}))
defer server.Close()
agent := &Agent{
logger: zerolog.Nop(),
hostID: "host1",
httpClients: map[bool]*http.Client{
false: server.Client(),
},
docker: &fakeDockerClient{
containerInspectFn: func(_ context.Context, id string) (containertypes.InspectResponse, error) {
inspect := baseInspect()
if id == "new123" {
inspect.Image = "sha256:new0000000000"
}
return inspect, nil
},
imagePullFn: func(context.Context, string, dockerImagePullOptions) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("{}")), nil
},
containerStopFn: func(context.Context, string, dockerContainerStopOptions) error {
return nil
},
containerRenameFn: func(context.Context, string, string) error {
return nil
},
containerCreateFn: func(context.Context, *containertypes.Config, *containertypes.HostConfig, *network.NetworkingConfig, *v1.Platform, string) (containertypes.CreateResponse, error) {
return containertypes.CreateResponse{ID: "new123"}, nil
},
containerStartFn: func(context.Context, string, dockerContainerStartOptions) error {
return nil
},
},
}
cmd := agentsdocker.Command{
ID: "cmd5",
Type: agentsdocker.CommandTypeUpdateContainer,
Payload: map[string]any{
"containerId": "container1",
},
}
if err := agent.handleUpdateContainerCommand(context.Background(), TargetConfig{URL: server.URL, Token: "token"}, cmd); err != nil {
t.Fatalf("unexpected error: %v", err)
}
})
}
func TestUpdateContainer_SharedNamespaceCreateConfig(t *testing.T) {
logger := zerolog.Nop()
swap(t, &sleepFn, func(time.Duration) {})
swap(t, &nowFn, func() time.Time {
return time.Date(2024, 3, 1, 12, 0, 0, 0, time.UTC)
})
newInspect := func(networkMode string) containertypes.InspectResponse {
inspect := baseInspect()
inspect.HostConfig.NetworkMode = containertypes.NetworkMode(networkMode)
inspect.Config.Hostname = "nstest"
inspect.Config.Domainname = "lan"
inspect.Config.ExposedPorts = network.PortSet{network.MustParsePort("8080/tcp"): {}}
inspect.HostConfig.PortBindings = network.PortMap{network.MustParsePort("8080/tcp"): {{HostPort: "8080"}}}
inspect.HostConfig.PublishAllPorts = true
inspect.HostConfig.Links = []string{"db:db"}
inspect.HostConfig.DNS = []netip.Addr{netip.MustParseAddr("192.168.1.53")}
inspect.HostConfig.DNSOptions = []string{"ndots:1"}
inspect.HostConfig.DNSSearch = []string{"lan"}
inspect.HostConfig.ExtraHosts = []string{"gw:192.168.1.1"}
return inspect
}
runUpdate := func(t *testing.T, networkMode string) (*containertypes.Config, *containertypes.HostConfig) {
t.Helper()
var (
mu sync.Mutex
createdConfig *containertypes.Config
createdHost *containertypes.HostConfig
cleanupCh = make(chan struct{})
cleanupCloseOnce sync.Once
)
agent := &Agent{
docker: &fakeDockerClient{
containerInspectFn: func(_ context.Context, id string) (containertypes.InspectResponse, error) {
inspect := newInspect(networkMode)
if id == "new123" {
inspect.Image = "sha256:new0000000000"
}
return inspect, nil
},
imagePullFn: func(context.Context, string, dockerImagePullOptions) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("{}")), nil
},
containerStopFn: func(context.Context, string, dockerContainerStopOptions) error {
return nil
},
containerRenameFn: func(context.Context, string, string) error {
return nil
},
containerCreateFn: func(_ context.Context, config *containertypes.Config, hostConfig *containertypes.HostConfig, _ *network.NetworkingConfig, _ *v1.Platform, _ string) (containertypes.CreateResponse, error) {
mu.Lock()
createdConfig = config
createdHost = hostConfig
mu.Unlock()
return containertypes.CreateResponse{ID: "new123"}, nil
},
networkConnectFn: func(context.Context, string, string, *network.EndpointSettings) error {
return nil
},
containerStartFn: func(context.Context, string, dockerContainerStartOptions) error {
return nil
},
containerRemoveFn: func(context.Context, string, dockerContainerRemoveOptions) error {
cleanupCloseOnce.Do(func() { close(cleanupCh) })
return nil
},
},
logger: logger,
newTimerFn: immediateTimer,
}
result := agent.updateContainerWithProgress(context.Background(), "container1", nil)
if !result.Success {
t.Fatalf("expected success, got error %q", result.Error)
}
<-cleanupCh
mu.Lock()
defer mu.Unlock()
if createdConfig == nil || createdHost == nil {
t.Fatal("expected ContainerCreate to be called")
}
return createdConfig, createdHost
}
t.Run("container network mode strips namespace-owned settings", func(t *testing.T) {
config, hostConfig := runUpdate(t, "container:owner123")
if config.Hostname != "" || config.Domainname != "" {
t.Fatalf("expected hostname/domainname cleared, got %q/%q", config.Hostname, config.Domainname)
}
if len(config.ExposedPorts) != 0 {
t.Fatalf("expected exposed ports cleared, got %v", config.ExposedPorts)
}
if len(hostConfig.PortBindings) != 0 || hostConfig.PublishAllPorts {
t.Fatalf("expected port bindings cleared, got %v publishAll=%v", hostConfig.PortBindings, hostConfig.PublishAllPorts)
}
if len(hostConfig.Links) != 0 || len(hostConfig.DNS) != 0 || len(hostConfig.DNSOptions) != 0 || len(hostConfig.DNSSearch) != 0 || len(hostConfig.ExtraHosts) != 0 {
t.Fatal("expected links/dns/extra hosts cleared")
}
if got := string(hostConfig.NetworkMode); got != "container:owner123" {
t.Fatalf("expected network mode preserved, got %q", got)
}
})
t.Run("host network mode strips hostname only", func(t *testing.T) {
if runtime.GOOS == "windows" {
t.Skip("Docker does not support host network mode on Windows")
}
config, hostConfig := runUpdate(t, "host")
if config.Hostname != "" || config.Domainname != "" {
t.Fatalf("expected hostname/domainname cleared, got %q/%q", config.Hostname, config.Domainname)
}
if len(config.ExposedPorts) != 1 || len(hostConfig.PortBindings) != 1 || !hostConfig.PublishAllPorts {
t.Fatal("expected port settings preserved for host network mode")
}
if len(hostConfig.DNS) != 1 || len(hostConfig.ExtraHosts) != 1 {
t.Fatal("expected dns/extra hosts preserved for host network mode")
}
})
t.Run("bridge network mode left untouched", func(t *testing.T) {
config, hostConfig := runUpdate(t, "bridge")
if config.Hostname != "nstest" || config.Domainname != "lan" {
t.Fatalf("expected hostname/domainname preserved, got %q/%q", config.Hostname, config.Domainname)
}
if len(config.ExposedPorts) != 1 || len(hostConfig.PortBindings) != 1 {
t.Fatal("expected port settings preserved for bridge network mode")
}
})
}
// TestDecodeUpdateContainerPayloadUsesReceiverMarshaller pins the update-command
// decode path to the receiver's own JSON seam. decodeUpdateContainerPayload was
// a package function reading a package-level jsonMarshalFn; a test swapping that
// global raced with async goroutines leaked from earlier tests. It is now an
// Agent method reading a per-Agent field injected at construction, so a failure
// injected into one Agent can never be observed by another.
func TestDecodeUpdateContainerPayloadUsesReceiverMarshaller(t *testing.T) {
t.Run("injected marshaller fails only its own Agent", func(t *testing.T) {
marshalErr := errors.New("injected update marshal failure")
failing := &Agent{
jsonMarshalFn: func(any) ([]byte, error) {
return nil, marshalErr
},
}
if _, err := failing.decodeUpdateContainerPayload(map[string]any{"containerId": "abc"}); err == nil {
t.Fatal("expected the injected marshaller to fail the decode")
} else if !errors.Is(err, marshalErr) {
t.Fatalf("decode error %v does not wrap the injected failure; the decode is not using a.jsonMarshal", err)
} else if !strings.Contains(err.Error(), "marshal update command payload") {
t.Fatalf("decode error %v lost its marshal context", err)
}
plain := &Agent{}
decoded, err := plain.decodeUpdateContainerPayload(map[string]any{"containerId": "abc"})
if err != nil {
t.Fatalf("sibling Agent must not observe the injected marshaller: %v", err)
}
if decoded.ContainerID != "abc" {
t.Fatalf("sibling Agent decoded %q, want %q", decoded.ContainerID, "abc")
}
})
t.Run("concurrent Agents keep their own marshaller", func(t *testing.T) {
marshalErr := errors.New("injected update marshal failure")
failing := &Agent{
jsonMarshalFn: func(any) ([]byte, error) {
return nil, marshalErr
},
}
plain := &Agent{}
var wg sync.WaitGroup
problems := make(chan string, 64)
for range 16 {
wg.Add(2)
go func() {
defer wg.Done()
if _, err := failing.decodeUpdateContainerPayload(map[string]any{"containerId": "abc"}); !errors.Is(err, marshalErr) {
problems <- "injected Agent lost its marshaller"
}
}()
go func() {
defer wg.Done()
decoded, err := plain.decodeUpdateContainerPayload(map[string]any{"containerId": "abc"})
if err != nil || decoded.ContainerID != "abc" {
problems <- "sibling Agent observed the injected marshaller"
}
}()
}
wg.Wait()
close(problems)
for problem := range problems {
t.Error(problem)
}
})
t.Run("decodeUpdateContainerPayload is an Agent method", func(t *testing.T) {
// A compile-time pin: if the decode is ever hoisted back to a package
// function reading a package global, this assignment stops building.
var decode func(*Agent, map[string]any) (updateContainerCommandPayload, error) = (*Agent).decodeUpdateContainerPayload
decoded, err := decode(&Agent{}, map[string]any{"containerId": " abc "})
if err != nil {
t.Fatalf("unexpected error: %v", err)
}
if decoded.ContainerID != "abc" {
t.Fatalf("decoded %q, want %q", decoded.ContainerID, "abc")
}
})
}
func TestUpdateContainer_WaitsForReplacementHealthToSettle(t *testing.T) {
swap(t, &sleepFn, func(time.Duration) {})
maxPolls := int(dockerUpdateHealthSettleCap / dockerUpdateHealthPollInterval)
for _, tc := range []struct {
name string
sequence []string // replacement health per inspect; the last value repeats
after string // once the sequence is exhausted: "exited", "restarting" or "error"
wantOK bool
wantPolls int
}{
{name: "starting then healthy", sequence: []string{"starting", "starting", "healthy"}, wantOK: true, wantPolls: 3},
{name: "starting then unhealthy rolls back", sequence: []string{"starting", "unhealthy"}, wantPolls: 2},
{name: "exits while starting rolls back", sequence: []string{"starting", "starting"}, after: "exited", wantPolls: 3},
{name: "restarting while starting rolls back", sequence: []string{"starting"}, after: "restarting", wantPolls: 2},
{name: "inspect failure while starting rolls back", sequence: []string{"starting"}, after: "error", wantPolls: 2},
{name: "still starting at the cap is kept", sequence: []string{"starting"}, wantOK: true, wantPolls: maxPolls + 1},
{name: "no health check does not wait", sequence: []string{""}, wantOK: true, wantPolls: 1},
} {
t.Run(tc.name, func(t *testing.T) {
var (
mu sync.Mutex
polls int
removed []string
cleanupCh = make(chan struct{}, 1)
)
agent := &Agent{
docker: &fakeDockerClient{
containerInspectFn: func(_ context.Context, id string) (containertypes.InspectResponse, error) {
if id != "new123" {
return baseInspect(), nil
}
mu.Lock()
defer mu.Unlock()
inspect := baseInspect()
inspect.Image = "sha256:new0000000000"
health := tc.sequence[min(polls, len(tc.sequence)-1)]
exhausted := polls >= len(tc.sequence)
polls++
switch {
case exhausted && tc.after == "error":
return containertypes.InspectResponse{}, errors.New("daemon unavailable")
case exhausted && tc.after == "exited":
inspect.State = &containertypes.State{Running: false, ExitCode: 1}
case exhausted && tc.after == "restarting":
inspect.State = &containertypes.State{Running: true, Restarting: true, ExitCode: 1, Health: &containertypes.Health{Status: containertypes.Starting}}
case health != "":
inspect.State = &containertypes.State{Running: true, Health: &containertypes.Health{Status: containertypes.HealthStatus(health)}}
}
return inspect, nil
},
imagePullFn: func(context.Context, string, dockerImagePullOptions) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("{}")), nil
},
containerStopFn: func(context.Context, string, dockerContainerStopOptions) error { return nil },
containerRenameFn: func(context.Context, string, string) error { return nil },
containerCreateFn: func(context.Context, *containertypes.Config, *containertypes.HostConfig, *network.NetworkingConfig, *v1.Platform, string) (containertypes.CreateResponse, error) {
return containertypes.CreateResponse{ID: "new123"}, nil
},
containerStartFn: func(context.Context, string, dockerContainerStartOptions) error { return nil },
containerRemoveFn: func(_ context.Context, id string, _ dockerContainerRemoveOptions) error {
mu.Lock()
removed = append(removed, id)
mu.Unlock()
select {
case cleanupCh <- struct{}{}:
default:
}
return nil
},
},
logger: zerolog.Nop(),
newTimerFn: immediateTimer,
}
result := agent.updateContainerWithProgress(context.Background(), "container1", nil)
if result.Success != tc.wantOK {
t.Fatalf("success = %t (error %q), want %t", result.Success, result.Error, tc.wantOK)
}
if tc.wantOK {
select { // the backup is removed asynchronously after success
case <-cleanupCh:
case <-time.After(5 * time.Second):
t.Fatal("backup cleanup did not run")
}
}
mu.Lock()
defer mu.Unlock()
if polls != tc.wantPolls {
t.Fatalf("replacement inspected %d times, want %d", polls, tc.wantPolls)
}
if tc.wantOK {
if result.RollbackAttempted {
t.Fatal("a healthy or still-starting replacement must not be rolled back")
}
return
}
if !result.RollbackAttempted || len(removed) == 0 || removed[0] != "new123" {
t.Fatalf("expected rollback to remove the replacement, removed=%v rollback=%t", removed, result.RollbackAttempted)
}
})
}
}
func TestUpdateContainer_SettleCapEndsHungHealthInspect(t *testing.T) {
swap(t, &sleepFn, func(time.Duration) {})
var (
mu sync.Mutex
polls int
rolled bool
)
agent := &Agent{
docker: &fakeDockerClient{
containerInspectFn: func(ctx context.Context, id string) (containertypes.InspectResponse, error) {
if id != "new123" {
return baseInspect(), nil
}
mu.Lock()
polls++
first := polls == 1
mu.Unlock()
if !first {
<-ctx.Done() // a daemon that stops answering mid-wait
return containertypes.InspectResponse{}, ctx.Err()
}
inspect := baseInspect()
inspect.State = &containertypes.State{Running: true, Health: &containertypes.Health{Status: containertypes.Starting}}
return inspect, nil
},
imagePullFn: func(context.Context, string, dockerImagePullOptions) (io.ReadCloser, error) {
return io.NopCloser(strings.NewReader("{}")), nil
},
containerStopFn: func(context.Context, string, dockerContainerStopOptions) error { return nil },
containerRenameFn: func(context.Context, string, string) error { return nil },
containerCreateFn: func(context.Context, *containertypes.Config, *containertypes.HostConfig, *network.NetworkingConfig, *v1.Platform, string) (containertypes.CreateResponse, error) {
return containertypes.CreateResponse{ID: "new123"}, nil
},
containerStartFn: func(context.Context, string, dockerContainerStartOptions) error { return nil },
containerRemoveFn: func(_ context.Context, id string, _ dockerContainerRemoveOptions) error {
mu.Lock()
rolled = rolled || id == "new123"
mu.Unlock()
return nil
},
},
logger: zerolog.Nop(),
newTimerFn: immediateTimer,
healthSettleCap: 4 * time.Second,
}
started := time.Now()
result := agent.updateContainerWithProgress(context.Background(), "container1", nil)
if elapsed := time.Since(started); elapsed > 30*time.Second {
t.Fatalf("hung inspect held the update for %s, want it bounded by the settle cap", elapsed)
}
if !result.Success || result.RollbackAttempted {
t.Fatalf("reaching the settle cap mid-inspect must keep the replacement: success=%t rollback=%t error=%q", result.Success, result.RollbackAttempted, result.Error)
}
mu.Lock()
defer mu.Unlock()
if rolled {
t.Fatal("the replacement was removed")
}
}