Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.
Go aialertsdashboarddockerdocker-monitoringgohomelabinfrastructure-monitoringkubernetesmonitoringmspproxmoxproxmox-backup-serverproxmox-mail-gatewayproxmox-veself-hostedtruenastypescriptvspherewebhooks
package hostagent
import (
"context"
"encoding/json"
"errors"
"fmt"
"net"
"os"
"os/exec"
"regexp"
"strconv"
"strings"
"time"
"github.com/rcourtman/pulse-go-rewrite/internal/agentexec"
)
var dockerContextNamePattern = regexp.MustCompile(`^[a-zA-Z0-9._-]{1,128}$`)
const dockerLifecycleHealthPollInterval = 2 * time.Second
type dockerLifecycleManager interface {
Apply(context.Context, agentexec.DockerContainerLifecyclePayload) agentexec.DockerContainerLifecycleResultPayload
Preflight(context.Context, agentexec.DockerContainerLifecyclePayload) (bool, string)
}
func (m *localDockerLifecycleManager) Observe(ctx context.Context, runtime, containerID string) (agentexec.DockerContainerObservationSnapshot, error) {
snapshot, err := m.inspect(ctx, runtime, containerID)
if err != nil {
return agentexec.DockerContainerObservationSnapshot{}, err
}
return agentexec.DockerContainerObservationSnapshot{
ContainerID: snapshot.ContainerID, State: snapshot.State, Running: snapshot.Running, Health: snapshot.Health,
StartedAt: snapshot.StartedAt, RestartCount: snapshot.RestartCount, ObservedAt: snapshot.ObservedAt,
}, nil
}
// DockerContainerLifecycleOperator is the narrow bridge from the host command
// channel to the unified agent's connected Docker / Podman module. It keeps
// lifecycle execution on the daemon API and avoids requiring a second runtime
// client executable inside the agent image.
type DockerContainerLifecycleOperator interface {
InspectDockerContainerLifecycle(context.Context, string, string) (agentexec.DockerContainerLifecycleSnapshot, error)
MutateDockerContainerLifecycle(context.Context, string, string, string) error
}
type dockerLifecycleCommandRunner func(context.Context, string, ...string) ([]byte, error)
type localDockerLifecycleManager struct {
run dockerLifecycleCommandRunner
operator DockerContainerLifecycleOperator
now func() time.Time
healthPollInterval time.Duration
}
func newLocalDockerLifecycleManager(operators ...DockerContainerLifecycleOperator) *localDockerLifecycleManager {
manager := &localDockerLifecycleManager{
run: func(ctx context.Context, name string, args ...string) ([]byte, error) {
return exec.CommandContext(ctx, name, args...).CombinedOutput()
},
now: time.Now,
}
if len(operators) > 0 {
manager.operator = operators[0]
}
return manager
}
func (m *localDockerLifecycleManager) Apply(ctx context.Context, req agentexec.DockerContainerLifecyclePayload) (result agentexec.DockerContainerLifecycleResultPayload) {
started := m.currentTime()
result = agentexec.DockerContainerLifecycleResultPayload{
RequestID: req.RequestID, ActionID: req.ActionID, Operation: req.Operation,
OperationVersion: req.OperationVersion, RequestDigest: req.RequestDigest, ContainerID: req.ContainerID,
ExecutionPhase: agentexec.DockerContainerPhasePreflight,
}
defer func() { result.Duration = time.Since(started).Milliseconds() }()
if err := agentexec.ValidateDockerContainerLifecyclePayload(&req); err != nil {
result.ReasonCode = agentexec.ActionRefusalContractInvalid
result.Error = "typed container lifecycle preflight refused"
return result
}
before, err := m.inspect(ctx, req.Runtime, req.ContainerID)
result.Before = before
if err != nil {
result.ReasonCode = agentexec.ActionRefusalTargetInspectionUnavailable
result.Error = "container preflight inspect unavailable"
return result
}
if !dockerLifecycleBeforeMatches(req, before) {
result.ReasonCode = agentexec.ActionRefusalTargetStateChanged
result.Error = "container preflight state no longer matches the request"
return result
}
result.ExecutionPhase = agentexec.DockerContainerPhaseMutate
result.MutationStarted = true
verb := strings.TrimSuffix(req.Operation, "_container")
mutationErr := m.mutate(ctx, req.Runtime, verb, req.ContainerID)
if mutationErr != nil {
// A lost acknowledgement does not prove that the daemon did nothing.
// Reconcile using reads under the original deadline, without retrying
// the mutation or claiming its execution was acknowledged.
result.Error = dockerLifecycleMutationError(mutationErr)
if ctx.Err() != nil {
return result
}
} else {
result.MutationCompleted = true
}
result.ExecutionPhase = agentexec.DockerContainerPhaseVerify
after, err := m.inspect(ctx, req.Runtime, req.ContainerID)
result.After = after
if err != nil {
if mutationErr == nil {
result.Error = "container postcondition inspect unavailable"
}
return result
}
result.ReadbackRan = true
if dockerLifecyclePostcondition(req, before, after) {
if result.MutationCompleted {
result.ExecutionPhase = agentexec.DockerContainerPhaseComplete
}
return result
}
if dockerLifecycleHealthPending(req, before, after) {
verifyCtx, cancel := context.WithTimeout(ctx, time.Duration(req.Timeout)*time.Second)
defer cancel()
for dockerLifecycleHealthPending(req, before, after) {
if err := m.waitForDockerLifecycleHealth(verifyCtx); err != nil {
break
}
after, err = m.inspect(verifyCtx, req.Runtime, req.ContainerID)
if err != nil {
if mutationErr == nil {
result.Error = "container postcondition inspect unavailable"
}
return result
}
result.After = after
if dockerLifecyclePostcondition(req, before, after) {
if result.MutationCompleted {
result.ExecutionPhase = agentexec.DockerContainerPhaseComplete
}
return result
}
}
}
if mutationErr == nil {
result.Error = "container postcondition did not reach the requested state"
}
return result
}
func dockerLifecycleMutationError(err error) string {
var timeout net.Error
switch {
case errors.Is(err, context.DeadlineExceeded), errors.As(err, &timeout) && timeout.Timeout():
return "container lifecycle mutation acknowledgement timed out"
case errors.Is(err, context.Canceled):
return "container lifecycle mutation acknowledgement canceled"
default:
return "container lifecycle mutation acknowledgement unavailable"
}
}
func (m *localDockerLifecycleManager) Preflight(ctx context.Context, req agentexec.DockerContainerLifecyclePayload) (bool, string) {
if err := agentexec.ValidateDockerContainerLifecyclePayload(&req); err != nil {
return false, agentexec.ActionRefusalContractInvalid
}
before, err := m.inspect(ctx, req.Runtime, req.ContainerID)
if err != nil {
return false, agentexec.ActionRefusalTargetInspectionUnavailable
}
if !dockerLifecycleBeforeMatches(req, before) {
return false, agentexec.ActionRefusalTargetStateChanged
}
return true, ""
}
func (m *localDockerLifecycleManager) command(ctx context.Context, runtime string, args ...string) ([]byte, error) {
if m == nil || m.run == nil {
return nil, fmt.Errorf("container runtime command runner unavailable")
}
if runtime != "docker" && runtime != "podman" {
return nil, fmt.Errorf("unsupported container runtime")
}
if runtime == "docker" {
if dockerContext := strings.TrimSpace(os.Getenv("DOCKER_CONTEXT")); dockerContext != "" {
if !dockerContextNamePattern.MatchString(dockerContext) {
return nil, fmt.Errorf("invalid Docker context name")
}
args = append([]string{"--context", dockerContext}, args...)
}
}
return m.run(ctx, runtime, args...)
}
func (m *localDockerLifecycleManager) mutate(ctx context.Context, runtime, operation, containerID string) error {
if m != nil && m.operator != nil {
return m.operator.MutateDockerContainerLifecycle(ctx, runtime, operation, containerID)
}
_, err := m.command(ctx, runtime, operation, containerID)
return err
}
func (m *localDockerLifecycleManager) inspect(ctx context.Context, runtime, containerID string) (agentexec.DockerContainerLifecycleSnapshot, error) {
if m != nil && m.operator != nil {
return m.operator.InspectDockerContainerLifecycle(ctx, runtime, containerID)
}
const format = `{{json .State}}`
raw, err := m.command(ctx, runtime, "inspect", "--format", format, containerID)
if err != nil {
return agentexec.DockerContainerLifecycleSnapshot{}, err
}
var state struct {
Status string `json:"Status"`
Running bool `json:"Running"`
StartedAt string `json:"StartedAt"`
RestartCount int `json:"RestartCount"`
Health *struct {
Status string `json:"Status"`
} `json:"Health"`
}
if err := json.Unmarshal([]byte(strings.TrimSpace(string(raw))), &state); err != nil {
return agentexec.DockerContainerLifecycleSnapshot{}, fmt.Errorf("decode container state: %w", err)
}
startedAt := time.Time{}
if value := strings.TrimSpace(state.StartedAt); value != "" && !strings.HasPrefix(value, "0001-") {
startedAt, err = time.Parse(time.RFC3339Nano, value)
if err != nil {
return agentexec.DockerContainerLifecycleSnapshot{}, fmt.Errorf("decode container start time: %w", err)
}
}
// Docker exposes RestartCount at the inspect root. Query it separately to
// keep the state envelope bounded and avoid persisting labels or config.
restartRaw, restartErr := m.command(ctx, runtime, "inspect", "--format", `{{.RestartCount}}`, containerID)
if restartErr == nil {
if count, parseErr := strconv.Atoi(strings.TrimSpace(string(restartRaw))); parseErr == nil && count >= 0 {
state.RestartCount = count
}
}
health := agentexec.DockerContainerHealthNone
if state.Health != nil {
health = strings.ToLower(strings.TrimSpace(state.Health.Status))
if !agentexec.IsDockerContainerHealth(health) {
return agentexec.DockerContainerLifecycleSnapshot{}, fmt.Errorf("container inspect returned unsupported health status")
}
}
return agentexec.DockerContainerLifecycleSnapshot{
ContainerID: strings.ToLower(containerID), State: strings.ToLower(strings.TrimSpace(state.Status)), Running: state.Running,
Health: health, StartedAt: startedAt.UTC(), RestartCount: state.RestartCount, ObservedAt: m.currentTime(),
}, nil
}
func (m *localDockerLifecycleManager) currentTime() time.Time {
if m != nil && m.now != nil {
return m.now().UTC()
}
return time.Now().UTC()
}
func dockerLifecycleBeforeMatches(req agentexec.DockerContainerLifecyclePayload, before agentexec.DockerContainerLifecycleSnapshot) bool {
if !strings.EqualFold(req.ContainerID, before.ContainerID) || !strings.EqualFold(req.ExpectedState, before.State) {
return false
}
if !req.ExpectedStartedAt.IsZero() && !req.ExpectedStartedAt.Equal(before.StartedAt) {
return false
}
return true
}
func dockerLifecyclePostcondition(req agentexec.DockerContainerLifecyclePayload, before, after agentexec.DockerContainerLifecycleSnapshot) bool {
if !strings.EqualFold(req.ContainerID, after.ContainerID) {
return false
}
switch req.Operation {
case agentexec.DockerContainerOperationStart:
return after.Running && after.State == "running" && agentexec.DockerContainerHealthAllowsVerifiedRunningState(after.Health)
case agentexec.DockerContainerOperationStop:
return !after.Running && after.State == "exited"
case agentexec.DockerContainerOperationRestart:
return after.Running && after.State == "running" && !after.StartedAt.IsZero() && after.StartedAt.After(before.StartedAt) && agentexec.DockerContainerHealthAllowsVerifiedRunningState(after.Health)
default:
return false
}
}
func dockerLifecycleHealthPending(req agentexec.DockerContainerLifecyclePayload, before, after agentexec.DockerContainerLifecycleSnapshot) bool {
health := strings.ToLower(strings.TrimSpace(after.Health))
if health != agentexec.DockerContainerHealthStarting && health != agentexec.DockerContainerHealthUnhealthy {
return false
}
switch req.Operation {
case agentexec.DockerContainerOperationStart:
return after.Running && after.State == "running"
case agentexec.DockerContainerOperationRestart:
return after.Running && after.State == "running" && !after.StartedAt.IsZero() && after.StartedAt.After(before.StartedAt)
default:
return false
}
}
func (m *localDockerLifecycleManager) waitForDockerLifecycleHealth(ctx context.Context) error {
interval := dockerLifecycleHealthPollInterval
if m != nil && m.healthPollInterval > 0 {
interval = m.healthPollInterval
}
timer := time.NewTimer(interval)
defer timer.Stop()
select {
case <-ctx.Done():
return ctx.Err()
case <-timer.C:
return nil
}
}