Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

11.9 KB
Raw Blame
package hostagent

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net"
	"os"
	"os/exec"
	"regexp"
	"strconv"
	"strings"
	"time"

	"github.com/rcourtman/pulse-go-rewrite/internal/agentexec"
)

var dockerContextNamePattern = regexp.MustCompile(`^[a-zA-Z0-9._-]{1,128}$`)

const dockerLifecycleHealthPollInterval = 2 * time.Second

type dockerLifecycleManager interface {
	Apply(context.Context, agentexec.DockerContainerLifecyclePayload) agentexec.DockerContainerLifecycleResultPayload
	Preflight(context.Context, agentexec.DockerContainerLifecyclePayload) (bool, string)
}

func (m *localDockerLifecycleManager) Observe(ctx context.Context, runtime, containerID string) (agentexec.DockerContainerObservationSnapshot, error) {
	snapshot, err := m.inspect(ctx, runtime, containerID)
	if err != nil {
		return agentexec.DockerContainerObservationSnapshot{}, err
	}
	return agentexec.DockerContainerObservationSnapshot{
		ContainerID: snapshot.ContainerID, State: snapshot.State, Running: snapshot.Running, Health: snapshot.Health,
		StartedAt: snapshot.StartedAt, RestartCount: snapshot.RestartCount, ObservedAt: snapshot.ObservedAt,
	}, nil
}

// DockerContainerLifecycleOperator is the narrow bridge from the host command
// channel to the unified agent's connected Docker / Podman module. It keeps
// lifecycle execution on the daemon API and avoids requiring a second runtime
// client executable inside the agent image.
type DockerContainerLifecycleOperator interface {
	InspectDockerContainerLifecycle(context.Context, string, string) (agentexec.DockerContainerLifecycleSnapshot, error)
	MutateDockerContainerLifecycle(context.Context, string, string, string) error
}

type dockerLifecycleCommandRunner func(context.Context, string, ...string) ([]byte, error)

type localDockerLifecycleManager struct {
	run                dockerLifecycleCommandRunner
	operator           DockerContainerLifecycleOperator
	now                func() time.Time
	healthPollInterval time.Duration
}

func newLocalDockerLifecycleManager(operators ...DockerContainerLifecycleOperator) *localDockerLifecycleManager {
	manager := &localDockerLifecycleManager{
		run: func(ctx context.Context, name string, args ...string) ([]byte, error) {
			return exec.CommandContext(ctx, name, args...).CombinedOutput()
		},
		now: time.Now,
	}
	if len(operators) > 0 {
		manager.operator = operators[0]
	}
	return manager
}

func (m *localDockerLifecycleManager) Apply(ctx context.Context, req agentexec.DockerContainerLifecyclePayload) (result agentexec.DockerContainerLifecycleResultPayload) {
	started := m.currentTime()
	result = agentexec.DockerContainerLifecycleResultPayload{
		RequestID: req.RequestID, ActionID: req.ActionID, Operation: req.Operation,
		OperationVersion: req.OperationVersion, RequestDigest: req.RequestDigest, ContainerID: req.ContainerID,
		ExecutionPhase: agentexec.DockerContainerPhasePreflight,
	}
	defer func() { result.Duration = time.Since(started).Milliseconds() }()
	if err := agentexec.ValidateDockerContainerLifecyclePayload(&req); err != nil {
		result.ReasonCode = agentexec.ActionRefusalContractInvalid
		result.Error = "typed container lifecycle preflight refused"
		return result
	}
	before, err := m.inspect(ctx, req.Runtime, req.ContainerID)
	result.Before = before
	if err != nil {
		result.ReasonCode = agentexec.ActionRefusalTargetInspectionUnavailable
		result.Error = "container preflight inspect unavailable"
		return result
	}
	if !dockerLifecycleBeforeMatches(req, before) {
		result.ReasonCode = agentexec.ActionRefusalTargetStateChanged
		result.Error = "container preflight state no longer matches the request"
		return result
	}

	result.ExecutionPhase = agentexec.DockerContainerPhaseMutate
	result.MutationStarted = true
	verb := strings.TrimSuffix(req.Operation, "_container")
	mutationErr := m.mutate(ctx, req.Runtime, verb, req.ContainerID)
	if mutationErr != nil {
		// A lost acknowledgement does not prove that the daemon did nothing.
		// Reconcile using reads under the original deadline, without retrying
		// the mutation or claiming its execution was acknowledged.
		result.Error = dockerLifecycleMutationError(mutationErr)
		if ctx.Err() != nil {
			return result
		}
	} else {
		result.MutationCompleted = true
	}
	result.ExecutionPhase = agentexec.DockerContainerPhaseVerify
	after, err := m.inspect(ctx, req.Runtime, req.ContainerID)
	result.After = after
	if err != nil {
		if mutationErr == nil {
			result.Error = "container postcondition inspect unavailable"
		}
		return result
	}
	result.ReadbackRan = true
	if dockerLifecyclePostcondition(req, before, after) {
		if result.MutationCompleted {
			result.ExecutionPhase = agentexec.DockerContainerPhaseComplete
		}
		return result
	}
	if dockerLifecycleHealthPending(req, before, after) {
		verifyCtx, cancel := context.WithTimeout(ctx, time.Duration(req.Timeout)*time.Second)
		defer cancel()
		for dockerLifecycleHealthPending(req, before, after) {
			if err := m.waitForDockerLifecycleHealth(verifyCtx); err != nil {
				break
			}
			after, err = m.inspect(verifyCtx, req.Runtime, req.ContainerID)
			if err != nil {
				if mutationErr == nil {
					result.Error = "container postcondition inspect unavailable"
				}
				return result
			}
			result.After = after
			if dockerLifecyclePostcondition(req, before, after) {
				if result.MutationCompleted {
					result.ExecutionPhase = agentexec.DockerContainerPhaseComplete
				}
				return result
			}
		}
	}
	if mutationErr == nil {
		result.Error = "container postcondition did not reach the requested state"
	}
	return result
}

func dockerLifecycleMutationError(err error) string {
	var timeout net.Error
	switch {
	case errors.Is(err, context.DeadlineExceeded), errors.As(err, &timeout) && timeout.Timeout():
		return "container lifecycle mutation acknowledgement timed out"
	case errors.Is(err, context.Canceled):
		return "container lifecycle mutation acknowledgement canceled"
	default:
		return "container lifecycle mutation acknowledgement unavailable"
	}
}

func (m *localDockerLifecycleManager) Preflight(ctx context.Context, req agentexec.DockerContainerLifecyclePayload) (bool, string) {
	if err := agentexec.ValidateDockerContainerLifecyclePayload(&req); err != nil {
		return false, agentexec.ActionRefusalContractInvalid
	}
	before, err := m.inspect(ctx, req.Runtime, req.ContainerID)
	if err != nil {
		return false, agentexec.ActionRefusalTargetInspectionUnavailable
	}
	if !dockerLifecycleBeforeMatches(req, before) {
		return false, agentexec.ActionRefusalTargetStateChanged
	}
	return true, ""
}

func (m *localDockerLifecycleManager) command(ctx context.Context, runtime string, args ...string) ([]byte, error) {
	if m == nil || m.run == nil {
		return nil, fmt.Errorf("container runtime command runner unavailable")
	}
	if runtime != "docker" && runtime != "podman" {
		return nil, fmt.Errorf("unsupported container runtime")
	}
	if runtime == "docker" {
		if dockerContext := strings.TrimSpace(os.Getenv("DOCKER_CONTEXT")); dockerContext != "" {
			if !dockerContextNamePattern.MatchString(dockerContext) {
				return nil, fmt.Errorf("invalid Docker context name")
			}
			args = append([]string{"--context", dockerContext}, args...)
		}
	}
	return m.run(ctx, runtime, args...)
}

func (m *localDockerLifecycleManager) mutate(ctx context.Context, runtime, operation, containerID string) error {
	if m != nil && m.operator != nil {
		return m.operator.MutateDockerContainerLifecycle(ctx, runtime, operation, containerID)
	}
	_, err := m.command(ctx, runtime, operation, containerID)
	return err
}

func (m *localDockerLifecycleManager) inspect(ctx context.Context, runtime, containerID string) (agentexec.DockerContainerLifecycleSnapshot, error) {
	if m != nil && m.operator != nil {
		return m.operator.InspectDockerContainerLifecycle(ctx, runtime, containerID)
	}
	const format = `{{json .State}}`
	raw, err := m.command(ctx, runtime, "inspect", "--format", format, containerID)
	if err != nil {
		return agentexec.DockerContainerLifecycleSnapshot{}, err
	}
	var state struct {
		Status       string `json:"Status"`
		Running      bool   `json:"Running"`
		StartedAt    string `json:"StartedAt"`
		RestartCount int    `json:"RestartCount"`
		Health       *struct {
			Status string `json:"Status"`
		} `json:"Health"`
	}
	if err := json.Unmarshal([]byte(strings.TrimSpace(string(raw))), &state); err != nil {
		return agentexec.DockerContainerLifecycleSnapshot{}, fmt.Errorf("decode container state: %w", err)
	}
	startedAt := time.Time{}
	if value := strings.TrimSpace(state.StartedAt); value != "" && !strings.HasPrefix(value, "0001-") {
		startedAt, err = time.Parse(time.RFC3339Nano, value)
		if err != nil {
			return agentexec.DockerContainerLifecycleSnapshot{}, fmt.Errorf("decode container start time: %w", err)
		}
	}
	// Docker exposes RestartCount at the inspect root. Query it separately to
	// keep the state envelope bounded and avoid persisting labels or config.
	restartRaw, restartErr := m.command(ctx, runtime, "inspect", "--format", `{{.RestartCount}}`, containerID)
	if restartErr == nil {
		if count, parseErr := strconv.Atoi(strings.TrimSpace(string(restartRaw))); parseErr == nil && count >= 0 {
			state.RestartCount = count
		}
	}
	health := agentexec.DockerContainerHealthNone
	if state.Health != nil {
		health = strings.ToLower(strings.TrimSpace(state.Health.Status))
		if !agentexec.IsDockerContainerHealth(health) {
			return agentexec.DockerContainerLifecycleSnapshot{}, fmt.Errorf("container inspect returned unsupported health status")
		}
	}
	return agentexec.DockerContainerLifecycleSnapshot{
		ContainerID: strings.ToLower(containerID), State: strings.ToLower(strings.TrimSpace(state.Status)), Running: state.Running,
		Health: health, StartedAt: startedAt.UTC(), RestartCount: state.RestartCount, ObservedAt: m.currentTime(),
	}, nil
}

func (m *localDockerLifecycleManager) currentTime() time.Time {
	if m != nil && m.now != nil {
		return m.now().UTC()
	}
	return time.Now().UTC()
}

func dockerLifecycleBeforeMatches(req agentexec.DockerContainerLifecyclePayload, before agentexec.DockerContainerLifecycleSnapshot) bool {
	if !strings.EqualFold(req.ContainerID, before.ContainerID) || !strings.EqualFold(req.ExpectedState, before.State) {
		return false
	}
	if !req.ExpectedStartedAt.IsZero() && !req.ExpectedStartedAt.Equal(before.StartedAt) {
		return false
	}
	return true
}

func dockerLifecyclePostcondition(req agentexec.DockerContainerLifecyclePayload, before, after agentexec.DockerContainerLifecycleSnapshot) bool {
	if !strings.EqualFold(req.ContainerID, after.ContainerID) {
		return false
	}
	switch req.Operation {
	case agentexec.DockerContainerOperationStart:
		return after.Running && after.State == "running" && agentexec.DockerContainerHealthAllowsVerifiedRunningState(after.Health)
	case agentexec.DockerContainerOperationStop:
		return !after.Running && after.State == "exited"
	case agentexec.DockerContainerOperationRestart:
		return after.Running && after.State == "running" && !after.StartedAt.IsZero() && after.StartedAt.After(before.StartedAt) && agentexec.DockerContainerHealthAllowsVerifiedRunningState(after.Health)
	default:
		return false
	}
}

func dockerLifecycleHealthPending(req agentexec.DockerContainerLifecyclePayload, before, after agentexec.DockerContainerLifecycleSnapshot) bool {
	health := strings.ToLower(strings.TrimSpace(after.Health))
	if health != agentexec.DockerContainerHealthStarting && health != agentexec.DockerContainerHealthUnhealthy {
		return false
	}
	switch req.Operation {
	case agentexec.DockerContainerOperationStart:
		return after.Running && after.State == "running"
	case agentexec.DockerContainerOperationRestart:
		return after.Running && after.State == "running" && !after.StartedAt.IsZero() && after.StartedAt.After(before.StartedAt)
	default:
		return false
	}
}

func (m *localDockerLifecycleManager) waitForDockerLifecycleHealth(ctx context.Context) error {
	interval := dockerLifecycleHealthPollInterval
	if m != nil && m.healthPollInterval > 0 {
		interval = m.healthPollInterval
	}
	timer := time.NewTimer(interval)
	defer timer.Stop()
	select {
	case <-ctx.Done():
		return ctx.Err()
	case <-timer.C:
		return nil
	}
}