Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

18.5 KB
Raw Blame
package monitoring

import (
	"fmt"
	"strings"
	"time"

	"github.com/google/uuid"
	"github.com/rcourtman/pulse-go-rewrite/internal/models"
	"github.com/rcourtman/pulse-go-rewrite/internal/unifiedresources"
	"github.com/rs/zerolog/log"
)

const (
	// DockerCommandTypeStop instructs the agent to stop reporting and uninstall itself.
	DockerCommandTypeStop = "stop"
	// DockerCommandTypeUpdateContainer instructs the agent to update a container to its latest image.
	DockerCommandTypeUpdateContainer = "update_container"
	// DockerCommandTypeUpdateAll instructs the agent to update all containers with updates available.
	DockerCommandTypeUpdateAll = "update_all"
	// DockerCommandTypeCheckUpdates instructs the agent to check for container updates immediately.
	DockerCommandTypeCheckUpdates = "check_updates"

	// DockerCommandStatusQueued indicates the command is queued and waiting to be dispatched.
	DockerCommandStatusQueued = "queued"
	// DockerCommandStatusDispatched indicates the command has been delivered to the agent.
	DockerCommandStatusDispatched = "dispatched"
	// DockerCommandStatusAcknowledged indicates the agent acknowledged receipt of the command.
	DockerCommandStatusAcknowledged = "acknowledged"
	// DockerCommandStatusCompleted indicates the command completed successfully.
	DockerCommandStatusCompleted = "completed"
	// DockerCommandStatusFailed indicates the command failed.
	DockerCommandStatusFailed = "failed"
	// DockerCommandStatusInProgress indicates the command is actively running.
	DockerCommandStatusInProgress = "in_progress"
	// DockerCommandStatusExpired indicates Pulse abandoned the command.
	DockerCommandStatusExpired = "expired"
)

const (
	dockerCommandDefaultTTL = 10 * time.Minute
)

type dockerHostCommand struct {
	status  models.DockerHostCommandStatus
	payload map[string]any
	ttl     time.Time
}

func newDockerHostCommand(commandType string, message string, ttl time.Duration, payload map[string]any) dockerHostCommand {
	now := time.Now().UTC()
	var expiresAt *time.Time
	if ttl > 0 {
		expiry := now.Add(ttl)
		expiresAt = &expiry
	}
	return dockerHostCommand{
		status: models.DockerHostCommandStatus{
			ID:        uuid.NewString(),
			Type:      commandType,
			Status:    DockerCommandStatusQueued,
			Message:   message,
			CreatedAt: now,
			UpdatedAt: now,
			ExpiresAt: expiresAt,
		},
		payload: payload,
		ttl:     now.Add(ttl),
	}
}

func (cmd *dockerHostCommand) markDispatched() {
	now := time.Now().UTC()
	cmd.status.Status = DockerCommandStatusDispatched
	cmd.status.DispatchedAt = &now
	cmd.status.UpdatedAt = now
}

func (cmd *dockerHostCommand) markInProgress(message string) {
	now := time.Now().UTC()
	cmd.status.Status = DockerCommandStatusInProgress
	cmd.status.UpdatedAt = now
	if message != "" {
		cmd.status.Message = message
	}
}

func (cmd *dockerHostCommand) markAcknowledged(message string) {
	now := time.Now().UTC()
	cmd.status.Status = DockerCommandStatusAcknowledged
	cmd.status.AcknowledgedAt = &now
	cmd.status.UpdatedAt = now
	if message != "" {
		cmd.status.Message = message
	}
}

func (cmd *dockerHostCommand) markCompleted(message string) {
	now := time.Now().UTC()
	cmd.status.Status = DockerCommandStatusCompleted
	cmd.status.CompletedAt = &now
	cmd.status.UpdatedAt = now
	if message != "" {
		cmd.status.Message = message
	}
}

func (cmd *dockerHostCommand) markFailed(reason string) {
	now := time.Now().UTC()
	cmd.status.Status = DockerCommandStatusFailed
	cmd.status.FailedAt = &now
	cmd.status.UpdatedAt = now
	cmd.status.FailureReason = reason
}

func (cmd *dockerHostCommand) hasExpired(now time.Time) bool {
	if cmd.status.ExpiresAt == nil {
		return false
	}
	return now.After(*cmd.status.ExpiresAt)
}

// queueDockerStopCommand enqueues a stop command for the specified docker host.
func (m *Monitor) queueDockerStopCommand(hostID string) (models.DockerHostCommandStatus, error) {
	m.mu.Lock()
	defer m.mu.Unlock()

	hostID = normalizeDockerHostID(hostID)
	if hostID == "" {
		return models.DockerHostCommandStatus{}, fmt.Errorf("docker host id is required")
	}

	_, canonicalHostID, ok := m.resolveDockerCommandHostLocked(hostID)
	if !ok {
		return models.DockerHostCommandStatus{}, fmt.Errorf("docker host %q not found", hostID)
	}
	hostID = canonicalHostID

	if existing, ok := m.dockerCommands[hostID]; ok {
		switch existing.status.Status {
		case DockerCommandStatusQueued, DockerCommandStatusDispatched, DockerCommandStatusAcknowledged, DockerCommandStatusInProgress:
			return existing.status, fmt.Errorf("docker host %q already has a command in progress", hostID)
		}
	}

	cmd := newDockerHostCommand(DockerCommandTypeStop, "Stopping agent", dockerCommandDefaultTTL, nil)
	if m.dockerCommands == nil {
		m.dockerCommands = make(map[string]*dockerHostCommand)
	}
	m.dockerCommands[hostID] = &cmd
	if m.dockerCommandIndex == nil {
		m.dockerCommandIndex = make(map[string]string)
	}
	m.dockerCommandIndex[cmd.status.ID] = hostID

	m.state.SetDockerHostPendingUninstall(hostID, true)
	m.state.SetDockerHostCommand(hostID, &cmd.status)
	log.Info().
		Str("dockerHostID", hostID).
		Str("commandID", cmd.status.ID).
		Msg("Queued docker host stop command")

	return cmd.status, nil
}

// QueueDockerContainerUpdateCommand enqueues an update command for a specific container.
func (m *Monitor) QueueDockerContainerUpdateCommand(hostID, containerID, containerName string) (models.DockerHostCommandStatus, error) {
	m.mu.Lock()
	defer m.mu.Unlock()

	hostID = normalizeDockerHostID(hostID)
	if hostID == "" {
		return models.DockerHostCommandStatus{}, fmt.Errorf("docker host id is required")
	}

	containerID = strings.TrimSpace(containerID)
	if containerID == "" {
		return models.DockerHostCommandStatus{}, fmt.Errorf("container id is required")
	}

	_, canonicalHostID, ok := m.resolveDockerCommandHostLocked(hostID)
	if !ok {
		return models.DockerHostCommandStatus{}, fmt.Errorf("docker host %q not found", hostID)
	}
	hostID = canonicalHostID
	if err := m.ensureDockerMutatingCommandsAllowedLocked(hostID); err != nil {
		return models.DockerHostCommandStatus{}, err
	}

	// Check for existing commands in progress for this host
	if existing, ok := m.dockerCommands[hostID]; ok {
		switch existing.status.Status {
		case DockerCommandStatusQueued, DockerCommandStatusDispatched, DockerCommandStatusAcknowledged, DockerCommandStatusInProgress:
			return existing.status, fmt.Errorf("docker host %q already has a command in progress", hostID)
		}
	}

	// Create payload with container information
	payload := map[string]any{
		"containerId":   containerID,
		"containerName": containerName,
	}

	cmd := newDockerHostCommand(DockerCommandTypeUpdateContainer, fmt.Sprintf("Updating container %s", containerName), dockerCommandDefaultTTL, payload)
	// Encode container ID in command ID so frontend can track it
	cmd.status.ID = fmt.Sprintf("%s:%s", cmd.status.ID, containerID)

	if m.dockerCommands == nil {
		m.dockerCommands = make(map[string]*dockerHostCommand)
	}
	m.dockerCommands[hostID] = &cmd
	if m.dockerCommandIndex == nil {
		m.dockerCommandIndex = make(map[string]string)
	}
	m.dockerCommandIndex[cmd.status.ID] = hostID

	m.state.SetDockerHostCommand(hostID, &cmd.status)
	log.Info().
		Str("dockerHostID", hostID).
		Str("containerId", containerID).
		Str("containerName", containerName).
		Str("commandID", cmd.status.ID).
		Msg("Queued docker container update command")

	return cmd.status, nil
}

// QueueDockerUpdateAllCommand enqueues an update_all command for a docker host.
// The monitor selects containers with UpdateStatus.UpdateAvailable == true and includes their IDs in the payload.
func (m *Monitor) QueueDockerUpdateAllCommand(hostID string) (models.DockerHostCommandStatus, error) {
	m.mu.Lock()
	defer m.mu.Unlock()

	hostID = normalizeDockerHostID(hostID)
	if hostID == "" {
		return models.DockerHostCommandStatus{}, fmt.Errorf("docker host id is required")
	}

	_, canonicalHostID, ok := m.resolveDockerCommandHostLocked(hostID)
	if !ok {
		return models.DockerHostCommandStatus{}, fmt.Errorf("docker host %q not found", hostID)
	}
	hostID = canonicalHostID
	if err := m.ensureDockerMutatingCommandsAllowedLocked(hostID); err != nil {
		return models.DockerHostCommandStatus{}, err
	}

	// Check for existing commands in progress for this host
	if existing, ok := m.dockerCommands[hostID]; ok {
		switch existing.status.Status {
		case DockerCommandStatusQueued, DockerCommandStatusDispatched, DockerCommandStatusAcknowledged, DockerCommandStatusInProgress:
			return existing.status, fmt.Errorf("docker host %q already has a command in progress", hostID)
		}
	}

	// Select containers that currently have updates available.
	readState := m.dockerCommandReadStateLocked()
	containerIDs := make([]string, 0)
	for _, c := range readState.DockerContainers() {
		if c == nil || normalizeDockerHostID(c.HostSourceID()) != hostID {
			continue
		}
		updateStatus := c.UpdateStatus()
		if updateStatus == nil || !updateStatus.UpdateAvailable {
			continue
		}
		containerID := strings.TrimSpace(c.ContainerID())
		if containerID == "" {
			continue
		}
		containerIDs = append(containerIDs, containerID)
	}

	if len(containerIDs) == 0 {
		// Fallback to host-level nested container data when read-state was built from
		// a snapshot shape that has not yet materialized per-container resources.
		if snapshotHost, found := m.stateDockerHostByIDLocked(hostID); found {
			for _, c := range snapshotHost.Containers {
				if c.UpdateStatus == nil || !c.UpdateStatus.UpdateAvailable {
					continue
				}
				if strings.TrimSpace(c.ID) == "" {
					continue
				}
				containerIDs = append(containerIDs, c.ID)
			}
		}
	}

	if len(containerIDs) == 0 {
		return models.DockerHostCommandStatus{}, fmt.Errorf("no containers have updates available")
	}

	payload := map[string]any{
		"containerIds": containerIDs,
	}

	cmd := newDockerHostCommand(
		DockerCommandTypeUpdateAll,
		fmt.Sprintf("Updating %d containers", len(containerIDs)),
		dockerCommandDefaultTTL,
		payload,
	)

	if m.dockerCommands == nil {
		m.dockerCommands = make(map[string]*dockerHostCommand)
	}
	m.dockerCommands[hostID] = &cmd
	if m.dockerCommandIndex == nil {
		m.dockerCommandIndex = make(map[string]string)
	}
	m.dockerCommandIndex[cmd.status.ID] = hostID

	m.state.SetDockerHostCommand(hostID, &cmd.status)
	log.Info().
		Str("dockerHostID", hostID).
		Int("containers", len(containerIDs)).
		Str("commandID", cmd.status.ID).
		Msg("Queued docker update all command")

	return cmd.status, nil
}

// QueueDockerCheckUpdatesCommand queues a command to check for container updates on a Docker host.
func (m *Monitor) QueueDockerCheckUpdatesCommand(hostID string) (models.DockerHostCommandStatus, error) {
	m.mu.Lock()
	defer m.mu.Unlock()

	hostID = normalizeDockerHostID(hostID)
	if hostID == "" {
		return models.DockerHostCommandStatus{}, fmt.Errorf("docker host id is required")
	}

	_, canonicalHostID, ok := m.resolveDockerCommandHostLocked(hostID)
	if !ok {
		return models.DockerHostCommandStatus{}, fmt.Errorf("docker host %q not found", hostID)
	}
	hostID = canonicalHostID

	// Check for existing commands in progress for this host
	if existing, ok := m.dockerCommands[hostID]; ok {
		switch existing.status.Status {
		case DockerCommandStatusQueued, DockerCommandStatusDispatched, DockerCommandStatusAcknowledged, DockerCommandStatusInProgress:
			return existing.status, fmt.Errorf("docker host %q already has a command in progress", hostID)
		}
	}

	cmd := newDockerHostCommand(DockerCommandTypeCheckUpdates, "Checking for container updates", dockerCommandDefaultTTL, nil)

	// Save the command
	if m.dockerCommands == nil {
		m.dockerCommands = make(map[string]*dockerHostCommand)
	}
	m.dockerCommands[hostID] = &cmd
	if m.dockerCommandIndex == nil {
		m.dockerCommandIndex = make(map[string]string)
	}
	m.dockerCommandIndex[cmd.status.ID] = hostID

	m.state.SetDockerHostCommand(hostID, &cmd.status)
	log.Info().
		Str("dockerHostID", hostID).
		Str("commandID", cmd.status.ID).
		Msg("Queued docker check updates command")

	return cmd.status, nil
}

// GetDockerCommandStatus returns the current status of a docker host command
// by ID. Active commands are looked up directly; terminal commands
// (completed/failed) are cleared from the active map on acknowledgement but
// persist as the host's last command in state, so those are found there.
func (m *Monitor) GetDockerCommandStatus(commandID string) (models.DockerHostCommandStatus, bool) {
	m.mu.Lock()
	defer m.mu.Unlock()

	commandID = strings.TrimSpace(commandID)
	if commandID == "" {
		return models.DockerHostCommandStatus{}, false
	}

	if hostID, ok := m.dockerCommandIndex[commandID]; ok {
		if cmd, ok := m.dockerCommands[hostID]; ok && cmd.status.ID == commandID {
			return cmd.status, true
		}
	}

	if m.state != nil {
		for _, host := range m.state.GetDockerHosts() {
			if host.Command != nil && host.Command.ID == commandID {
				return *host.Command, true
			}
		}
	}

	return models.DockerHostCommandStatus{}, false
}

func (m *Monitor) getDockerCommandPayload(hostID string) (map[string]any, *models.DockerHostCommandStatus) {
	m.mu.Lock()
	defer m.mu.Unlock()

	hostID = normalizeDockerHostID(hostID)
	if hostID == "" {
		return nil, nil
	}

	cmd, ok := m.dockerCommands[hostID]
	if !ok {
		return nil, nil
	}
	now := time.Now().UTC()
	if cmd.hasExpired(now) {
		cmd.status.Status = DockerCommandStatusExpired
		cmd.status.UpdatedAt = now
		cmd.status.FailureReason = "command expired before agent acknowledged it"
		m.state.SetDockerHostCommand(hostID, &cmd.status)

		// If this was a stop command (uninstall), clear the pending flag so the host isn't stuck
		if cmd.status.Type == DockerCommandTypeStop {
			m.state.SetDockerHostPendingUninstall(hostID, false)
		}

		log.Warn().
			Str("dockerHostID", hostID).
			Str("commandID", cmd.status.ID).
			Msg("Docker command expired prior to dispatch")
		delete(m.dockerCommands, hostID)
		delete(m.dockerCommandIndex, cmd.status.ID)
		return nil, nil
	}

	// Only return the command on the first dispatch (queued → dispatched).
	// Re-sending an already-dispatched command causes the agent to re-execute
	// it on every report cycle until the ack succeeds, creating an infinite
	// loop when the ack HTTP call fails (issue #1504).
	if cmd.status.Status != DockerCommandStatusQueued {
		return nil, nil
	}

	cmd.markDispatched()
	m.state.SetDockerHostCommand(hostID, &cmd.status)

	statusCopy := cmd.status
	return cmd.payload, &statusCopy
}

func (m *Monitor) acknowledgeDockerCommand(commandID, hostID, status, message string) (models.DockerHostCommandStatus, string, bool, error) {
	m.mu.Lock()
	defer m.mu.Unlock()

	commandID = strings.TrimSpace(commandID)
	if commandID == "" {
		return models.DockerHostCommandStatus{}, "", false, fmt.Errorf("command id is required")
	}

	resolvedHostID, ok := m.dockerCommandIndex[commandID]
	if !ok {
		return models.DockerHostCommandStatus{}, "", false, fmt.Errorf("docker host command %q not found", commandID)
	}

	if hostID != "" {
		normalized := normalizeDockerHostID(hostID)
		if normalized == "" {
			return models.DockerHostCommandStatus{}, "", false, fmt.Errorf("docker host id is required")
		}
		if normalized != resolvedHostID {
			return models.DockerHostCommandStatus{}, "", false, fmt.Errorf("command %q does not belong to host %q", commandID, normalized)
		}
	}

	cmd, ok := m.dockerCommands[resolvedHostID]
	if !ok || cmd.status.ID != commandID {
		return models.DockerHostCommandStatus{}, "", false, fmt.Errorf("docker host command %q not active", commandID)
	}

	if message != "" {
		message = strings.TrimSpace(message)
	}

	shouldRemove := false
	switch status {
	case DockerCommandStatusAcknowledged:
		cmd.markAcknowledged(message)
	case DockerCommandStatusInProgress:
		cmd.markInProgress(message)
	case DockerCommandStatusCompleted:
		cmd.markAcknowledged(message)
		cmd.markCompleted(message)
		// Only remove the Docker host if this was a "stop" command.
		if cmd.status.Type == DockerCommandTypeStop {
			shouldRemove = true
		}
	case DockerCommandStatusFailed:
		cmd.markFailed(message)
		m.state.SetDockerHostPendingUninstall(resolvedHostID, false)
	default:
		return models.DockerHostCommandStatus{}, "", false, fmt.Errorf("invalid command status %q", status)
	}

	m.state.SetDockerHostCommand(resolvedHostID, &cmd.status)

	log.Info().
		Str("dockerHostID", resolvedHostID).
		Str("commandID", cmd.status.ID).
		Str("status", cmd.status.Status).
		Msg("Docker host acknowledged command")

	// Completed/failed commands should not be sent to agents again. Keep the last status
	// in state for UI visibility, but clear the active command from memory.
	if status == DockerCommandStatusFailed || status == DockerCommandStatusCompleted || shouldRemove {
		delete(m.dockerCommands, resolvedHostID)
		delete(m.dockerCommandIndex, commandID)
	}

	return cmd.status, resolvedHostID, shouldRemove, nil
}

func normalizeDockerHostID(id string) string {
	return strings.TrimSpace(id)
}

func (m *Monitor) dockerCommandReadStateLocked() unifiedresources.ReadState {
	if m == nil {
		return nil
	}
	if readState, ok := m.resourceStore.(unifiedresources.ReadState); ok && readState != nil {
		return readState
	}
	if m.state == nil {
		return nil
	}

	registry := unifiedresources.NewRegistry(nil)
	registry.IngestSnapshot(m.state.GetSnapshot())
	return unifiedresources.NewMonitorAdapter(registry)
}

func (m *Monitor) resolveDockerCommandHostLocked(hostID string) (*unifiedresources.DockerHostView, string, bool) {
	hostID = normalizeDockerHostID(hostID)
	if hostID == "" {
		return nil, "", false
	}

	readState := m.dockerCommandReadStateLocked()
	if readState == nil {
		return nil, "", false
	}

	for _, host := range readState.DockerHosts() {
		if host == nil {
			continue
		}
		candidateID := normalizeDockerHostID(host.ID())
		sourceID := normalizeDockerHostID(host.HostSourceID())
		if hostID != candidateID && hostID != sourceID {
			continue
		}
		if sourceID == "" {
			sourceID = candidateID
		}
		if sourceID == "" {
			return nil, "", false
		}
		return host, sourceID, true
	}

	return nil, "", false
}

func (m *Monitor) stateDockerHostByIDLocked(hostID string) (models.DockerHost, bool) {
	if m == nil || m.state == nil {
		return models.DockerHost{}, false
	}

	for _, host := range m.state.GetDockerHosts() {
		if normalizeDockerHostID(host.ID) == hostID {
			return host, true
		}
	}
	return models.DockerHost{}, false
}

func (m *Monitor) ensureDockerMutatingCommandsAllowedLocked(hostID string) error {
	host, found := m.stateDockerHostByIDLocked(hostID)
	if !found {
		return fmt.Errorf("docker host %q not found", hostID)
	}
	if host.Security == nil || !host.Security.MutatingCommandsBlocked {
		return nil
	}

	reason := strings.TrimSpace(host.Security.MutatingCommandsBlockedReason)
	if reason == "" {
		reason = "Docker daemon-mutating commands are disabled for this host."
	}
	return fmt.Errorf("%s", reason)
}