Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

15.2 KB
Raw Blame
package config

import (
	"encoding/json"
	"errors"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"sync"
	"sync/atomic"
	"time"

	"github.com/rs/zerolog/log"
)

// GuestMetadata holds additional metadata for a guest (VM/container)
type GuestMetadata struct {
	ID          string   `json:"id"`          // Guest ID (e.g., "node:vmid" format)
	CustomURL   string   `json:"customUrl"`   // Custom URL for the guest
	Description string   `json:"description"` // Optional description
	Tags        []string `json:"tags"`        // Optional tags for categorization
	Notes       []string `json:"notes"`       // User annotations for AI context (e.g., "Runs PBS in Docker")
	// Last-known identity (persisted even after guest deletion)
	LastKnownName string `json:"lastKnownName,omitempty"` // Last known guest name
	LastKnownType string `json:"lastKnownType,omitempty"` // Last known guest type (qemu, lxc)
}

// GuestMetadataStore manages guest metadata
type GuestMetadataStore struct {
	mu       sync.RWMutex
	metadata map[string]*GuestMetadata // keyed by guest ID
	dataPath string
	fs       FileSystem

	// writeMu serializes disk snapshots with synchronous mutations. Background
	// I/O never holds mu, so a burst can accumulate in one follow-up snapshot.
	// Always acquire writeMu before mu when both locks are needed.
	writeMu           sync.Mutex
	revision          uint64
	attemptedRevision uint64
	lastWriteErr      error
	writerDone        chan struct{}
	closed            atomic.Bool
	closeMu           sync.Mutex
	closeAttempt      *guestMetadataCloseAttempt
}

type guestMetadataCloseAttempt struct {
	done chan struct{}
	err  error
}

var ErrGuestMetadataStoreClosed = errors.New("guest metadata store is closed")

// SetAsync admits a detached replacement before returning. One store-owned
// writer persists the newest complete snapshot, coalescing changes that arrive
// during I/O. It does not start a goroutine or rewrite the file per guest.
func (s *GuestMetadataStore) SetAsync(guestID string, meta *GuestMetadata) error {
	if s == nil {
		return nil
	}
	if meta == nil {
		return fmt.Errorf("metadata cannot be nil")
	}
	clone := cloneGuestMetadata(meta)
	clone.ID = guestID
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.closed.Load() {
		return ErrGuestMetadataStoreClosed
	}
	s.metadata[guestID] = clone
	s.queueSaveLocked()
	return nil
}

// RememberIdentity merges only monitor-owned identity fields at admission.
// A background name/type update must never restore an old URL, tag or note
// over a concurrent operator edit. OCI classification cannot be downgraded by
// a later poll lacking the classification evidence.
func (s *GuestMetadataStore) RememberIdentity(guestID, name, guestType string) error {
	if s == nil {
		return nil
	}
	guestType = strings.TrimSpace(guestType)
	if guestType == "" {
		return nil
	}
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.closed.Load() {
		return ErrGuestMetadataStoreClosed
	}
	meta := cloneGuestMetadata(s.metadata[guestID])
	if meta == nil {
		meta = &GuestMetadata{ID: guestID, Tags: []string{}}
	}
	if meta.LastKnownType == "oci" {
		guestType = "oci"
	}
	if meta.LastKnownName == name && meta.LastKnownType == guestType && s.lastWriteErr == nil {
		return nil
	}
	meta.LastKnownName = name
	meta.LastKnownType = guestType
	s.metadata[guestID] = meta
	s.queueSaveLocked()
	return nil
}

func (s *GuestMetadataStore) queueSaveLocked() {
	s.revision++
	if s.writerDone != nil {
		return
	}
	s.writerDone = make(chan struct{})
	go s.writePendingSnapshots()
}

func (s *GuestMetadataStore) writePendingSnapshots() {
	for {
		s.writeMu.Lock()
		s.mu.Lock()
		if s.attemptedRevision == s.revision {
			close(s.writerDone)
			s.writerDone = nil
			s.mu.Unlock()
			s.writeMu.Unlock()
			return
		}
		revision := s.revision
		data, err := json.Marshal(s.metadata)
		s.mu.Unlock()
		if err == nil {
			err = persistMetadata(s.fs, s.dataPath, "guest_metadata.json", data)
		}
		s.mu.Lock()
		s.attemptedRevision = revision
		s.lastWriteErr = err
		s.mu.Unlock()
		s.writeMu.Unlock()
		if err != nil {
			// No unbounded automatic retry on a failing filesystem. A later
			// admitted mutation (or ordinary identity poll) can try again.
			log.Error().Err(err).Msg("failed to persist guest metadata snapshot")
		}
	}
}

// WaitForPendingWrites reports quiescence, not persistence success. Producers
// must already be stopped if the caller plans to remove the data directory;
// Close seals admission as well. A timeout never makes teardown safe.
func (s *GuestMetadataStore) WaitForPendingWrites(timeout time.Duration) bool {
	if s == nil {
		return true
	}
	s.mu.RLock()
	done := s.writerDone
	s.mu.RUnlock()
	if done == nil {
		return true
	}
	timer := time.NewTimer(timeout)
	defer timer.Stop()
	select {
	case <-done:
		return true
	case <-timer.C:
		log.Warn().Dur("timeout", timeout).Msg("timed out waiting for guest metadata writes to drain")
		return false
	}
}

// Close rejects further mutations, drains the owned writer within the supplied
// budget, and reports any final persistence error. Failure keeps the writer
// owned and must stop tenant-directory removal; it is not a successful stop.
func (s *GuestMetadataStore) Close(timeout time.Duration) error {
	if s == nil {
		return nil
	}
	// Seal before waiting for a synchronous mutation's I/O-held memory lock.
	// One owned close observer, not one goroutine per waiter, covers both that
	// mutation and the background writer within the caller's unchanged budget.
	s.closed.Store(true)
	s.closeMu.Lock()
	attempt := s.closeAttempt
	retry := false
	if attempt != nil {
		select {
		case <-attempt.done:
			// A later explicit shutdown/offboarding attempt may flush retained
			// state after the filesystem is repaired. Never reopen admission,
			// retry an in-flight write or spin-retry a failed close on its own.
			if attempt.err != nil {
				attempt = nil
				retry = true
			}
		default:
		}
	}
	if attempt == nil {
		attempt = &guestMetadataCloseAttempt{done: make(chan struct{})}
		s.closeAttempt = attempt
		go func() {
			s.mu.Lock()
			if retry && s.lastWriteErr != nil && s.writerDone == nil {
				s.queueSaveLocked()
			}
			done := s.writerDone
			s.mu.Unlock()
			if done != nil {
				<-done
			}
			s.mu.RLock()
			attempt.err = s.lastWriteErr
			s.mu.RUnlock()
			close(attempt.done)
		}()
	}
	s.closeMu.Unlock()
	timer := time.NewTimer(timeout)
	defer timer.Stop()
	select {
	case <-attempt.done:
		return attempt.err
	case <-timer.C:
		return fmt.Errorf("guest metadata writes did not drain within %s", timeout)
	}
}

func cloneGuestMetadata(meta *GuestMetadata) *GuestMetadata {
	if meta == nil {
		return nil
	}

	clone := *meta
	if meta.Tags != nil {
		clone.Tags = make([]string, len(meta.Tags))
		copy(clone.Tags, meta.Tags)
	}
	if meta.Notes != nil {
		clone.Notes = make([]string, len(meta.Notes))
		copy(clone.Notes, meta.Notes)
	}
	return &clone
}

// NewGuestMetadataStore creates a new metadata store
func NewGuestMetadataStore(dataPath string, fs FileSystem) *GuestMetadataStore {
	store := &GuestMetadataStore{
		metadata: make(map[string]*GuestMetadata),
		dataPath: dataPath,
		fs:       fs,
	}

	if store.fs == nil {
		store.fs = defaultFileSystem{}
	}

	// Load existing metadata
	if err := store.Load(); err != nil {
		log.Warn().Err(err).Msg("Failed to load guest metadata")
	}

	return store
}

// Load reads metadata from disk
func (s *GuestMetadataStore) Load() error {
	s.writeMu.Lock()
	defer s.writeMu.Unlock()
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.closed.Load() {
		return ErrGuestMetadataStoreClosed
	}
	if s.revision != s.attemptedRevision || s.lastWriteErr != nil {
		return fmt.Errorf("cannot reload guest metadata with unpersisted changes")
	}
	filePath := filepath.Join(s.dataPath, "guest_metadata.json")

	log.Debug().Str("path", filePath).Msg("Loading guest metadata from disk")

	data, err := readLimitedRegularFileFS(s.fs, filePath, maxGuestMetadataFileSizeBytes)
	if err != nil {
		if os.IsNotExist(err) {
			// File doesn't exist yet, not an error
			log.Debug().Str("path", filePath).Msg("Guest metadata file does not exist yet")
			return nil
		}
		return fmt.Errorf("failed to read metadata file: %w", err)
	}

	if err := json.Unmarshal(data, &s.metadata); err != nil {
		return fmt.Errorf("failed to unmarshal metadata: %w", err)
	}

	log.Info().Int("count", len(s.metadata)).Msg("Loaded guest metadata")
	return nil
}

// save writes a synchronous mutation (writeMu and mu must both be held).
func (s *GuestMetadataStore) save() error {
	s.revision++
	data, err := json.Marshal(s.metadata)
	if err == nil {
		err = persistMetadata(s.fs, s.dataPath, "guest_metadata.json", data)
	}
	s.attemptedRevision = s.revision
	s.lastWriteErr = err
	return err
}

// Get retrieves metadata for a guest
func (s *GuestMetadataStore) Get(guestID string) *GuestMetadata {
	s.mu.RLock()
	defer s.mu.RUnlock()

	if meta, exists := s.metadata[guestID]; exists {
		return cloneGuestMetadata(meta)
	}
	return nil
}

// GetWithLegacyMigration retrieves metadata for a guest, attempting legacy ID formats if needed
// and migrating them to the new stable format (instance:node:vmid).
// This should be called when full guest context (instance, node, vmid) is available.
//
// Legacy formats attempted (in order):
// 1. instance-node-vmid (e.g., "delly-minipc-201") - most specific legacy format
// 2. instance-vmid (e.g., "delly-201") - old cluster format without node
// 3. node-vmid (e.g., "minipc-201") - standalone format
func (s *GuestMetadataStore) GetWithLegacyMigration(guestID, instance, node string, vmID int) *GuestMetadata {
	s.mu.RLock()
	meta, exists := s.metadata[guestID]
	s.mu.RUnlock()

	if exists {
		return cloneGuestMetadata(meta)
	}

	// Helper to migrate a legacy ID to the new format
	migrate := func(legacyID string) *GuestMetadata {
		s.writeMu.Lock()
		defer s.writeMu.Unlock()
		s.mu.Lock()
		defer s.mu.Unlock()
		if s.closed.Load() {
			return nil
		}

		if current := s.metadata[guestID]; current != nil {
			return cloneGuestMetadata(current)
		}
		legacyMeta := s.metadata[legacyID]
		if legacyMeta == nil {
			return cloneGuestMetadata(s.metadata[guestID])
		}

		log.Info().
			Str("legacyID", legacyID).
			Str("newID", guestID).
			Msg("Migrating guest metadata from legacy ID format")

		migrated := cloneGuestMetadata(legacyMeta)
		migrated.ID = guestID
		s.metadata[guestID] = migrated
		delete(s.metadata, legacyID)
		// Persist while holding the store lock. save() assumes locked access.
		if err := s.save(); err != nil {
			log.Error().Err(err).Msg("Failed to save guest metadata after migration")
			// The admitted guest now uses the new node coordinates. Keep its
			// operator fields available in memory while atomic persistence
			// retains the last good disk snapshot. save records the failure;
			// ordinary identity updates retry through the existing writer.
		}

		return cloneGuestMetadata(migrated)
	}

	// Try legacy format 1: instance-node-vmid (most specific)
	if instance != node {
		if result := migrate(fmt.Sprintf("%s-%s-%d", instance, node, vmID)); result != nil {
			return result
		}
	}

	// Try legacy format 2: instance-vmid (old cluster format)
	// This was used when cluster name was used without node differentiation
	if result := migrate(fmt.Sprintf("%s-%d", instance, vmID)); result != nil {
		return result
	}

	// Try legacy format 3: node-vmid (standalone format or node-only reference)
	// Only try if instance != node to avoid duplicate check
	if instance != node {
		if result := migrate(fmt.Sprintf("%s-%d", node, vmID)); result != nil {
			return result
		}
	}

	// A guest that migrated to another cluster node keeps its VMID but shows
	// up under a new node-scoped ID, orphaning its metadata (#1669). VMIDs
	// are unique within a Proxmox cluster, so an entry for the same instance
	// and VMID on a different node belongs to this guest.
	if movedID := s.findGuestIDOnOtherNode(guestID, instance, vmID); movedID != "" {
		if result := migrate(movedID); result != nil {
			return result
		}
	}

	return nil
}

// findGuestIDOnOtherNode returns a stored metadata ID for the same instance
// and VMID under a different node, or "" when none exists.
func (s *GuestMetadataStore) findGuestIDOnOtherNode(guestID, instance string, vmID int) string {
	prefix := instance + ":"
	suffix := fmt.Sprintf(":%d", vmID)

	s.mu.RLock()
	defer s.mu.RUnlock()
	for id := range s.metadata {
		if id == guestID || len(id) <= len(prefix)+len(suffix) {
			continue
		}
		if !strings.HasPrefix(id, prefix) || !strings.HasSuffix(id, suffix) {
			continue
		}
		// The middle segment must be exactly one node name: an ID with more
		// separators is a different key shape, not a node-scoped guest ID.
		middle := id[len(prefix) : len(id)-len(suffix)]
		if middle == "" || strings.Contains(middle, ":") {
			continue
		}
		return id
	}
	return ""
}

// GetAll retrieves all guest metadata
func (s *GuestMetadataStore) GetAll() map[string]*GuestMetadata {
	s.mu.RLock()
	defer s.mu.RUnlock()

	// Return a copy to prevent external modifications
	result := make(map[string]*GuestMetadata)
	for k, v := range s.metadata {
		result[k] = cloneGuestMetadata(v)
	}
	return result
}

// Set updates or creates metadata for a guest
func (s *GuestMetadataStore) Set(guestID string, meta *GuestMetadata) error {
	s.writeMu.Lock()
	defer s.writeMu.Unlock()
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.closed.Load() {
		return ErrGuestMetadataStoreClosed
	}

	if meta == nil {
		return fmt.Errorf("metadata cannot be nil")
	}

	clone := cloneGuestMetadata(meta)
	clone.ID = guestID
	s.metadata[guestID] = clone

	// Save to disk
	return s.save()
}

// Delete removes metadata for a guest
func (s *GuestMetadataStore) Delete(guestID string) error {
	s.writeMu.Lock()
	defer s.writeMu.Unlock()
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.closed.Load() {
		return ErrGuestMetadataStoreClosed
	}

	delete(s.metadata, guestID)

	// Save to disk
	return s.save()
}

// ReplaceAll replaces all metadata entries and persists them to disk.
func (s *GuestMetadataStore) ReplaceAll(metadata map[string]*GuestMetadata) error {
	s.writeMu.Lock()
	defer s.writeMu.Unlock()
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.closed.Load() {
		return ErrGuestMetadataStoreClosed
	}

	s.metadata = make(map[string]*GuestMetadata)

	for guestID, meta := range metadata {
		if meta == nil {
			continue
		}

		clone := cloneGuestMetadata(meta)
		clone.ID = guestID
		// Ensure slice copy is not nil to allow JSON marshalling of empty tags
		if clone.Tags == nil {
			clone.Tags = []string{}
		}
		s.metadata[guestID] = clone
	}

	return s.save()
}

// UpdateAll applies one atomic in-memory mutation and persists it once.
// The callback receives a deep-cloned working set and returns whether it
// changed. Concurrent Set/Delete calls cannot be lost between snapshot and
// replacement.
func (s *GuestMetadataStore) UpdateAll(
	update func(map[string]*GuestMetadata) bool,
) error {
	if update == nil {
		return nil
	}

	s.writeMu.Lock()
	defer s.writeMu.Unlock()
	s.mu.Lock()
	defer s.mu.Unlock()
	if s.closed.Load() {
		return ErrGuestMetadataStoreClosed
	}

	working := make(map[string]*GuestMetadata, len(s.metadata))
	for id, meta := range s.metadata {
		working[id] = cloneGuestMetadata(meta)
	}
	if !update(working) {
		return nil
	}
	for id, meta := range working {
		if meta == nil {
			delete(working, id)
			continue
		}
		meta.ID = id
		if meta.Tags == nil {
			meta.Tags = []string{}
		}
	}

	previous := s.metadata
	s.metadata = working
	if err := s.save(); err != nil {
		s.metadata = previous
		return err
	}
	return nil
}