Skip to content

armgit / Pulse

Public Imported from GitHub
0

Real-time monitoring dashboard for Proxmox VE, PBS, Docker, Kubernetes, TrueNAS and vSphere. Self-hosted, with alerts for failed backups, full disks, unhealthy containers and offline nodes.

48.2 KB
Raw Blame
package dockeragent

import (
	"context"
	"crypto/rand"
	"encoding/json"
	"fmt"
	"math"
	"math/big"
	"net/netip"
	"net/url"
	"regexp"
	"sort"
	"strconv"
	"strings"
	"time"

	containertypes "github.com/moby/moby/api/types/container"
	"github.com/moby/moby/api/types/image"
	networktypes "github.com/moby/moby/api/types/network"
	systemtypes "github.com/moby/moby/api/types/system"
	"github.com/moby/moby/api/types/volume"
	"github.com/moby/moby/client"
	agentsdocker "github.com/rcourtman/pulse-go-rewrite/pkg/agents/docker"
)

// buildReport gathers all system and container metrics into a single report
func (a *Agent) buildReport(ctx context.Context) (agentsdocker.Report, error) {
	// Cycle-level containment: per-call deadlines below should bound every
	// docker call, but one live exercise saw a call park for minutes with its
	// deadline never firing, so the whole cycle gets its own ceiling plus an
	// independent watchdog that dumps goroutines if even this deadline fails
	// to abort the cycle.
	ctx, cancel := context.WithTimeout(ctx, dockerCollectCycleTimeout)
	defer cancel()
	stopWatchdog := startCollectCycleWatchdog(a.logger, dockerCollectCycleTimeout+dockerCollectWatchdogGrace)
	defer stopWatchdog()

	info, err := dockerCallWithRetry(ctx, dockerInfoCallTimeout, func(callCtx context.Context) (systemtypes.Info, error) {
		return a.docker.Info(callCtx)
	})
	if err != nil {
		return agentsdocker.Report{}, fmt.Errorf("failed to query docker info: %w", annotateDockerConnectionError(err))
	}
	if a.cfg.HelperInventory != nil {
		if err := validateCollectorDirectRuntime(a.docker, info); err != nil {
			return agentsdocker.Report{}, fmt.Errorf("%w: %v", errCollectorRuntimeBoundaryChanged, err)
		}
	}

	a.runtimeVer = info.ServerVersion
	if a.daemonHost == "" {
		a.daemonHost = a.docker.DaemonHost()
	}

	// Use current runtime as preference to avoid spurious switching.
	// This preserves user's --docker-runtime choice (stored in a.runtime at init).
	newRuntime := detectRuntime(info, a.daemonHost, a.runtime)
	if newRuntime != a.runtime {
		if a.runtime != "" {
			a.logger.Info().
				Str("runtime_previous", string(a.runtime)).
				Str("runtime_current", string(newRuntime)).
				Msg("Detected container runtime change")
		}
		a.runtime = newRuntime
		a.supportsSwarm = newRuntime == RuntimeDocker
		if newRuntime == RuntimePodman {
			if a.cfg.IncludeServices {
				a.logger.Warn().Msg("Podman runtime detected during report; disabling Swarm service collection")
			}
			if a.cfg.IncludeTasks {
				a.logger.Warn().Msg("Podman runtime detected during report; disabling Swarm task collection")
			}
			a.cfg.IncludeServices = false
			a.cfg.IncludeTasks = false
		}
		a.cfg.Runtime = string(newRuntime)
	}

	a.cpuCount = info.NCPU

	agentID := a.cfg.AgentID
	if agentID == "" {
		// In unified mode, use the EXACT same fallback chain as hostagent:
		// machineID -> hostname. Never use daemonID in unified mode because
		// hostagent doesn't use it, and using different IDs causes token
		// binding conflicts on the server (reported in #985, #986).
		if a.cfg.AgentType == "unified" {
			agentID = a.machineID
			if agentID == "" {
				agentID = a.hostName
			}
		} else {
			// Standalone mode: prefer daemonID for backward compatibility,
			// then fall back to machineID -> hostname.
			// Use cached daemon ID from init rather than info.ID from current call.
			// Podman can return different/empty IDs across calls, causing token
			// binding conflicts on the server.
			agentID = a.daemonID
			if agentID == "" {
				agentID = a.machineID
			}
			if agentID == "" {
				agentID = a.hostName
			}
		}
	}
	a.hostID = agentID

	hostName := a.hostName
	if hostName == "" {
		hostName = info.Name
	}

	uptime := readSystemUptime()

	metricsCtx, metricsCancel := context.WithTimeout(ctx, 10*time.Second)
	snapshot, err := hostmetricsCollectWithDiskFilters(metricsCtx, a.cfg.DiskExclude, a.cfg.DiskInclude)
	metricsCancel()
	if err != nil {
		return agentsdocker.Report{}, fmt.Errorf("collect host metrics: %w", err)
	}

	collectContainers := a.cfg.IncludeContainers
	if !collectContainers && (a.cfg.IncludeServices || a.cfg.IncludeTasks) && !info.Swarm.ControlAvailable {
		collectContainers = true
	}

	var containers []agentsdocker.Container
	if collectContainers {
		var err error
		containers, err = a.collectContainers(ctx)
		if err != nil {
			return agentsdocker.Report{}, err
		}
	}

	services, tasks, nodes, secrets, configs, swarmInfo := a.collectSwarmData(ctx, info, containers)
	diskUsageResult, storageUsage, err := a.collectStorageUsage(ctx)
	if err != nil {
		a.logger.Warn().Err(err).Msg("failed to collect Docker storage usage")
	}
	images, err := a.collectImages(ctx, diskUsageResult.Images.Items)
	if err != nil {
		a.logger.Warn().Err(err).Msg("failed to collect Docker images")
	}
	volumes, err := a.collectVolumes(ctx, diskUsageResult.Volumes.Items)
	if err != nil {
		a.logger.Warn().Err(err).Msg("failed to collect Docker volumes")
	}
	networks, err := a.collectNetworks(ctx)
	if err != nil {
		a.logger.Warn().Err(err).Msg("failed to collect Docker networks")
	}

	// Use Docker's MemTotal, but fall back to gopsutil's reading if Docker returns 0.
	// This can happen in Docker-in-LXC setups where Docker daemon can't read host memory.
	totalMemory := info.MemTotal
	if totalMemory <= 0 && snapshot.Memory.TotalBytes > 0 {
		totalMemory = snapshot.Memory.TotalBytes
	}

	report := agentsdocker.Report{
		Agent: agentsdocker.AgentInfo{
			ID:              agentID,
			Version:         a.agentVersion,
			Type:            a.cfg.AgentType,
			IntervalSeconds: int(a.cfg.Interval / time.Second),
			Modules:         a.helperOperationModuleStatuses(),
		},
		Host: agentsdocker.HostInfo{
			Hostname:         hostName,
			Name:             info.Name,
			MachineID:        a.machineID,
			OS:               info.OperatingSystem,
			Runtime:          string(a.runtime),
			RuntimeVersion:   a.runtimeVer,
			KernelVersion:    info.KernelVersion,
			Architecture:     info.Architecture,
			DockerVersion:    info.ServerVersion,
			TotalCPU:         info.NCPU,
			TotalMemoryBytes: totalMemory,
			UptimeSeconds:    uptime,
			CPUUsagePercent:  safeFloat(snapshot.CPUUsagePercent),
			LoadAverage:      append([]float64(nil), snapshot.LoadAverage...),
			Memory:           snapshot.Memory,
			Disks:            append([]agentsdocker.Disk(nil), snapshot.Disks...),
			Network:          append([]agentsdocker.NetworkInterface(nil), snapshot.Network...),
			Security:         buildHostSecurityInfo(info),
		},
		SequenceID: a.nextReportSequenceID(),
		Timestamp:  time.Now().UTC(),
	}

	if swarmInfo != nil {
		report.Host.Swarm = swarmInfo
	}

	if a.cfg.IncludeContainers {
		report.Containers = containers
	}
	if len(images) > 0 {
		report.Images = images
	}
	if len(volumes) > 0 {
		report.Volumes = volumes
	}
	if len(networks) > 0 {
		report.Networks = networks
	}
	if a.cfg.IncludeServices && len(services) > 0 {
		report.Services = services
	}
	if a.cfg.IncludeTasks && len(tasks) > 0 {
		report.Tasks = tasks
	}
	if len(nodes) > 0 {
		report.Nodes = nodes
	}
	if len(secrets) > 0 {
		report.Secrets = secrets
	}
	if len(configs) > 0 {
		report.Configs = configs
	}
	if storageUsage != nil {
		report.StorageUsage = storageUsage
	}

	if report.Agent.IntervalSeconds <= 0 {
		report.Agent.IntervalSeconds = int(30 * time.Second / time.Second)
	}

	return report, nil
}

func buildHostSecurityInfo(info systemtypes.Info) *agentsdocker.HostSecurityInfo {
	authzPlugins := normalizedNonEmptyStrings(info.Plugins.Authorization)
	if len(authzPlugins) == 0 {
		return nil
	}
	return &agentsdocker.HostSecurityInfo{
		AuthorizationPlugins: authzPlugins,
	}
}

func normalizedNonEmptyStrings(values []string) []string {
	if len(values) == 0 {
		return nil
	}

	normalized := make([]string, 0, len(values))
	seen := make(map[string]struct{}, len(values))
	for _, value := range values {
		value = strings.TrimSpace(value)
		if value == "" {
			continue
		}
		if _, ok := seen[value]; ok {
			continue
		}
		seen[value] = struct{}{}
		normalized = append(normalized, value)
	}
	if len(normalized) == 0 {
		return nil
	}
	return normalized
}

func cloneStringMap(src map[string]string) map[string]string {
	if len(src) == 0 {
		return nil
	}
	out := make(map[string]string, len(src))
	for k, v := range src {
		out[k] = v
	}
	return out
}

func (a *Agent) collectContainers(ctx context.Context) ([]agentsdocker.Container, error) {
	options := dockerContainerListOptions{All: true}
	if len(a.stateFilters) > 0 {
		filterArgs := newDockerFilters()
		for _, state := range a.stateFilters {
			filterArgs.Add("status", state)
		}
		options.Filters = filterArgs
	}

	list, err := dockerCallWithRetry(ctx, dockerContainerListCallTimeout, func(callCtx context.Context) ([]containertypes.Summary, error) {
		return a.docker.ContainerList(callCtx, options)
	})
	if err != nil {
		return nil, fmt.Errorf("failed to list containers: %w", annotateDockerConnectionError(err))
	}

	containers := make([]agentsdocker.Container, 0, len(list))
	active := make(map[string]struct{}, len(list))
	activeImages := make(map[string]struct{}, len(list))
	for _, summary := range list {
		if len(a.allowedStates) > 0 {
			if _, ok := a.allowedStates[strings.ToLower(string(summary.State))]; !ok {
				continue
			}
		}

		// Skip backup containers created during updates - they're temporary
		if isBackupContainer(summary.Names) {
			continue
		}

		active[summary.ID] = struct{}{}
		activeImages[summary.ImageID] = struct{}{}

		container, err := a.collectContainer(ctx, summary)
		if err != nil {
			a.logger.Warn().Str("container", strings.Join(summary.Names, ",")).Err(err).Msg("failed to collect container stats")
			continue
		}
		containers = append(containers, container)
	}
	a.pruneStaleCPUSamples(active)
	a.pruneInactiveContainerInspectCache(active)
	a.registryChecker.pruneLocalImages(activeImages)
	return containers, nil
}

func (a *Agent) collectImages(ctx context.Context, usageImages []image.Summary) ([]agentsdocker.Image, error) {
	usageByID := make(map[string]image.Summary, len(usageImages))
	for _, summary := range usageImages {
		if id := strings.TrimSpace(summary.ID); id != "" {
			usageByID[id] = summary
		}
	}

	list, err := dockerCallWithRetry(ctx, dockerInventoryCallTimeout, func(callCtx context.Context) ([]image.Summary, error) {
		// Keep image identity and tags on the live telemetry cadence without
		// asking the daemon to recompute shared layer sizes every 30 seconds.
		// The throttled /system/df snapshot above already owns that expensive
		// calculation and supplies its cached values here (#1729).
		return a.docker.ImageList(callCtx, dockerImageListOptions{All: true})
	})
	if err != nil {
		return nil, annotateDockerConnectionError(err)
	}

	images := make([]agentsdocker.Image, 0, len(list))
	for _, summary := range list {
		sharedSize := summary.SharedSize
		containers := summary.Containers
		if usage, ok := usageByID[strings.TrimSpace(summary.ID)]; ok {
			sharedSize = usage.SharedSize
			containers = usage.Containers
		}
		// Docker uses -1 when these optional calculations were not requested.
		// Do not project that sentinel as a real negative byte/container count.
		sharedSize = max(sharedSize, 0)
		containers = max(containers, 0)
		images = append(images, agentsdocker.Image{
			ID:              strings.TrimSpace(summary.ID),
			RepoTags:        normalizedNonEmptyStrings(summary.RepoTags),
			RepoDigests:     normalizedNonEmptyStrings(summary.RepoDigests),
			SizeBytes:       summary.Size,
			SharedSizeBytes: sharedSize,
			Containers:      containers,
			CreatedAt:       dockerUnixTimestamp(summary.Created),
			Labels:          cloneStringMap(summary.Labels),
		})
	}
	return images, nil
}

func (a *Agent) collectVolumes(ctx context.Context, usageVolumes []volume.Volume) ([]agentsdocker.Volume, error) {
	usageByName := make(map[string]volume.UsageData, len(usageVolumes))
	for _, volume := range usageVolumes {
		name := strings.TrimSpace(volume.Name)
		if name == "" || volume.UsageData == nil {
			continue
		}
		usageByName[name] = *volume.UsageData
	}

	list, err := dockerCallWithRetry(ctx, dockerInventoryCallTimeout, func(callCtx context.Context) ([]volume.Volume, error) {
		return a.docker.VolumeList(callCtx, dockerVolumeListOptions{})
	})
	if err != nil {
		return nil, annotateDockerConnectionError(err)
	}

	volumes := make([]agentsdocker.Volume, 0, len(list))
	for _, v := range list {
		sizeBytes := int64(0)
		refCount := int64(0)
		if v.UsageData != nil {
			sizeBytes = v.UsageData.Size
			refCount = v.UsageData.RefCount
		} else if usage, ok := usageByName[strings.TrimSpace(v.Name)]; ok {
			sizeBytes = usage.Size
			refCount = usage.RefCount
		}
		volumes = append(volumes, agentsdocker.Volume{
			Name:       strings.TrimSpace(v.Name),
			Driver:     strings.TrimSpace(v.Driver),
			Mountpoint: strings.TrimSpace(v.Mountpoint),
			Scope:      strings.TrimSpace(v.Scope),
			CreatedAt:  strings.TrimSpace(v.CreatedAt),
			SizeBytes:  sizeBytes,
			RefCount:   refCount,
			Labels:     cloneStringMap(v.Labels),
			Options:    cloneStringMap(v.Options),
		})
	}
	return volumes, nil
}

func (a *Agent) collectNetworks(ctx context.Context) ([]agentsdocker.Network, error) {
	list, err := dockerCallWithRetry(ctx, dockerInventoryCallTimeout, func(callCtx context.Context) ([]networktypes.Summary, error) {
		return a.docker.NetworkList(callCtx, dockerNetworkListOptions{})
	})
	if err != nil {
		return nil, annotateDockerConnectionError(err)
	}

	networks := make([]agentsdocker.Network, 0, len(list))
	for _, n := range list {
		subnets := make([]agentsdocker.NetworkSubnet, 0, len(n.IPAM.Config))
		for _, config := range n.IPAM.Config {
			subnet := ""
			if config.Subnet.IsValid() {
				subnet = config.Subnet.String()
			}
			gateway := ""
			if config.Gateway.IsValid() {
				gateway = config.Gateway.String()
			}
			subnets = append(subnets, agentsdocker.NetworkSubnet{
				Subnet:  subnet,
				Gateway: gateway,
			})
		}
		networks = append(networks, agentsdocker.Network{
			ID:         strings.TrimSpace(n.ID),
			Name:       strings.TrimSpace(n.Name),
			Driver:     strings.TrimSpace(n.Driver),
			Scope:      strings.TrimSpace(n.Scope),
			CreatedAt:  n.Created,
			EnableIPv4: n.EnableIPv4,
			EnableIPv6: n.EnableIPv6,
			Internal:   n.Internal,
			Attachable: n.Attachable,
			Ingress:    n.Ingress,
			ConfigOnly: n.ConfigOnly,
			Subnets:    subnets,
			Labels:     cloneStringMap(n.Labels),
			Options:    cloneStringMap(n.Options),
		})
	}
	return networks, nil
}

func (a *Agent) collectStorageUsage(ctx context.Context) (client.DiskUsageResult, *agentsdocker.StorageUsage, error) {
	a.storageUsageMu.Lock()
	defer a.storageUsageMu.Unlock()

	now := time.Now()
	if !a.storageUsageCache.nextRefresh.IsZero() && now.Before(a.storageUsageCache.nextRefresh) {
		if a.storageUsageCache.valid {
			return a.storageUsageCache.result, cloneDockerStorageUsage(a.storageUsageCache.usage), nil
		}
		// The last refresh attempt already surfaced its error. Keep the
		// expensive scan suppressed until the bounded refresh window expires.
		return client.DiskUsageResult{}, nil, nil
	}
	a.storageUsageCache.nextRefresh = now.Add(dockerStorageUsageRefreshInterval)

	// A full daemon storage walk must not be retried immediately. If it times
	// out on an appliance daemon, the normal retry policy would double the load
	// and the next live telemetry tick would begin another walk.
	result, err := dockerCallWithRetryAttempts(ctx, dockerInventoryCallTimeout, 1, func(callCtx context.Context) (client.DiskUsageResult, error) {
		return a.docker.DiskUsage(callCtx, dockerDiskUsageOptions{
			Containers: true,
			Images:     true,
			Volumes:    true,
			BuildCache: true,
			Verbose:    true,
		})
	})
	if err != nil {
		annotatedErr := annotateDockerConnectionError(err)
		if a.storageUsageCache.valid {
			return a.storageUsageCache.result, cloneDockerStorageUsage(a.storageUsageCache.usage), annotatedErr
		}
		return client.DiskUsageResult{}, nil, annotatedErr
	}

	usage := &agentsdocker.StorageUsage{
		Images: agentsdocker.StorageUsageBucket{
			TotalCount:       result.Images.TotalCount,
			ActiveCount:      result.Images.ActiveCount,
			TotalSizeBytes:   result.Images.TotalSize,
			ReclaimableBytes: result.Images.Reclaimable,
		},
		Containers: agentsdocker.StorageUsageBucket{
			TotalCount:       result.Containers.TotalCount,
			ActiveCount:      result.Containers.ActiveCount,
			TotalSizeBytes:   result.Containers.TotalSize,
			ReclaimableBytes: result.Containers.Reclaimable,
		},
		Volumes: agentsdocker.StorageUsageBucket{
			TotalCount:       result.Volumes.TotalCount,
			ActiveCount:      result.Volumes.ActiveCount,
			TotalSizeBytes:   result.Volumes.TotalSize,
			ReclaimableBytes: result.Volumes.Reclaimable,
		},
		BuildCache: agentsdocker.StorageUsageBucket{
			TotalCount:       result.BuildCache.TotalCount,
			ActiveCount:      result.BuildCache.ActiveCount,
			TotalSizeBytes:   result.BuildCache.TotalSize,
			ReclaimableBytes: result.BuildCache.Reclaimable,
		},
	}
	a.storageUsageCache.result = result
	a.storageUsageCache.usage = cloneDockerStorageUsage(usage)
	a.storageUsageCache.valid = true
	a.storageUsageCache.nextRefresh = time.Now().Add(dockerStorageUsageRefreshInterval)
	return result, usage, nil
}

func cloneDockerStorageUsage(usage *agentsdocker.StorageUsage) *agentsdocker.StorageUsage {
	if usage == nil {
		return nil
	}
	cloned := *usage
	return &cloned
}

func dockerUnixTimestamp(seconds int64) time.Time {
	if seconds <= 0 {
		return time.Time{}
	}
	return time.Unix(seconds, 0).UTC()
}

func (a *Agent) pruneStaleCPUSamples(active map[string]struct{}) {
	a.cpuMu.Lock()
	defer a.cpuMu.Unlock()

	if len(a.prevContainerCPU) == 0 {
		return
	}

	for containerID := range a.prevContainerCPU {
		if _, ok := active[containerID]; !ok {
			delete(a.prevContainerCPU, containerID)
			// Reset stats failure counter when containers are removed,
			// though it's global per agent so not strictly necessary but good hygiene
		}
	}
}

func (a *Agent) collectContainer(ctx context.Context, summary containertypes.Summary) (agentsdocker.Container, error) {
	const perContainerTimeout = 15 * time.Second

	containerCtx, cancel := context.WithTimeout(ctx, perContainerTimeout)
	defer cancel()

	requestSize := a.cfg.CollectDiskMetrics
	inspect, err := a.inspectContainer(containerCtx, summary, requestSize)
	if err != nil {
		return agentsdocker.Container{}, fmt.Errorf("inspect: %w", err)
	}

	var (
		cpuPercent float64
		memUsage   int64
		memLimit   int64
		memPercent float64
		blockIO    *agentsdocker.ContainerBlockIO
		networkRX  uint64
		networkTX  uint64
	)

	if inspect.State.Running || inspect.State.Paused {
		statsResp, err := a.docker.ContainerStatsOneShot(containerCtx, summary.ID)
		if err != nil {
			return agentsdocker.Container{}, fmt.Errorf("stats: %w", err)
		}
		defer func() {
			if closeErr := statsResp.Body.Close(); closeErr != nil {
				a.logger.Warn().Err(closeErr).Str("container", summary.ID).Msg("Failed to close container stats response body")
			}
		}()

		payload, err := readBodyWithLimit(statsResp.Body, maxContainerStatsBodyBytes)
		if err != nil {
			return agentsdocker.Container{}, fmt.Errorf("read stats: %w", err)
		}

		stats, err := decodeContainerStatsPayload(payload)
		if err != nil {
			return agentsdocker.Container{}, fmt.Errorf("decode stats: %w", err)
		}

		// Podman's compatibility cpu percentage is not tied to the interval
		// between our one-shot reads. Use consecutive counter/read pairs for
		// every Podman sample, including idle and rebaseline samples. The result
		// stays on the 100%-per-core scale (#1391), as for Docker; only the server
		// converts it into a share of host capacity.
		cpuPercent = a.calculateContainerCPUPercent(summary.ID, stats, parseTime(inspect.State.StartedAt))
		memUsage, memLimit, memPercent = calculateMemoryUsage(stats)
		blockIO = summarizeBlockIO(stats)
		networkRX, networkTX = summarizeNetworkIO(stats)
	} else {
		a.cpuMu.Lock()
		delete(a.prevContainerCPU, summary.ID)
		a.cpuMu.Unlock()
	}
	a.logger.Debug().
		Str("container_id", summary.ID).
		Str("runtime", string(a.runtime)).
		Float64("cpu_percent", cpuPercent).
		Msg("Container CPU selected for report")

	createdAt := time.Unix(summary.Created, 0)

	startedAt := parseTime(inspect.State.StartedAt)
	finishedAt := parseTime(inspect.State.FinishedAt)

	uptimeSeconds := int64(0)
	if !startedAt.IsZero() && inspect.State.Running {
		uptimeSeconds = int64(time.Since(startedAt).Seconds())
		if uptimeSeconds < 0 {
			uptimeSeconds = 0
		}
	}

	health := ""
	if inspect.State.Health != nil {
		health = string(inspect.State.Health.Status)
	}
	healthcheckTargets := []string(nil)
	if inspect.Config != nil && inspect.Config.Healthcheck != nil {
		healthcheckTargets = extractHealthcheckTargets(inspect.Config.Healthcheck.Test)
	}

	ports := make([]agentsdocker.ContainerPort, len(summary.Ports))
	for i, port := range summary.Ports {
		ports[i] = agentsdocker.ContainerPort{
			PrivatePort: int(port.PrivatePort),
			PublicPort:  int(port.PublicPort),
			Protocol:    port.Type,
			IP:          addrString(port.IP),
		}
	}

	labels := make(map[string]string, len(summary.Labels))
	for k, v := range summary.Labels {
		labels[k] = v
	}

	networks := make([]agentsdocker.ContainerNetwork, 0)
	if inspect.NetworkSettings != nil {
		for name, cfg := range inspect.NetworkSettings.Networks {
			networks = append(networks, agentsdocker.ContainerNetwork{
				Name: name,
				IPv4: addrString(cfg.IPAddress),
				IPv6: addrString(cfg.GlobalIPv6Address),
			})
		}
	}

	var startedPtr, finishedPtr *time.Time
	if !startedAt.IsZero() {
		started := startedAt
		startedPtr = &started
	}
	if !finishedAt.IsZero() && !inspect.State.Running {
		finished := finishedAt
		finishedPtr = &finished
	}

	var writableLayerBytes int64
	if inspect.SizeRw != nil {
		writableLayerBytes = *inspect.SizeRw
	}

	var rootFsBytes int64
	if inspect.SizeRootFs != nil {
		rootFsBytes = *inspect.SizeRootFs
	}

	var mounts []agentsdocker.ContainerMount
	if len(inspect.Mounts) > 0 {
		mounts = make([]agentsdocker.ContainerMount, 0, len(inspect.Mounts))
		for _, mount := range inspect.Mounts {
			mounts = append(mounts, agentsdocker.ContainerMount{
				Type:        string(mount.Type),
				Source:      mount.Source,
				Destination: mount.Destination,
				Mode:        mount.Mode,
				RW:          mount.RW,
				Propagation: string(mount.Propagation),
				Name:        mount.Name,
				Driver:      mount.Driver,
			})
		}
	}
	// Docker's --tmpfs mounts can exist only in HostConfig.Tmpfs. Preserve
	// their configuration alongside inspected mounts without inventing usage.
	if inspect.HostConfig != nil && len(inspect.HostConfig.Tmpfs) > 0 {
		reported := make(map[string]bool, len(mounts))
		for _, mount := range mounts {
			reported[mount.Destination] = true
		}
		destinations := make([]string, 0, len(inspect.HostConfig.Tmpfs))
		for destination := range inspect.HostConfig.Tmpfs {
			if !reported[destination] {
				destinations = append(destinations, destination)
			}
		}
		sort.Strings(destinations)
		for _, destination := range destinations {
			options := inspect.HostConfig.Tmpfs[destination]
			writable := true
			for _, option := range strings.Split(options, ",") {
				switch strings.TrimSpace(option) {
				case "ro":
					writable = false
				case "rw":
					writable = true
				}
			}
			mounts = append(mounts, agentsdocker.ContainerMount{
				Type: "tmpfs", Destination: destination, Mode: options, RW: writable,
			})
		}
	}

	oomKilled := inspect.State.OOMKilled
	container := agentsdocker.Container{
		ID:                  summary.ID,
		Name:                trimLeadingSlash(summary.Names),
		Image:               summary.Image,
		ImageDigest:         summary.ImageID, // sha256:... digest of the image
		CreatedAt:           createdAt,
		State:               string(summary.State),
		Status:              summary.Status,
		Health:              health,
		HealthcheckTargets:  healthcheckTargets,
		CPUPercent:          cpuPercent,
		MemoryUsageBytes:    memUsage,
		MemoryLimitBytes:    memLimit,
		MemoryPercent:       memPercent,
		UptimeSeconds:       uptimeSeconds,
		RestartCount:        inspect.RestartCount,
		ExitCode:            inspect.State.ExitCode,
		OOMKilled:           &oomKilled,
		StartedAt:           startedPtr,
		FinishedAt:          finishedPtr,
		Ports:               ports,
		Labels:              labels,
		Env:                 maskSensitiveEnvVars(inspect.Config.Env),
		Networks:            networks,
		NetworkRXBytes:      networkRX,
		NetworkTXBytes:      networkTX,
		WritableLayerBytes:  writableLayerBytes,
		RootFilesystemBytes: rootFsBytes,
		BlockIO:             blockIO,
		Mounts:              mounts,
		Filesystems:         a.collectContainerFilesystems(containerCtx, summary.ID, inspect, mounts),
	}

	if a.runtime == RuntimePodman {
		if meta := extractPodmanMetadata(labels); meta != nil {
			container.Podman = meta
		}
	}

	// Check for image updates if registry checker is enabled
	if a.registryChecker != nil && a.registryChecker.Enabled() {
		// Get the actual manifest digest (RepoDigest) from the image for accurate comparison.
		// The ImageID is a local content-addressable ID that differs from the registry manifest digest.
		// We also get the architecture details to correctly resolve manifest lists from the registry.
		local, localKnown := a.localImageDigestsForUpdateCheck(containerCtx, summary.ImageID, summary.Image)
		currentDigests, arch, os, variant := local.digests, local.arch, local.os, local.variant
		digestForComparison := ""
		if len(currentDigests) > 0 {
			digestForComparison = currentDigests[0]
		}

		var imageToCheck string
		// Always prefer the image name from inspect config as it's the authoritative source
		// and avoids issues with short IDs or digests in summary.
		// HOWEVER, if the config image IS a digest (starts with sha256:), fall back to container.Image
		// which usually contains the human-readable tag/name.
		imageToCheck = container.Image
		if inspect.Config != nil && inspect.Config.Image != "" {
			if !strings.HasPrefix(inspect.Config.Image, "sha256:") {
				imageToCheck = inspect.Config.Image
			}
		}

		// Additional safety: if imageToCheck is still a SHA, we can't check it
		if strings.HasPrefix(imageToCheck, "sha256:") {
			container.UpdateStatus = &agentsdocker.UpdateStatus{
				UpdateAvailable: false,
				CurrentDigest:   digestForComparison,
				LastChecked:     time.Now(),
				Error:           "digest-pinned image",
			}
			// Skip to end of update check block - don't call registry
		} else if !localKnown {
			// Without the local digest no comparison is possible. Comparing an
			// empty digest reports "no update", which the server takes as an
			// affirmative clear: a pending update alert resolved and re-fired a
			// full delay later (#2353). Leave the status unknown this cycle.
			a.logger.Debug().
				Str("container", container.Name).
				Str("image", imageToCheck).
				Msg("Skipping update check: local image digest unavailable")
		} else if registry, _, _ := parseImageReference(imageToCheck); len(currentDigests) == 0 && registry != "" {
			// An image with no RepoDigests was built or loaded on this host and
			// never pulled, so no registry holds it to compare against. Asking
			// anyway reads as a failed check ("authentication required" from
			// Docker Hub for an unknown repository); report it as local.
			container.UpdateStatus = &agentsdocker.UpdateStatus{
				UpdateAvailable: false,
				LastChecked:     time.Now(),
				Error:           localImageUpdateStatus,
			}
		} else {
			a.logger.Debug().
				Str("container", container.Name).
				Str("image", imageToCheck).
				Str("compareDigest", digestForComparison).
				Str("arch", arch).
				Str("os", os).
				Str("variant", variant).
				Msg("Checking update for container")

			// Pass the whole local digest set. Docker records every RepoDigest
			// the image is known by, and a tag's manifest digest is often not
			// the first entry, so comparing only one digest reports a current
			// image as outdated (#2110).
			result := a.registryChecker.CheckImageUpdate(ctx, imageToCheck, strings.Join(currentDigests, ","), arch, os, variant)
			if result != nil {
				container.UpdateStatus = &agentsdocker.UpdateStatus{
					UpdateAvailable: result.UpdateAvailable,
					CurrentDigest:   result.CurrentDigest,
					LatestDigest:    result.LatestDigest,
					LastChecked:     result.CheckedAt,
					Error:           result.Error,
				}
			}
		}
	}

	if requestSize {
		a.logger.Debug().
			Str("container", container.Name).
			Int64("writableLayerBytes", writableLayerBytes).
			Int64("rootFilesystemBytes", rootFsBytes).
			Int("mountCount", len(mounts)).
			Msg("Collected container disk metrics")
	}

	return container, nil
}

func (a *Agent) inspectContainer(ctx context.Context, summary containertypes.Summary, requestSize bool) (containertypes.InspectResponse, error) {
	// Empty IDs/statuses do not provide enough evidence to detect a lifecycle
	// change between reports (some Podman compatibility endpoints omit Status),
	// so keep those on the live path rather than accepting bounded stale state.
	cacheable := strings.TrimSpace(summary.ID) != "" && strings.TrimSpace(summary.Status) != "" && isInactiveContainerState(summary.State)
	fingerprint := inactiveContainerInspectFingerprint(summary)
	if cacheable {
		a.inactiveInspectMu.Lock()
		entry, ok := a.inactiveInspects[summary.ID]
		if ok && entry.withSize == requestSize && entry.fingerprint == fingerprint && time.Now().Before(entry.expiresAt) {
			inspect := entry.inspect
			a.inactiveInspectMu.Unlock()
			return inspect, nil
		}
		a.inactiveInspectMu.Unlock()
	}

	inspect, _, err := a.docker.ContainerInspectWithRaw(ctx, summary.ID, requestSize)
	if err != nil {
		return containertypes.InspectResponse{}, err
	}

	a.inactiveInspectMu.Lock()
	defer a.inactiveInspectMu.Unlock()
	if !cacheable || inspect.State == nil || inspect.State.Running || inspect.State.Paused || inspect.State.Restarting {
		delete(a.inactiveInspects, summary.ID)
		return inspect, nil
	}
	if a.inactiveInspects == nil {
		a.inactiveInspects = make(map[string]inactiveContainerInspectCacheEntry)
	}
	a.inactiveInspects[summary.ID] = inactiveContainerInspectCacheEntry{
		inspect:     inspect,
		fingerprint: fingerprint,
		withSize:    requestSize,
		expiresAt:   time.Now().Add(dockerInactiveInspectRefreshInterval),
	}
	return inspect, nil
}

func isInactiveContainerState(state containertypes.ContainerState) bool {
	switch strings.ToLower(strings.TrimSpace(string(state))) {
	case "created", "exited", "dead":
		return true
	default:
		return false
	}
}

func inactiveContainerInspectFingerprint(summary containertypes.Summary) string {
	// Status includes the latest exit result/time, so an inactive container
	// that ran and stopped entirely between two reports cannot reuse its old
	// exit code, restart count, or lifecycle timestamps.
	return strings.ToLower(strings.TrimSpace(string(summary.State))) + "\x00" + strings.TrimSpace(summary.Status)
}

func (a *Agent) pruneInactiveContainerInspectCache(active map[string]struct{}) {
	a.inactiveInspectMu.Lock()
	defer a.inactiveInspectMu.Unlock()
	for containerID := range a.inactiveInspects {
		if _, ok := active[containerID]; !ok {
			delete(a.inactiveInspects, containerID)
		}
	}
}

func (a *Agent) clearDockerCollectionCaches() {
	a.storageUsageMu.Lock()
	a.storageUsageCache = dockerStorageUsageCache{}
	a.storageUsageMu.Unlock()

	a.inactiveInspectMu.Lock()
	a.inactiveInspects = nil
	a.inactiveInspectMu.Unlock()
}

var (
	healthcheckURLPattern    = regexp.MustCompile(`(?i)https?://[^\s'"<>]+`)
	healthcheckTargetPattern = regexp.MustCompile(`(?i)^[a-z0-9][a-z0-9._-]{0,252}$`)
)

// extractHealthcheckTargets derives only secret-free URL hostnames from a
// container health check. Raw health-check command text can contain paths,
// query strings, credentials, or environment expansion and must never cross
// the agent reporting boundary.
func extractHealthcheckTargets(test []string) []string {
	const maxTargets = 16
	seen := make(map[string]struct{})
	targets := make([]string, 0)
	for _, part := range test {
		for _, candidate := range healthcheckURLPattern.FindAllString(part, -1) {
			if len(candidate) > 2048 {
				continue
			}
			parsed, err := url.Parse(candidate)
			if err != nil {
				continue
			}
			target := strings.ToLower(strings.TrimSpace(parsed.Hostname()))
			if address, err := netip.ParseAddr(target); err == nil {
				target = address.String()
			} else if !healthcheckTargetPattern.MatchString(target) {
				continue
			}
			if _, exists := seen[target]; exists {
				continue
			}
			seen[target] = struct{}{}
			targets = append(targets, target)
			if len(targets) == maxTargets {
				return targets
			}
		}
	}
	if len(targets) == 0 {
		return nil
	}
	return targets
}

// getImageRepoDigest retrieves the RepoDigest for an image and its platform details.
// It returns the digest, architecture, OS, and variant.
func (a *Agent) getImageRepoDigest(ctx context.Context, imageID, imageName string) (string, string, string, string) {
	digests, arch, os, variant := a.getImageRepoDigests(ctx, imageID, imageName)
	if len(digests) == 0 {
		return "", arch, os, variant
	}
	return digests[0], arch, os, variant
}

// getImageRepoDigests returns every manifest digest the local image is known
// by, with digests whose repository matches the requested image reference
// first, plus the platform details. A single image can carry several
// RepoDigests when it was pulled under more than one reference (for example a
// tag and a platform manifest), so an update check must treat the whole set as
// the local identity rather than trusting one arbitrary entry (#2110).
func (a *Agent) getImageRepoDigests(ctx context.Context, imageID, imageName string) ([]string, string, string, string) {
	local, err := a.inspectLocalImageDigests(ctx, imageID, imageName)
	if err != nil {
		return nil, "", "", ""
	}
	return local.digests, local.arch, local.os, local.variant
}

// localImageDigestsForUpdateCheck returns the local digests to compare against
// the registry. Image inspect shares the per-container collection budget, so it
// can time out on a busy host; the last successful inspect of the same image
// ID stands in, since an image ID names immutable content. known is false only
// when neither is available.
func (a *Agent) localImageDigestsForUpdateCheck(ctx context.Context, imageID, imageName string) (localImageDigests, bool) {
	local, err := a.inspectLocalImageDigests(ctx, imageID, imageName)
	if err == nil {
		a.registryChecker.rememberLocalImage(imageID, local)
		return local, true
	}
	return a.registryChecker.lastLocalImage(imageID)
}

// inspectLocalImageDigests is getImageRepoDigests with the inspect failure
// kept distinct from an image that simply has no RepoDigests (a locally built
// image), so update checks can treat the first as unknown.
func (a *Agent) inspectLocalImageDigests(ctx context.Context, imageID, imageName string) (localImageDigests, error) {
	imageInspect, _, err := a.docker.ImageInspectWithRaw(ctx, imageID)
	if err != nil {
		a.logger.Debug().
			Err(err).
			Str("imageID", imageID).
			Str("imageName", imageName).
			Msg("Failed to inspect image for RepoDigest")
		return localImageDigests{}, err
	}

	local := localImageDigests{
		arch:    imageInspect.Architecture,
		os:      imageInspect.Os,
		variant: imageInspect.Variant,
	}

	if len(imageInspect.RepoDigests) == 0 {
		// Locally built images won't have RepoDigests
		return local, nil
	}

	matched := make([]string, 0, len(imageInspect.RepoDigests))
	unmatched := make([]string, 0, len(imageInspect.RepoDigests))
	// RepoDigests format: "registry/repo@sha256:..."
	for _, repoDigest := range imageInspect.RepoDigests {
		idx := strings.LastIndex(repoDigest, "@")
		if idx < 0 {
			continue
		}
		digest := strings.TrimSpace(repoDigest[idx+1:])
		if digest == "" {
			continue
		}
		if matchesImageReference(imageName, repoDigest[:idx]) {
			matched = append(matched, digest)
		} else {
			unmatched = append(unmatched, digest)
		}
	}

	local.digests = append(matched, unmatched...)
	return local, nil
}

func addrString(addr netip.Addr) string {
	if !addr.IsValid() {
		return ""
	}
	return addr.String()
}

// matchesImageReference checks if a RepoDigest repository matches an image reference.
// It handles Docker Hub's various naming conventions.
func matchesImageReference(imageName, repoRef string) bool {
	// Normalize image name by removing tag
	if idx := strings.LastIndex(imageName, ":"); idx >= 0 {
		// Make sure it's a tag, not a port (check if there's a / after it)
		if !strings.Contains(imageName[idx:], "/") {
			imageName = imageName[:idx]
		}
	}

	// Direct match
	if imageName == repoRef {
		return true
	}

	// Docker Hub library images: "nginx" == "docker.io/library/nginx"
	if repoRef == "docker.io/library/"+imageName {
		return true
	}

	// Docker Hub with namespace: "myuser/myapp" == "docker.io/myuser/myapp"
	if repoRef == "docker.io/"+imageName {
		return true
	}

	// Registry prefix matching (e.g., "ghcr.io/user/repo" matches "ghcr.io/user/repo")
	// Already handled by direct match above

	return false
}

func extractPodmanMetadata(labels map[string]string) *agentsdocker.PodmanContainer {
	if len(labels) == 0 {
		return nil
	}

	meta := &agentsdocker.PodmanContainer{}

	if v := strings.TrimSpace(labels["io.podman.annotations.pod.name"]); v != "" {
		meta.PodName = v
	}

	if v := strings.TrimSpace(labels["io.podman.annotations.pod.id"]); v != "" {
		meta.PodID = v
	}

	if v := strings.TrimSpace(labels["io.podman.annotations.pod.infra"]); v != "" {
		if parsed, err := strconv.ParseBool(v); err == nil {
			meta.Infra = parsed
		} else if strings.EqualFold(v, "yes") || strings.EqualFold(v, "true") {
			meta.Infra = true
		}
	}

	if v := strings.TrimSpace(labels["io.podman.compose.project"]); v != "" {
		meta.ComposeProject = v
	}

	if v := strings.TrimSpace(labels["io.podman.compose.service"]); v != "" {
		meta.ComposeService = v
	}

	if v := strings.TrimSpace(labels["io.podman.compose.working_dir"]); v != "" {
		meta.ComposeWorkdir = v
	}

	if v := strings.TrimSpace(labels["io.podman.compose.config-hash"]); v != "" {
		meta.ComposeConfig = v
	}

	if v := strings.TrimSpace(labels["io.containers.autoupdate"]); v != "" {
		meta.AutoUpdatePolicy = v
	}

	if v := strings.TrimSpace(labels["io.containers.autoupdate.restart"]); v != "" {
		meta.AutoUpdateRestart = v
	}

	if v := strings.TrimSpace(labels["io.podman.annotations.userns"]); v != "" {
		meta.UserNS = v
	} else if v := strings.TrimSpace(labels["io.containers.userns"]); v != "" {
		meta.UserNS = v
	}

	if meta.PodName == "" && meta.PodID == "" && meta.ComposeProject == "" && meta.AutoUpdatePolicy == "" && meta.UserNS == "" && !meta.Infra {
		return nil
	}

	return meta
}

func decodeContainerStatsPayload(payload []byte) (containertypes.StatsResponse, error) {
	var stats containertypes.StatsResponse
	if err := json.Unmarshal(payload, &stats); err != nil {
		return containertypes.StatsResponse{}, err
	}
	return stats, nil
}

func (a *Agent) calculateContainerCPUPercent(containerID string, stats containertypes.StatsResponse, startedAt time.Time) float64 {
	a.cpuMu.Lock()
	defer a.cpuMu.Unlock()

	current := cpuSample{
		totalUsage:  stats.CPUStats.CPUUsage.TotalUsage,
		systemUsage: stats.CPUStats.SystemUsage,
		onlineCPUs:  stats.CPUStats.OnlineCPUs,
		read:        stats.Read,
		startedAt:   startedAt,
	}

	// Always use manual delta tracking. Docker's PreCPUStats is unreliable for
	// one-shot stats because many Docker versions don't update their internal
	// cache between non-streaming reads, causing PreCPUStats to remain stale
	// (from container start) and producing a constant lifetime-average CPU%
	// instead of a current value.
	prev, ok := a.prevContainerCPU[containerID]
	if a.runtime == RuntimePodman && ok && !prev.startedAt.IsZero() && !current.startedAt.IsZero() && !current.startedAt.Equal(prev.startedAt) {
		// A new lifetime must establish its own baseline even if the runtime's
		// clock has moved back. Timestamp ordering only compares one lifetime.
		a.prevContainerCPU[containerID] = current
		return 0
	}
	if a.runtime == RuntimePodman && ok && !prev.read.IsZero() && !current.read.IsZero() && !current.read.After(prev.read) {
		// Repeated or out-of-order responses must not move the baseline back
		// and manufacture usage in the next interval.
		return 0
	}
	if a.prevContainerCPU == nil {
		a.prevContainerCPU = make(map[string]cpuSample)
	}
	if !ok {
		// First time seeing this container - store current sample and return 0
		// On next collection cycle we'll have a previous sample to compare against
		a.prevContainerCPU[containerID] = current
		a.logger.Debug().
			Str("container_id", containerID[:12]).
			Uint64("total_usage", current.totalUsage).
			Uint64("system_usage", current.systemUsage).
			Msg("First CPU sample collected, no previous data for delta calculation")
		return 0
	}

	// We have a previous sample - update it after calculation
	a.prevContainerCPU[containerID] = current
	if a.runtime == RuntimePodman && current.totalUsage < prev.totalUsage {
		// A counter reset is not an interval sample. Start again from this baseline.
		return 0
	}

	var totalDelta float64
	if current.totalUsage >= prev.totalUsage {
		totalDelta = float64(current.totalUsage - prev.totalUsage)
	} else {
		// Counter likely reset (container restart); fall back to current reading.
		totalDelta = float64(current.totalUsage)
	}

	if totalDelta <= 0 {
		return 0
	}

	if a.runtime == RuntimePodman {
		if !prev.read.IsZero() && !current.read.IsZero() {
			elapsed := current.read.Sub(prev.read).Seconds()
			if elapsed > 0 {
				denominator := elapsed * 1e9
				if denominator > 0 {
					cpuPercent := (totalDelta / denominator) * 100.0
					result := safeFloat(cpuPercent)
					a.logger.Debug().
						Str("container_id", containerID[:12]).
						Float64("cpu_percent", result).
						Float64("total_delta", totalDelta).
						Float64("elapsed_seconds", elapsed).
						Msg("CPU calculated from Podman wall-clock delta")
					return result
				}
			}
		}

		a.logger.Debug().
			Str("container_id", containerID[:12]).
			Float64("total_delta", totalDelta).
			Bool("prev_read_zero", prev.read.IsZero()).
			Bool("current_read_zero", current.read.IsZero()).
			Msg("Podman CPU calculation failed: no valid wall-clock delta available")
		return 0
	}

	onlineCPUs := current.onlineCPUs
	if onlineCPUs == 0 {
		onlineCPUs = prev.onlineCPUs
	}
	if onlineCPUs == 0 && a.cpuCount > 0 {
		onlineCPUs = uint32(a.cpuCount)
	}
	if onlineCPUs == 0 {
		return 0
	}

	var systemDelta float64
	if current.systemUsage >= prev.systemUsage {
		systemDelta = float64(current.systemUsage - prev.systemUsage)
	}
	// If systemUsage went backward (counter reset), leave systemDelta as 0
	// to fall through to time-based calculation below

	if systemDelta > 0 {
		cpuPercent := safeFloat((totalDelta / systemDelta) * float64(onlineCPUs) * 100.0)
		a.logger.Debug().
			Str("container_id", containerID[:12]).
			Float64("cpu_percent", cpuPercent).
			Float64("total_delta", totalDelta).
			Float64("system_delta", systemDelta).
			Uint32("online_cpus", onlineCPUs).
			Msg("CPU calculated from system delta")
		return cpuPercent
	}

	// Fall back to time-based calculation
	if !prev.read.IsZero() && !current.read.IsZero() {
		elapsed := current.read.Sub(prev.read).Seconds()
		if elapsed > 0 {
			denominator := elapsed * float64(onlineCPUs) * 1e9
			if denominator > 0 {
				cpuPercent := (totalDelta / denominator) * 100.0
				result := safeFloat(cpuPercent)
				a.logger.Debug().
					Str("container_id", containerID[:12]).
					Float64("cpu_percent", result).
					Float64("total_delta", totalDelta).
					Float64("elapsed_seconds", elapsed).
					Uint32("online_cpus", onlineCPUs).
					Msg("CPU calculated from time-based delta")
				return result
			}
		}
	}

	a.logger.Debug().
		Str("container_id", containerID[:12]).
		Float64("total_delta", totalDelta).
		Float64("system_delta", systemDelta).
		Bool("prev_read_zero", prev.read.IsZero()).
		Bool("current_read_zero", current.read.IsZero()).
		Msg("CPU calculation failed: no valid delta method available")
	return 0
}

func calculateCPUPercent(stats containertypes.StatsResponse, hostCPUs int) float64 {
	totalDelta := float64(stats.CPUStats.CPUUsage.TotalUsage - stats.PreCPUStats.CPUUsage.TotalUsage)
	systemDelta := float64(stats.CPUStats.SystemUsage - stats.PreCPUStats.SystemUsage)

	if totalDelta <= 0 || systemDelta <= 0 {
		return 0
	}

	onlineCPUs := stats.CPUStats.OnlineCPUs
	if onlineCPUs == 0 {
		onlineCPUs = uint32(len(stats.CPUStats.CPUUsage.PercpuUsage))
	}
	if onlineCPUs == 0 && hostCPUs > 0 {
		onlineCPUs = uint32(hostCPUs)
	}

	if onlineCPUs == 0 {
		return 0
	}

	return safeFloat((totalDelta / systemDelta) * float64(onlineCPUs) * 100.0)
}

func calculateMemoryUsage(stats containertypes.StatsResponse) (usage int64, limit int64, percent float64) {
	usage = int64(stats.MemoryStats.Usage)

	// Subtract reclaimable cache from usage to match `docker stats` behavior.
	// Docker subtracts cache/file to show "actual" memory usage rather than
	// memory.current which includes reclaimable filesystem cache.
	//
	// cgroup v1: "cache" stat contains the reclaimable cache
	// cgroup v2: "cache" doesn't exist, use "inactive_file" (preferred) or "file"
	var cacheBytes uint64
	if cache, ok := stats.MemoryStats.Stats["cache"]; ok {
		// cgroup v1
		cacheBytes = cache
	} else if inactiveFile, ok := stats.MemoryStats.Stats["inactive_file"]; ok {
		// cgroup v2: inactive_file is the reclaimable portion of file cache
		// This matches what docker CLI does internally
		cacheBytes = inactiveFile
	}

	if cacheBytes > 0 && int64(cacheBytes) < usage {
		usage -= int64(cacheBytes)
	}

	limit = int64(stats.MemoryStats.Limit)
	if limit > 0 {
		percent = (float64(usage) / float64(limit)) * 100.0
	}

	return usage, limit, safeFloat(percent)
}

func safeFloat(val float64) float64 {
	if math.IsNaN(val) || math.IsInf(val, 0) {
		return 0
	}
	return val
}

func parseTime(value string) time.Time {
	if value == "" || value == "0001-01-01T00:00:00Z" {
		return time.Time{}
	}
	if strings.Contains(value, ".") {
		if t, err := time.Parse(time.RFC3339Nano, value); err == nil {
			return t
		}
	} else {
		if t, err := time.Parse(time.RFC3339, value); err == nil {
			return t
		}
	}
	return time.Time{}
}

func trimLeadingSlash(names []string) string {
	if len(names) == 0 {
		return ""
	}
	name := names[0]
	return strings.TrimPrefix(name, "/")
}

func randomDuration(max time.Duration) time.Duration {
	if max <= 0 {
		return 0
	}

	n, err := randIntFn(rand.Reader, big.NewInt(int64(max)))
	if err != nil {
		return 0
	}

	return time.Duration(n.Int64())
}

func summarizeBlockIO(stats containertypes.StatsResponse) *agentsdocker.ContainerBlockIO {
	if len(stats.BlkioStats.IoServiceBytesRecursive) == 0 {
		return nil
	}

	var readBytes, writeBytes uint64
	var readPresent, writePresent bool

	for _, entry := range stats.BlkioStats.IoServiceBytesRecursive {
		op := strings.ToLower(entry.Op)
		switch op {
		case "read":
			readPresent = true
			readBytes += entry.Value
		case "write":
			writePresent = true
			writeBytes += entry.Value
		}
	}

	if !readPresent && !writePresent {
		return nil
	}

	return &agentsdocker.ContainerBlockIO{
		ReadBytes:         readBytes,
		WriteBytes:        writeBytes,
		ReadBytesPresent:  &readPresent,
		WriteBytesPresent: &writePresent,
	}
}

func summarizeNetworkIO(stats containertypes.StatsResponse) (uint64, uint64) {
	if len(stats.Networks) == 0 {
		return 0, 0
	}

	var rxBytes uint64
	var txBytes uint64
	for _, network := range stats.Networks {
		rxBytes += network.RxBytes
		txBytes += network.TxBytes
	}

	return rxBytes, txBytes
}

// sensitiveEnvPatterns are substrings that, when found in an env var name (case-insensitive),
// indicate the value should be masked for security.
var sensitiveEnvPatterns = []string{
	"password", "passwd", "secret", "key", "token", "credential", "auth",
	"api_key", "apikey", "private", "access_token", "refresh_token",
	"database_url", "connection_string", "encryption",
}

// maskSensitiveEnvVars returns a copy of the environment variables with sensitive values masked.
// Environment variables whose names contain sensitive keywords will have their values replaced with "***".
func maskSensitiveEnvVars(envVars []string) []string {
	if len(envVars) == 0 {
		return nil
	}

	result := make([]string, 0, len(envVars))
	for _, env := range envVars {
		parts := strings.SplitN(env, "=", 2)
		if len(parts) != 2 {
			result = append(result, env)
			continue
		}

		name := parts[0]
		value := parts[1]

		// Check if the environment variable name contains a sensitive pattern
		lowerName := strings.ToLower(name)
		isSensitive := false
		for _, pattern := range sensitiveEnvPatterns {
			if strings.Contains(lowerName, pattern) {
				isSensitive = true
				break
			}
		}

		if isSensitive && value != "" {
			result = append(result, name+"=***")
		} else {
			result = append(result, env)
		}
	}

	return result
}