Skip to content

Modern Docker Management, Designed for Everyone

18.2 KB
Raw Blame
package bootstrap

import (
	"cmp"
	"context"
	"errors"
	"fmt"
	"io"
	"log/slog"
	"net/http"
	"os"
	"strings"
	"time"

	httpxtypes "github.com/getarcaneapp/arcane/types/v2/httpx"
	"github.com/labstack/echo/v5"
	"github.com/moby/moby/client"
	"github.com/subosito/gotenv"
	kit "go.getarcane.app/kit/pkg"
	"go.getarcane.app/sys/crypto"
	"go.opentelemetry.io/contrib/bridges/otelslog"
	"go.uber.org/fx"
	"go.uber.org/fx/fxevent"

	"github.com/getarcaneapp/arcane/backend/v2/internal/apikey"
	"github.com/getarcaneapp/arcane/backend/v2/internal/auth"
	"github.com/getarcaneapp/arcane/backend/v2/internal/config"
	"github.com/getarcaneapp/arcane/backend/v2/internal/container"
	"github.com/getarcaneapp/arcane/backend/v2/internal/database"
	"github.com/getarcaneapp/arcane/backend/v2/internal/di"
	"github.com/getarcaneapp/arcane/backend/v2/internal/docker"
	"github.com/getarcaneapp/arcane/backend/v2/internal/environment"
	"github.com/getarcaneapp/arcane/backend/v2/internal/gitops"
	"github.com/getarcaneapp/arcane/backend/v2/internal/image"
	"github.com/getarcaneapp/arcane/backend/v2/internal/project"
	"github.com/getarcaneapp/arcane/backend/v2/internal/role"
	"github.com/getarcaneapp/arcane/backend/v2/internal/settings"
	"github.com/getarcaneapp/arcane/backend/v2/internal/swarm"
	"github.com/getarcaneapp/arcane/backend/v2/internal/user"
	"github.com/getarcaneapp/arcane/backend/v2/internal/variable"
	"github.com/getarcaneapp/arcane/backend/v2/internal/volume"
	"github.com/getarcaneapp/arcane/backend/v2/internal/vulnerability"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/edge"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/startup"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/libarcane/timeouts"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/utils"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/httpx"
	"github.com/getarcaneapp/arcane/backend/v2/pkg/utils/tracing"
)

func Bootstrap(ctx context.Context) error {
	if err := gotenv.Load(); err != nil && !errors.Is(err, os.ErrNotExist) {
		return fmt.Errorf("load .env: %w", err)
	}
	cfg := config.Load()
	cfg.DockerHost = startup.ResolveDockerHost(cfg.DockerHost, os.Stat)
	runtimeIdentityCfg := &startup.RuntimeIdentityConfig{
		PUID:              cfg.PUID,
		PGID:              cfg.PGID,
		DockerHost:        cfg.DockerHost,
		DockerConfig:      cfg.DockerConfig,
		DatabaseURL:       cfg.DatabaseURL,
		ProjectsDirectory: cfg.ProjectsDirectory,
	}
	if err := startup.ApplyRequestedRuntimeIdentity(ctx, runtimeIdentityCfg); err != nil {
		return fmt.Errorf("apply runtime identity: %w", err)
	}
	cfg.DockerConfig = runtimeIdentityCfg.DockerConfig

	SetupSlogLogger(cfg, nil)
	database.SetGormLogger(BuildGormLogger(cfg))
	slog.InfoContext(ctx, "Arcane is starting...", "version", config.Version)
	startup.ApplyMemoryLimit(ctx)
	slog.InfoContext(ctx, "Arcane Identity Configuration", "puid", os.Getuid(), "pgid", os.Getgid())

	appCtx, cancelApp := context.WithCancelCause(ctx)
	appCtx = utils.WithAppLifecycleContext(appCtx)

	db, err := database.Initialize(appCtx, cfg.DatabaseURL, database.MigrationOptions{AllowDowngrade: cfg.AllowDowngrade})
	if err != nil {
		cancelApp(nil)
		return fmt.Errorf("failed to initialize database: %w", err)
	}
	slog.InfoContext(appCtx, "Database initialized successfully")
	defer func() {
		cancelApp(nil)
		if closeErr := db.Close(); closeErr != nil {
			slog.ErrorContext(ctx, "Error closing database", "error", closeErr)
		}
	}()

	// Telemetry identifies this installation by its persisted instance ID, so it starts once the
	// database is open but before Docker clients and services. Exporter errors go to the console-only logger.
	instanceID, err := settings.EnsureInstanceID(appCtx, db)
	if err != nil {
		return err
	}
	providers, err := setupTelemetry(ctx, cfg, slog.Default(), instanceID)
	if err != nil {
		return fmt.Errorf("setup telemetry: %w", err)
	}
	defer func() {
		shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 4*time.Second)
		defer cancel()
		if shutdownErr := providers.Shutdown(shutdownCtx); shutdownErr != nil {
			slog.ErrorContext(shutdownCtx, "Error shutting down telemetry", "error", shutdownErr)
		}
	}()
	if providers.loggerProvider != nil {
		SetupSlogLogger(cfg, otelslog.NewHandler(tracing.InstrumentationName, otelslog.WithLoggerProvider(providers.loggerProvider)))
		db.Logger = BuildGormLogger(cfg)
	}
	if providers.tracerProvider != nil || providers.meterProvider != nil || providers.loggerProvider != nil {
		slog.InfoContext(ctx, "OpenTelemetry export enabled", "traces", providers.tracerProvider != nil, "metrics", providers.meterProvider != nil, "logs", providers.loggerProvider != nil)
	}

	app := fx.New(applicationOptions(appCtx, cfg, db, cancelApp))

	startCtx, cancelStart := context.WithTimeout(ctx, app.StartTimeout())
	defer cancelStart()
	if startErr := app.Start(startCtx); startErr != nil {
		return fmt.Errorf("start application: %w", startErr)
	}

	// appCtx also ends when a component stops the app itself; its cause is then the run error.
	var runErr error
	select {
	case <-appCtx.Done():
		if ctx.Err() == nil {
			runErr = context.Cause(appCtx)
		} else {
			slog.InfoContext(appCtx, "Context canceled")
		}
	case signal := <-app.Done():
		slog.InfoContext(appCtx, "Received shutdown signal", "signal", signal)
	}

	stopCtx, cancelStop := context.WithTimeout(context.WithoutCancel(ctx), app.StopTimeout())
	defer cancelStop()
	if stopErr := app.Stop(stopCtx); stopErr != nil {
		return errors.Join(runErr, fmt.Errorf("stop application: %w", stopErr))
	}

	slog.InfoContext(context.WithoutCancel(appCtx), "Arcane shutdown complete")
	return runErr
}

func applicationOptions(appCtx context.Context, cfg *config.Config, db *database.DB, cancelApp context.CancelCauseFunc) fx.Option {
	return fx.Options(
		fx.Supply(cfg, db, cancelApp),
		fx.Provide(
			func() context.Context { return appCtx },
			func(settingsService *settings.SettingsService) *http.Client {
				return httpx.NewHTTPClient(httpxtypes.ClientOptions{
					Timeout:             timeouts.GetDuration(settingsService.GetSettingsOrDefaults(appCtx).HTTPClientTimeout.AsInt(), timeouts.DefaultHTTPClient),
					TLSHandshakeTimeout: 10 * time.Second,
				})
			},
		),
		di.ActorOptions,
		di.ServiceOptions,
		di.JobOptions,
		serverOptions,
		fx.Invoke(
			registerAppCancelHook,
			initializeStartupState,
			registerJobs,
			startEdgeTunnelClient,
			startGlobalVariableSync,
		),
		fx.WithLogger(func() fxevent.Logger {
			logger := &fxevent.SlogLogger{Logger: slog.Default()}
			logger.UseLogLevel(slog.LevelDebug)
			return logger
		}),
		fx.StartTimeout(5*time.Minute),
		fx.StopTimeout(7*time.Second),
	)
}

// isWeakProductionEncryptionKey reports whether a production ENCRYPTION_KEY is an
// unprefixed passphrase shorter than 32 characters.
func isWeakProductionEncryptionKey(encryptionKey, localEnvironment string, agentMode bool) bool {
	if localEnvironment != "production" || agentMode {
		return false
	}
	key := strings.TrimSpace(encryptionKey)
	if key == "" || strings.HasPrefix(key, "hex:") || strings.HasPrefix(key, "base64:") {
		return false
	}
	return len(strings.TrimPrefix(key, "raw:")) < 32
}

type initializeStartupStateParams struct {
	fx.In

	AppCtx context.Context
	Config *config.Config
	DB     *database.DB

	Volume        *volume.VolumeService
	Settings      *settings.SettingsService
	Environment   *environment.EnvironmentService
	GitOpsSync    *gitops.GitOpsSyncService
	Project       *project.ProjectService
	Container     *container.ContainerService
	Image         *image.ImageService
	Variable      *variable.VariableService
	Docker        *docker.DockerClientService
	Swarm         *swarm.SwarmService
	Role          *role.RoleService
	Auth          *auth.AuthService
	User          *user.UserService
	ApiKey        *apikey.ApiKeyService
	Vulnerability *vulnerability.VulnerabilityService
}

const startupTaskTimeout = 30 * time.Second

func initializeStartupState(p initializeStartupStateParams) {
	appCtx := p.AppCtx
	cfg := p.Config

	helperCtx, cancelHelpers := context.WithTimeout(appCtx, startupTaskTimeout)
	startup.CleanupOrphanedVolumeHelpers(helperCtx, p.Volume.CleanupOrphanedVolumeHelpers)
	cancelHelpers()

	runtimeCfg := &startup.RuntimeConfig{
		AgentMode:         cfg.AgentMode,
		AgentToken:        cfg.AgentToken,
		Environment:       string(cfg.Environment),
		EncryptionKey:     cfg.EncryptionKey,
		AutoLoginUsername: cfg.AutoLoginUsername,
		AdminStaticAPIKey: cfg.AdminStaticAPIKey,
	}

	for _, envName := range cfg.DeprecatedEnvVarsSet() {
		slog.WarnContext(appCtx, "Deprecated environment variable is set and no longer used; remove it from your environment", "env", envName)
	}

	startup.LoadAgentToken(appCtx, runtimeCfg, p.Settings.GetStringSetting)
	startup.EnsureEncryptionKey(appCtx, runtimeCfg, p.Settings.EnsureEncryptionKey)
	cfg.AgentToken = runtimeCfg.AgentToken
	cfg.EncryptionKey = runtimeCfg.EncryptionKey

	if isWeakProductionEncryptionKey(cfg.EncryptionKey, string(cfg.Environment), cfg.AgentMode) {
		panic("ENCRYPTION_KEY passphrase must be at least 32 characters in production (or use a hex:/base64: encoded 32-byte key)")
	}

	crypto.InitEncryption(&crypto.Config{
		EncryptionKey: cfg.EncryptionKey,
		Environment:   string(cfg.Environment),
		AgentMode:     cfg.AgentMode,
	})
	if reloadErr := p.Settings.LoadDatabaseSettings(appCtx); reloadErr != nil {
		slog.WarnContext(appCtx, "Failed to reload settings after initializing encryption", "error", reloadErr)
	}
	startup.InitializeDefaultSettings(appCtx, runtimeCfg, p.Settings)

	// Startup repairs are best-effort: log each failure and continue.
	for _, task := range []struct {
		name string
		run  func(context.Context) error
	}{
		{name: "normalize projects directory", run: func(ctx context.Context) error {
			return p.Settings.NormalizeProjectsDirectory(ctx, cfg.ProjectsDirectory)
		}},
		{name: "normalize builds directory", run: p.Settings.NormalizeBuildsDirectory},
		{name: "encrypt stored sensitive settings", run: p.Settings.EncryptStoredSensitiveSettings},
		{name: "ensure local environment", run: func(ctx context.Context) error { return p.Environment.EnsureLocalEnvironment(ctx, cfg.AppUrl) }},
		{name: "clean up orphaned GitOps syncs", run: p.GitOpsSync.CleanupOrphanedSyncsOnStartup},
		{name: "clean up leaked GitOps scratch directories", run: p.GitOpsSync.CleanupLeakedScratchDirsOnStartup},
		{name: "clean up leaked git clone directories", run: p.GitOpsSync.CleanupLeakedCloneDirsOnStartup},
		{name: "reconcile directory GitOps projects", run: p.GitOpsSync.ReconcileDirectorySyncProjectsOnStartup},
		{name: "recover interrupted project renames", run: p.Project.RecoverProjectRenameJournals},
		{name: "recover interrupted container redeploys", run: p.Container.RecoverInterruptedRedeploys},
		{name: "recover stuck project statuses", run: p.Project.RecoverStuckProjectStatuses},
		{name: "fail interrupted image patches", run: p.Image.RecoverInterruptedPatches},
	} {
		taskCtx, cancelTask := context.WithTimeout(appCtx, startupTaskTimeout)
		if err := task.run(taskCtx); err != nil {
			slog.WarnContext(appCtx, "Startup task failed", "task", task.name, "error", err)
		}
		cancelTask()
	}
	p.Vulnerability.ImportLegacyReportFiles(appCtx)
	p.Vulnerability.BackfillScanItems(appCtx)

	if !cfg.AgentMode {
		if err := p.Environment.ReconcileEdgeStatusesOnStartup(appCtx); err != nil {
			slog.WarnContext(appCtx, "Failed to reconcile edge environment statuses on startup", "error", err)
		}
		p.Environment.RegisterMetrics()

		// Global variables are manager-owned: import a legacy local .env.global once, then sync everywhere.
		p.Environment.SetVariableSyncer(p.Variable)
		p.Environment.SetUserInvalidator(func(userID string) {
			p.Role.InvalidateUser(userID)
			p.Auth.InvalidateUserTokenCache(userID)
		})
		if err := p.Variable.ImportLegacyLocalEnvFile(appCtx); err != nil {
			slog.WarnContext(appCtx, "Failed to import legacy global variables", "error", err)
		}
	}

	dockerCtx, cancelDocker := context.WithTimeout(appCtx, startupTaskTimeout)
	defer cancelDocker()
	startup.TestDockerConnection(dockerCtx, func(ctx context.Context) error {
		_, err := p.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (struct{}, error) {
			version, err := dockerClient.ServerVersion(ctx, client.ServerVersionOptions{})
			if err != nil {
				return struct{}{}, err
			}

			effectiveAPIVersion := cmp.Or(strings.TrimSpace(dockerClient.ClientVersion()), strings.TrimSpace(version.APIVersion))
			slog.InfoContext(ctx, "Docker API versions detected", "clientApiVersion", dockerClient.ClientVersion(), "serverApiVersion", version.APIVersion, "effectiveApiVersion", effectiveAPIVersion)
			return struct{}{}, nil
		})
		return err
	})
	startup.WarnDeprecatedImage(dockerCtx, func(ctx context.Context) (string, error) {
		return p.Docker.Call(ctx, func(ctx context.Context, dockerClient *client.Client) (string, error) {
			self, err := libarcane.InspectCurrentArcaneContainer(ctx, dockerClient)
			if err != nil {
				return "", err
			}
			return kit.FromPtr(self.Config).Image, nil
		})
	})
	if err := p.Swarm.SyncSwarmEnabledState(dockerCtx); err != nil {
		slog.WarnContext(appCtx, "Failed to persist swarm enabled state", "error", err)
	}

	startup.InitializeNonAgentFeatures(appCtx, runtimeCfg,
		p.Role.EnsureBuiltInRoles,
		func(ctx context.Context) error {
			// Backfill legacy users.roles first so CreateDefaultAdmin's zero-global-admin gate sees them;
			// the role startup tasks below repeat it for agent mode and retries.
			if err := p.Role.BackfillLegacyRoleAssignments(ctx); err != nil {
				slog.WarnContext(ctx, "Failed to backfill legacy role assignments before admin bootstrap", "error", err)
			}
			return p.User.CreateDefaultAdmin(ctx)
		},
		func(ctx context.Context) error {
			return p.ApiKey.ReconcileDefaultAdminAPIKey(ctx, runtimeCfg.AdminStaticAPIKey)
		},
		func(ctx context.Context) error {
			startup.InitializeAutoLogin(ctx, runtimeCfg)
			return nil
		},
	)
	startup.CleanupUnknownSettings(appCtx, p.Settings)

	if err := p.Role.EnsureBuiltInRoles(appCtx); err != nil {
		slog.ErrorContext(appCtx, "Failed to reconcile built-in roles", "error", err)
	}
	// Runs after EnsureBuiltInRoles seeds role IDs and before the API key backfill and admin guard read assignments.
	if err := p.Role.BackfillLegacyRoleAssignments(appCtx); err != nil {
		slog.ErrorContext(appCtx, "Failed to backfill legacy users.roles into user_role_assignments", "error", err)
	}
	if err := p.ApiKey.BackfillApiKeyPermissions(appCtx); err != nil {
		slog.WarnContext(appCtx, "Failed to backfill API key permissions", "error", err)
	}
	if err := p.Role.ReconcileEnvOidcMappings(appCtx, cfg.OidcRoleMappings); err != nil {
		slog.ErrorContext(appCtx, "Failed to reconcile OIDC_ROLE_MAPPINGS", "error", err)
	}
	if !cfg.AgentMode {
		if err := p.Role.AssertGlobalAdminExists(appCtx, p.DB.WithContext(appCtx)); err != nil {
			slog.ErrorContext(appCtx, "RBAC global admin guard failed", "error", err)
		}
	}

	// Only edge agents auto-pair; direct agents are passive and the manager dials them.
	if cfg.AgentMode && cfg.EdgeAgent && cfg.AgentToken != "" && cfg.ManagerApiUrl != "" {
		if err := handleAgentBootstrapPairing(appCtx, cfg); err != nil {
			slog.WarnContext(appCtx, "Failed to auto-pair agent with manager", "error", err)
		}
	} else if cfg.AgentMode && !cfg.EdgeAgent {
		slog.InfoContext(appCtx, "Direct mode active: agent operates as a passive HTTP server; no outbound connection to manager required")
	}
}

func startEdgeTunnelClient(appCtx context.Context, lc fx.Lifecycle, cfg *config.Config, router *echo.Echo, _ *http.Server) {
	var stop func(context.Context) error
	lc.Append(fx.Hook{
		OnStart: func(context.Context) error {
			if !cfg.EdgeAgent || cfg.ManagerApiUrl == "" || cfg.AgentToken == "" {
				return nil
			}
			tunnelStop, err := edge.StartTunnelClient(appCtx, buildEdgeRuntimeConfig(cfg), router)
			if err != nil {
				slog.ErrorContext(appCtx, "Failed to start edge tunnel client", "error", err)
				return nil
			}
			stop = tunnelStop
			slog.InfoContext(appCtx, "Edge tunnel client started", "managerUrl", cfg.ManagerApiUrl)
			return nil
		},
		OnStop: func(ctx context.Context) error {
			if stop == nil {
				return nil
			}
			return stop(ctx)
		},
	})
}

func startGlobalVariableSync(appCtx context.Context, lc fx.Lifecycle, cfg *config.Config, variables *variable.VariableService, _ *http.Server) {
	if cfg.AgentMode {
		return
	}
	lc.Append(fx.Hook{OnStart: func(context.Context) error {
		go variables.SyncAll(appCtx)
		return nil
	}})
}

func registerAppCancelHook(lc fx.Lifecycle, cancelApp context.CancelCauseFunc) {
	lc.Append(fx.Hook{
		OnStop: func(context.Context) error {
			cancelApp(nil)
			return nil
		},
	})
}

func handleAgentBootstrapPairing(ctx context.Context, cfg *config.Config) error {
	slog.InfoContext(ctx, "Agent mode detected with token, attempting auto-pairing", "managerUrl", cfg.ManagerApiUrl)

	pairURL := strings.TrimRight(httpx.ManagerBaseURL(cfg.ManagerApiUrl), "/") + "/api/environments/pair"

	reqCtx, cancel := context.WithTimeout(ctx, 30*time.Second)
	defer cancel()

	req, err := http.NewRequestWithContext(reqCtx, http.MethodPost, pairURL, http.NoBody)
	if err != nil {
		return fmt.Errorf("failed to create pairing request: %w", err)
	}

	req.Header.Set("X-Api-Key", cfg.AgentToken)

	httpClient, err := edge.NewManagerHTTPClient(buildEdgeRuntimeConfig(cfg), 10*time.Second)
	if err != nil {
		return fmt.Errorf("failed to configure edge pairing client: %w", err)
	}

	resp, err := httpClient.Do(req)
	if err != nil {
		return fmt.Errorf("pairing request failed: %w", err)
	}
	defer func() { _ = resp.Body.Close() }()

	body, _ := io.ReadAll(io.LimitReader(resp.Body, 512))

	switch {
	case resp.StatusCode == http.StatusOK:
		slog.InfoContext(ctx, "Successfully paired agent with manager", "managerUrl", cfg.ManagerApiUrl)
		return nil
	case resp.StatusCode == http.StatusBadRequest && strings.Contains(string(body), "not in pending status"):
		slog.InfoContext(ctx, "Agent already paired with manager", "managerUrl", cfg.ManagerApiUrl)
		return nil
	case resp.StatusCode == http.StatusUnauthorized:
		// Not fatal: the key may belong to an earlier pairing, and a configured token still works.
		slog.DebugContext(ctx, "Pairing skipped - API key not recognized (agent may already be paired)", "managerUrl", cfg.ManagerApiUrl)
		return nil
	}
	return fmt.Errorf("pairing failed with status %d: %s", resp.StatusCode, strings.TrimSpace(string(body)))
}