Modern Docker Management, Designed for Everyone
Go composecontainer-managementcontainersdockerdocker-composedocker-managementgoself-hostedsveltekittypescriptweb-ui
fix: unable to use templates due to 'not found' error (#2634)
Kyle Mendell7da176b1585b
diff --git a/backend/api/handlers/templates.go b/backend/api/handlers/templates.goindex 7752e4d5a..66d26ed4a 100644--- a/backend/api/handlers/templates.go+++ b/backend/api/handlers/templates.go@@ -4,6 +4,7 @@ import ("context""encoding/json""net/http"+ "net/url""github.com/danielgtaylor/huma/v2"humamw "github.com/getarcaneapp/arcane/backend/api/middleware"@@ -465,9 +466,17 @@ func (h *TemplateHandler) GetTemplate(ctx context.Context, input *GetTemplateInpreturn nil, huma.Error400BadRequest((&common.TemplateIDRequiredError{}).Error())}- tmpl, err := h.templateService.GetTemplate(ctx, input.ID)+ // Path parameter arrives URL-encoded (e.g. "remote%3Areg%3Aslug" for remote IDs that+ // contain ':' separators). Chi/Huma do not auto-decode, so decode here before+ // matching against cached / stored template IDs.+ id, decodeErr := url.PathUnescape(input.ID)+ if decodeErr != nil {+ return nil, huma.Error400BadRequest((&common.TemplateIDRequiredError{}).Error())+ }++ tmpl, err := h.templateService.GetTemplate(ctx, id)if err != nil {- if err.Error() == "template not found" {+ if common.IsTemplateNotFoundError(err) {return nil, huma.Error404NotFound((&common.TemplateNotFoundError{}).Error())}return nil, huma.Error500InternalServerError((&common.TemplateRetrievalError{Err: err}).Error())@@ -496,8 +505,16 @@ func (h *TemplateHandler) GetTemplateContent(ctx context.Context, input *GetTempreturn nil, huma.Error400BadRequest((&common.TemplateIDRequiredError{}).Error())}- contentData, err := h.templateService.GetTemplateContentWithParsedData(ctx, input.ID)+ id, decodeErr := url.PathUnescape(input.ID)+ if decodeErr != nil {+ return nil, huma.Error400BadRequest((&common.TemplateIDRequiredError{}).Error())+ }++ contentData, err := h.templateService.GetTemplateContentWithParsedData(ctx, id)if err != nil {+ if common.IsTemplateNotFoundError(err) {+ return nil, huma.Error404NotFound((&common.TemplateNotFoundError{}).Error())+ }return nil, huma.Error500InternalServerError((&common.TemplateContentError{Err: err}).Error())}@@ -553,6 +570,11 @@ func (h *TemplateHandler) UpdateTemplate(ctx context.Context, input *UpdateTemplreturn nil, huma.Error400BadRequest((&common.TemplateIDRequiredError{}).Error())}+ id, decodeErr := url.PathUnescape(input.ID)+ if decodeErr != nil {+ return nil, huma.Error400BadRequest((&common.TemplateIDRequiredError{}).Error())+ }+updates := &models.ComposeTemplate{Name: input.Body.Name,Description: input.Body.Description,@@ -564,14 +586,14 @@ func (h *TemplateHandler) UpdateTemplate(ctx context.Context, input *UpdateTemplupdates.EnvContent = nil}- if err := h.templateService.UpdateTemplate(ctx, input.ID, updates); err != nil {- if err.Error() == "template not found" {+ if err := h.templateService.UpdateTemplate(ctx, id, updates); err != nil {+ if common.IsTemplateNotFoundError(err) {return nil, huma.Error404NotFound((&common.TemplateNotFoundError{}).Error())}return nil, huma.Error500InternalServerError((&common.TemplateUpdateError{Err: err}).Error())}- updated, err := h.templateService.GetTemplate(ctx, input.ID)+ updated, err := h.templateService.GetTemplate(ctx, id)if err != nil {return nil, huma.Error500InternalServerError((&common.TemplateRetrievalError{Err: err}).Error())}@@ -599,8 +621,13 @@ func (h *TemplateHandler) DeleteTemplate(ctx context.Context, input *DeleteTemplreturn nil, huma.Error400BadRequest((&common.TemplateIDRequiredError{}).Error())}- if err := h.templateService.DeleteTemplate(ctx, input.ID); err != nil {- if err.Error() == "template not found" {+ id, decodeErr := url.PathUnescape(input.ID)+ if decodeErr != nil {+ return nil, huma.Error400BadRequest((&common.TemplateIDRequiredError{}).Error())+ }++ if err := h.templateService.DeleteTemplate(ctx, id); err != nil {+ if common.IsTemplateNotFoundError(err) {return nil, huma.Error404NotFound((&common.TemplateNotFoundError{}).Error())}return nil, huma.Error500InternalServerError((&common.TemplateDeletionError{Err: err}).Error())@@ -626,9 +653,17 @@ func (h *TemplateHandler) DownloadTemplate(ctx context.Context, input *DownloadTreturn nil, huma.Error400BadRequest((&common.TemplateIDRequiredError{}).Error())}- tmpl, err := h.templateService.GetTemplate(ctx, input.ID)+ id, decodeErr := url.PathUnescape(input.ID)+ if decodeErr != nil {+ return nil, huma.Error400BadRequest((&common.TemplateIDRequiredError{}).Error())+ }++ tmpl, err := h.templateService.GetTemplate(ctx, id)if err != nil {- return nil, huma.Error404NotFound((&common.TemplateNotFoundError{}).Error())+ if common.IsTemplateNotFoundError(err) {+ return nil, huma.Error404NotFound((&common.TemplateNotFoundError{}).Error())+ }+ return nil, huma.Error500InternalServerError((&common.TemplateDownloadError{Err: err}).Error())}if !tmpl.IsRemote {return nil, huma.Error400BadRequest((&common.TemplateAlreadyLocalError{}).Error())diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.goindex 7d169c203..429a1c3ce 100644--- a/backend/internal/bootstrap/jobs_bootstrap.go+++ b/backend/internal/bootstrap/jobs_bootstrap.go@@ -84,6 +84,13 @@ func setupSettingsCallbacks(lifecycleCtx context.Context, appServices *Services,}}}+ appServices.Settings.OnTemplatesDirectoryChanged = func(_ context.Context) {+ if fsWatcherJob != nil {+ if err := fsWatcherJob.RestartTemplatesWatcher(lifecycleCtx); err != nil {+ slog.WarnContext(lifecycleCtx, "Failed to restart templates filesystem watcher", "error", err)+ }+ }+ }appServices.Settings.OnScheduledPruneSettingsChanged = func(_ context.Context) {if err := newScheduler.RescheduleJob(lifecycleCtx, scheduledPruneJob); err != nil {slog.WarnContext(lifecycleCtx, "Failed to reschedule scheduled-prune job", "error", err)diff --git a/backend/internal/common/errors.go b/backend/internal/common/errors.goindex 4c226a3c0..66b7d324e 100644--- a/backend/internal/common/errors.go+++ b/backend/internal/common/errors.go@@ -1046,6 +1046,10 @@ func (e *TemplateNotFoundError) Error() string {return "Template not found"}+func IsTemplateNotFoundError(err error) bool {+ return isErrorTypeInternal[*TemplateNotFoundError](err)+}+type TemplateRetrievalError struct {Err error}diff --git a/backend/internal/config/config.go b/backend/internal/config/config.goindex 0883fef8e..caeb806ed 100644--- a/backend/internal/config/config.go+++ b/backend/internal/config/config.go@@ -60,6 +60,7 @@ type Config struct {DockerHost string `env:"DOCKER_HOST" default:"unix:///var/run/docker.sock"`DockerConfig string `env:"DOCKER_CONFIG" default:""`ProjectsDirectory string `env:"PROJECTS_DIRECTORY" default:"/app/data/projects"`+ TemplatesDirectory string `env:"TEMPLATES_DIRECTORY" default:"/app/data/templates"`ProjectScanMaxDepth int `env:"PROJECT_SCAN_MAX_DEPTH" default:"3"`ProjectScanSkipDirs string `env:"PROJECT_SCAN_SKIP_DIRS" default:".git,node_modules,vendor,.venv,venv,__pycache__,.cache,dist,build,target,.next,.nuxt,.svelte-kit"`LogJson bool `env:"LOG_JSON" default:"false"`diff --git a/backend/internal/configschema/schema_test.go b/backend/internal/configschema/schema_test.goindex 1c172552d..617b7df3f 100644--- a/backend/internal/configschema/schema_test.go+++ b/backend/internal/configschema/schema_test.go@@ -257,6 +257,7 @@ var expectedEnvConfigVars = []string{"PROXY_REQUEST_TIMEOUT","PUID","REGISTRY_TIMEOUT",+ "TEMPLATES_DIRECTORY","TLS_CERT_FILE","TLS_ENABLED","TLS_KEY_FILE",@@ -353,6 +354,7 @@ var expectedSettingOverrideKeys = []string{"scheduledPruneVolumes","sidebarHoverExpansion","swarmStackSourcesDirectory",+ "templatesDirectory","trivyConcurrentScanContainers","trivyConfig","trivyCpuLimit",diff --git a/backend/internal/models/settings.go b/backend/internal/models/settings.goindex 9e5b5c209..8e5be4809 100644--- a/backend/internal/models/settings.go+++ b/backend/internal/models/settings.go@@ -70,6 +70,7 @@ func (s SettingVariable) AsDurationSeconds() time.Duration {type Settings struct {// General categoryProjectsDirectory SettingVariable `key:"projectsDirectory,envOverride" meta:"label=Projects Directory;type=text;keywords=projects,directory,path,folder,location,storage,files,compose,docker-compose;category=internal;description=Configure where project files are stored"`+ TemplatesDirectory SettingVariable `key:"templatesDirectory,envOverride" meta:"label=Templates Directory;type=text;keywords=templates,directory,path,folder,location,storage,compose,docker-compose;category=internal;description=Configure where local compose template folders are discovered"`FollowProjectSymlinks SettingVariable `key:"followProjectSymlinks,envOverride" meta:"label=Follow Project Symlinks;type=boolean;keywords=projects,symlink,symlinks,symbolic links,compose,directory,discovery;category=general;description=Treat symlinked child directories inside the projects directory as Docker Compose projects"`SwarmStackSourcesDirectory SettingVariable `key:"swarmStackSourcesDirectory,envOverride" meta:"label=Swarm Stack Sources Directory;type=text;keywords=swarm,stacks,stack,source,sources,directory,path,folder,location,storage,compose,env;category=internal;description=Configure where swarm stack source files are stored"`DiskUsagePath SettingVariable `key:"diskUsagePath" meta:"label=Disk Usage Path;type=text;keywords=disk,usage,path,storage,folder,files;category=general;description=Path used for disk usage calculations"`diff --git a/backend/internal/services/settings_service.go b/backend/internal/services/settings_service.goindex 85b021fba..cc0a3b1b5 100644--- a/backend/internal/services/settings_service.go+++ b/backend/internal/services/settings_service.go@@ -37,6 +37,7 @@ type SettingsService struct {OnImagePollingSettingsChanged func(ctx context.Context)OnAutoUpdateSettingsChanged func(ctx context.Context)OnProjectsDirectoryChanged func(ctx context.Context)+ OnTemplatesDirectoryChanged func(ctx context.Context)OnScheduledPruneSettingsChanged func(ctx context.Context)OnVulnerabilityScanSettingsChanged func(ctx context.Context)OnAutoHealSettingsChanged func(ctx context.Context)@@ -108,6 +109,7 @@ func (s *SettingsService) getDefaultSettings() *models.Settings {func DefaultSettingsConfig() *models.Settings {return &models.Settings{ProjectsDirectory: models.SettingVariable{Value: "/app/data/projects"},+ TemplatesDirectory: models.SettingVariable{Value: "/app/data/templates"},FollowProjectSymlinks: models.SettingVariable{Value: "false"},SwarmStackSourcesDirectory: models.SettingVariable{Value: "/app/data/swarm/sources"},DiskUsagePath: models.SettingVariable{Value: "/app/data/projects"},@@ -709,6 +711,11 @@ func (s *SettingsService) UpdateSettings(ctx context.Context, updates settings.U}) && s.OnProjectsDirectoryChanged != nil {s.OnProjectsDirectoryChanged(ctx)}+ if slices.ContainsFunc(valuesToUpdate, func(sv models.SettingVariable) bool {+ return sv.Key == "templatesDirectory"+ }) && s.OnTemplatesDirectoryChanged != nil {+ s.OnTemplatesDirectoryChanged(ctx)+ }if len(changedTimeouts) > 0 && s.OnTimeoutSettingsChanged != nil {s.OnTimeoutSettingsChanged(ctx, changedTimeouts)}diff --git a/backend/internal/services/template_service.go b/backend/internal/services/template_service.goindex 08509f04b..39b7f543c 100644--- a/backend/internal/services/template_service.go+++ b/backend/internal/services/template_service.go@@ -82,10 +82,6 @@ func NewTemplateService(ctx context.Context, db *database.DB, httpClient *http.ChttpClient = http.DefaultClient}- if err := projects.EnsureDefaultTemplates(ctx); err != nil {- slog.WarnContext(ctx, "failed to ensure default templates", "error", err)- }-service := &TemplateService{db: db,httpClient: httpClient,@@ -96,20 +92,44 @@ func NewTemplateService(ctx context.Context, db *database.DB, httpClient *http.CregistryErrors: make(map[string]string),}service.safeHTTPClient = service.newSafeHTTPClientInternal()++ if err := projects.EnsureDefaultTemplates(ctx, service.configuredTemplatesDirSettingInternal(ctx)); err != nil {+ slog.WarnContext(ctx, "failed to ensure default templates", "error", err)+ }+return service}+// configuredTemplatesDirSettingInternal returns the raw user-configured templates+// directory string (from the settingsService), without resolving it to an absolute+// path. Use this when calling helpers like projects.GetTemplatesDirectory that+// perform their own resolution.+func (s *TemplateService) configuredTemplatesDirSettingInternal(ctx context.Context) string {+ if s.settingsService == nil {+ return ""+ }+ return s.settingsService.GetStringSetting(ctx, "templatesDirectory", "/app/data/templates")+}++// getTemplatesDirectoryInternal resolves the effective templates directory by+// applying the same resolution rules used elsewhere for the projects directory.+func (s *TemplateService) getTemplatesDirectoryInternal(ctx context.Context) (string, error) {+ return projects.GetTemplatesDirectory(ctx, strings.TrimSpace(s.configuredTemplatesDirSettingInternal(ctx)))+}+func (s *TemplateService) ensureRemoteTemplatesLoaded(ctx context.Context) error {s.remoteMu.Lock()- // If cache is valid, return- if s.remoteCache.templates != nil && time.Since(s.remoteCache.lastFetch) < remoteCacheDuration {+ // If cache has entries and is fresh, return.+ // An empty slice is treated as "no cache" so callers always trigger a blocking+ // refresh — otherwise a single failed/empty refresh would poison the cache.+ if len(s.remoteCache.templates) > 0 && time.Since(s.remoteCache.lastFetch) < remoteCacheDuration {s.remoteMu.Unlock()return nil}- // If we have cache (even stale) and are not already refreshing, trigger background refresh- if s.remoteCache.templates != nil {+ // If we have a non-empty stale cache and nobody is refreshing, trigger background refresh.+ if len(s.remoteCache.templates) > 0 {if !s.remoteCache.refreshing {s.remoteCache.refreshing = trues.remoteMu.Unlock()@@ -236,8 +256,6 @@ func (s *TemplateService) GetAllTemplatesPaginated(ctx context.Context, params preturn result.Items, paginationResp, nil}-var ErrTemplateNotFound = errors.New("template not found")-// envKeyPattern is the POSIX env-name shape used to validate global variable// keys before they are persisted to .env.global. Keys that do not match are// rejected with a common.InvalidEnvKeyError.@@ -256,19 +274,48 @@ func (s *TemplateService) GetTemplate(ctx context.Context, id string) (*models.C}if err := s.ensureRemoteTemplatesLoaded(ctx); err != nil {- return nil, fmt.Errorf("template not found (failed to load remote templates): %w", err)+ return nil, fmt.Errorf("template %q lookup failed: registry refresh error: %w", id, err)}- s.remoteMu.RLock()- copied := cloneRemoteTemplates(s.remoteCache.templates)- s.remoteMu.RUnlock()- for _, remoteTemplate := range copied {- if remoteTemplate.ID == id {- return new(remoteTemplate), nil+ if found := s.lookupRemoteFromCacheInternal(id); found != nil {+ return found, nil+ }++ // Remote IDs (remote:registry:slug) deserve a synchronous force-refresh on miss+ // before we return "not found" — the cache may be stale or the previous refresh+ // silently returned empty.+ if strings.HasPrefix(id, remoteIDPrefix+":") {+ slog.InfoContext(ctx, "remote template not in cache, forcing registry refresh", "templateID", id, "cacheSize", s.remoteCacheSizeInternal())+ if _, refreshErr := s.refreshRemoteTemplates(ctx); refreshErr != nil {+ return nil, fmt.Errorf("template %q not found and registry refresh failed: %w", id, refreshErr)+ }+ if found := s.lookupRemoteFromCacheInternal(id); found != nil {+ return found, nil+ }+ return nil, &common.TemplateNotFoundError{+ Err: fmt.Errorf("template %q not found in any registered registry (cache size=%d after refresh)", id, s.remoteCacheSizeInternal()),}}- return nil, ErrTemplateNotFound+ return nil, &common.TemplateNotFoundError{}+}++func (s *TemplateService) lookupRemoteFromCacheInternal(id string) *models.ComposeTemplate {+ s.remoteMu.RLock()+ defer s.remoteMu.RUnlock()+ for i := range s.remoteCache.templates {+ if s.remoteCache.templates[i].ID == id {+ cloned := cloneRemoteTemplates(s.remoteCache.templates[i : i+1])+ return &cloned[0]+ }+ }+ return nil+}++func (s *TemplateService) remoteCacheSizeInternal() int {+ s.remoteMu.RLock()+ defer s.remoteMu.RUnlock()+ return len(s.remoteCache.templates)}func (s *TemplateService) CreateTemplate(ctx context.Context, template *models.ComposeTemplate) error {@@ -291,7 +338,7 @@ func (s *TemplateService) UpdateTemplate(ctx context.Context, id string, updatesvar existing models.ComposeTemplateif err := tx.Where("id = ?", id).First(&existing).Error; err != nil {if errors.Is(err, gorm.ErrRecordNotFound) {- return fmt.Errorf("template not found")+ return &common.TemplateNotFoundError{}}return fmt.Errorf("failed to find template: %w", err)}@@ -319,7 +366,7 @@ func (s *TemplateService) DeleteTemplate(ctx context.Context, id string) error {var existing models.ComposeTemplateif err := tx.Where("id = ?", id).First(&existing).Error; err != nil {if errors.Is(err, gorm.ErrRecordNotFound) {- return fmt.Errorf("template not found")+ return &common.TemplateNotFoundError{}}return fmt.Errorf("failed to find template: %w", err)}@@ -328,15 +375,14 @@ func (s *TemplateService) DeleteTemplate(ctx context.Context, id string) error {return fmt.Errorf("cannot delete remote template directly")}- baseDir, err := projects.GetTemplatesDirectory(ctx)+ baseDir, err := s.getTemplatesDirectoryInternal(ctx)if err != nil {return fmt.Errorf("failed to get templates directory: %w", err)} else {templatePath := filepath.Join(baseDir, existing.Name)if stat, err := os.Stat(templatePath); err == nil && stat.IsDir() {- composeFile := filepath.Join(templatePath, "compose.yaml")- if _, err := os.Stat(composeFile); err == nil {+ if _, err := projects.DetectComposeFile(templatePath); err == nil {if err := os.RemoveAll(templatePath); err != nil {return fmt.Errorf("failed to delete template directory: %w", err)}@@ -707,8 +753,15 @@ func (s *TemplateService) convertRemoteToLocal(remote tmpl.RemoteTemplate, regis}func (s *TemplateService) FetchTemplateContent(ctx context.Context, template *models.ComposeTemplate) (string, string, error) {- if !template.IsRemote || template.Metadata == nil || template.Metadata.RemoteURL == nil {- return template.Content, "", fmt.Errorf("not a remote template or missing remote URL")+ if !template.IsRemote {+ envContent := ""+ if template.EnvContent != nil {+ envContent = *template.EnvContent+ }+ return template.Content, envContent, nil+ }+ if template.Metadata == nil || template.Metadata.RemoteURL == nil || strings.TrimSpace(*template.Metadata.RemoteURL) == "" {+ return "", "", fmt.Errorf("remote template %q is missing compose_url in registry metadata", template.ID)}return s.fetchRemoteTemplateFiles(ctx, template)@@ -809,7 +862,7 @@ func (s *TemplateService) DownloadTemplate(ctx context.Context, remoteTemplate *base := s.templateBaseFromRemote(remoteTemplate)- dir, composePath, envPath, err := projects.EnsureTemplateDir(ctx, base)+ dir, composePath, envPath, err := projects.EnsureTemplateDir(ctx, s.configuredTemplatesDirSettingInternal(ctx), base)if err != nil {return nil, err}@@ -958,12 +1011,17 @@ func (s *TemplateService) upsertFilesystemTemplate(ctx context.Context, name, dereturn s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {var existing models.ComposeTemplate+ // Match on description (which encodes the absolute compose path) OR name (folder name).+ // Matching on name as a fallback keeps a single row across templates-directory+ // reconfigurations or compose-filename renames within the same folder, but+ // only for templates previously imported from disk.q := tx.- Where("is_remote = ? AND registry_id IS NULL AND description = ?", false, desc).+ Where("is_remote = ? AND registry_id IS NULL AND (description = ? OR (name = ? AND description LIKE ?))", false, desc, name, "Imported from %").First(&existing)if q.Error == nil {existing.Name = name+ existing.Description = descexisting.Content = composeexisting.EnvContent = envPtrexisting.IsCustom = true@@ -1014,7 +1072,7 @@ func (s *TemplateService) syncFilesystemTemplatesInternal(ctx context.Context) ereturn nil}- dir, err := projects.GetTemplatesDirectory(ctx)+ dir, err := s.getTemplatesDirectoryInternal(ctx)if err != nil {return fmt.Errorf("ensure templates dir: %w", err)}diff --git a/backend/internal/services/template_service_test.go b/backend/internal/services/template_service_test.goindex a97cd132f..dff9e5589 100644--- a/backend/internal/services/template_service_test.go+++ b/backend/internal/services/template_service_test.go@@ -257,10 +257,14 @@ services:client, lookupIP, baseURL := makePublicTestClient(t, server)+ settingsSvc := minimalSettingsServiceForTest(t)+ require.NoError(t, settingsSvc.UpdateSetting(context.Background(), "templatesDirectory", filepath.Join(tempDir, "templates")))+service := &TemplateService{db: setupTemplateServiceTestDB(t),httpClient: client,lookupIP: lookupIP,+ settingsService: settingsSvc,registryFetchMeta: make(map[string]*registryFetchMeta),}@@ -383,9 +387,9 @@ func TestFetchRaw_BlocksUnsafeRemoteURL(t *testing.T) {func TestSyncFilesystemTemplatesInternal_PopulatesIconURL(t *testing.T) {tempDir := t.TempDir()- setTestWorkingDir(t, tempDir)- templateDir := filepath.Join(tempDir, "data", "templates", "example")+ templatesRoot := filepath.Join(tempDir, "templates")+ templateDir := filepath.Join(templatesRoot, "example")require.NoError(t, os.MkdirAll(templateDir, 0o755))require.NoError(t, os.WriteFile(filepath.Join(templateDir, "compose.yaml"), []byte(`x-arcane:icon: https://cdn.example/local.png@@ -394,9 +398,13 @@ services:image: nginx:alpine`), 0o644))+ settingsSvc := minimalSettingsServiceForTest(t)+ require.NoError(t, settingsSvc.UpdateSetting(context.Background(), "templatesDirectory", templatesRoot))+service := &TemplateService{db: setupTemplateServiceTestDB(t),httpClient: http.DefaultClient,+ settingsService: settingsSvc,registryFetchMeta: make(map[string]*registryFetchMeta),}@@ -409,6 +417,88 @@ services:require.Equal(t, "https://cdn.example/local.png", *stored.Metadata.IconURL)}+func TestGetTemplate_ForceRefreshesRemoteCacheOnMiss(t *testing.T) {+ tempDir := t.TempDir()+ setTestWorkingDir(t, tempDir)++ var registryHits atomic.Int32+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {+ if r.URL.Path != "/registry.json" {+ http.NotFound(w, r)+ return+ }+ registryHits.Add(1)+ _, _ = w.Write([]byte(`{+ "name": "Demo Registry",+ "description": "Test",+ "version": "1.0.0",+ "author": "Arcane",+ "templates": [+ {+ "id": "affine",+ "name": "AFFiNE",+ "description": "",+ "version": "1.0.0",+ "author": "Arcane",+ "compose_url": "",+ "env_url": "",+ "documentation_url": "",+ "tags": []+ }+ ]+}`))+ }))+ defer server.Close()++ client, lookupIP, baseURL := makePublicTestClient(t, server)+ db := setupTemplateServiceTestDB(t)++ registry := &models.TemplateRegistry{+ BaseModel: models.BaseModel{ID: "reg-1"},+ Name: "Demo",+ URL: baseURL + "/registry.json",+ Enabled: true,+ }+ require.NoError(t, db.WithContext(context.Background()).Create(registry).Error)++ settingsSvc := minimalSettingsServiceForTest(t)+ // Point templates+projects directories at tempDir so sync calls don't try to+ // touch the real /app filesystem during tests.+ require.NoError(t, settingsSvc.UpdateSetting(context.Background(), "templatesDirectory", filepath.Join(tempDir, "templates")))+ require.NoError(t, settingsSvc.UpdateSetting(context.Background(), "projectsDirectory", filepath.Join(tempDir, "projects")))++ service := &TemplateService{+ db: db,+ httpClient: client,+ lookupIP: lookupIP,+ settingsService: settingsSvc,+ registryFetchMeta: make(map[string]*registryFetchMeta),+ }++ // Cache starts empty; GetTemplate for a remote ID should force a refresh and find the template.+ got, err := service.GetTemplate(context.Background(), "remote:reg-1:affine")+ require.NoError(t, err)+ require.NotNil(t, got)+ require.True(t, got.IsRemote)+ require.Equal(t, "AFFiNE", got.Name)+ require.GreaterOrEqual(t, registryHits.Load(), int32(1), "registry should be fetched at least once")++ // Unknown remote ID still surfaces a clear error after the forced refresh.+ _, err = service.GetTemplate(context.Background(), "remote:reg-1:does-not-exist")+ require.Error(t, err)+ require.Contains(t, err.Error(), "not found")+}++func minimalSettingsServiceForTest(t *testing.T) *SettingsService {+ t.Helper()+ db, err := gorm.Open(glsqlite.Open(":memory:"), &gorm.Config{})+ require.NoError(t, err)+ require.NoError(t, db.AutoMigrate(&models.SettingVariable{}))+ svc, err := NewSettingsService(context.Background(), &database.DB{DB: db})+ require.NoError(t, err)+ return svc+}+func TestUpdateGlobalVariables_RejectsNewlineInjectionKey(t *testing.T) {projectsDir := t.TempDir()diff --git a/backend/pkg/projects/fs_templates.go b/backend/pkg/projects/fs_templates.goindex 048142a04..20addb369 100644--- a/backend/pkg/projects/fs_templates.go+++ b/backend/pkg/projects/fs_templates.go@@ -2,6 +2,7 @@ package projectsimport ("context"+ "errors""fmt""os""path/filepath"@@ -12,12 +13,13 @@ import ()func ReadFolderComposeTemplate(baseDir, folder string) (string, *string, string, bool, error) {- composePath := filepath.Join(baseDir, folder, "compose.yaml")- if _, err := os.Stat(composePath); err != nil {- if os.IsNotExist(err) {+ folderPath := filepath.Join(baseDir, folder)+ composePath, err := DetectComposeFile(folderPath)+ if err != nil {+ if errors.Is(err, errComposeFileNotFoundInternal) {return "", nil, "", false, nil}- return "", nil, "", false, fmt.Errorf("stat compose: %w", err)+ return "", nil, "", false, fmt.Errorf("detect compose file: %w", err)}b, err := os.ReadFile(composePath)@@ -27,14 +29,15 @@ func ReadFolderComposeTemplate(baseDir, folder string) (string, *string, string,var envPtr *stringfor _, envName := range []string{".env.example", ".env"} {- envPath := filepath.Join(baseDir, folder, envName)- if eb, err := os.ReadFile(envPath); err == nil {- envPtr = new(string(eb))+ envPath := filepath.Join(folderPath, envName)+ if eb, rerr := os.ReadFile(envPath); rerr == nil {+ s := string(eb)+ envPtr = &sbreak}}- desc := fmt.Sprintf("Imported from %s/%s/compose.yaml", baseDir, folder)+ desc := fmt.Sprintf("Imported from %s", composePath)return string(b), envPtr, desc, true, nil}@@ -50,8 +53,8 @@ func Slugify(in string) string {return strings.Trim(in, "-")}-func EnsureTemplateDir(ctx context.Context, base string) (dir, composePath, envPath string, err error) {- baseDir, derr := GetTemplatesDirectory(ctx)+func EnsureTemplateDir(ctx context.Context, templatesDir, base string) (dir, composePath, envPath string, err error) {+ baseDir, derr := GetTemplatesDirectory(ctx, templatesDir)if derr != nil {return "", "", "", fmt.Errorf("ensure templates dir: %w", derr)}@@ -84,8 +87,8 @@ func WriteTemplateFiles(composePath, envPath, composeContent, envContent string)return &envContent, nil}-func EnsureDefaultTemplates(ctx context.Context) error {- templatesDir, err := GetTemplatesDirectory(ctx)+func EnsureDefaultTemplates(ctx context.Context, configuredTemplatesDir string) error {+ templatesDir, err := GetTemplatesDirectory(ctx, configuredTemplatesDir)if err != nil {return fmt.Errorf("get templates directory: %w", err)}diff --git a/backend/pkg/projects/fs_templates_test.go b/backend/pkg/projects/fs_templates_test.gonew file mode 100644index 000000000..53efc4a16--- /dev/null+++ b/backend/pkg/projects/fs_templates_test.go@@ -0,0 +1,62 @@+package projects++import (+ "os"+ "path/filepath"+ "strings"+ "testing"++ "github.com/stretchr/testify/assert"+ "github.com/stretchr/testify/require"+)++func TestReadFolderComposeTemplate_DetectsAllVariants(t *testing.T) {+ t.Parallel()++ composeContent := "services:\n app:\n image: nginx:alpine\n"++ testCases := []struct {+ name string+ fileName string+ }{+ {name: "compose.yaml", fileName: "compose.yaml"},+ {name: "compose.yml", fileName: "compose.yml"},+ {name: "docker-compose.yml", fileName: "docker-compose.yml"},+ }++ for _, tc := range testCases {+ t.Run(tc.name, func(t *testing.T) {+ t.Parallel()++ baseDir := t.TempDir()+ folder := "demo"+ folderPath := filepath.Join(baseDir, folder)+ require.NoError(t, os.MkdirAll(folderPath, 0o755))+ require.NoError(t, os.WriteFile(filepath.Join(folderPath, tc.fileName), []byte(composeContent), 0o600))++ content, env, desc, found, err := ReadFolderComposeTemplate(baseDir, folder)+ require.NoError(t, err)+ require.True(t, found, "expected template to be detected for %s", tc.fileName)+ assert.Equal(t, composeContent, content)+ assert.Nil(t, env)+ assert.True(t, strings.Contains(desc, tc.fileName), "description should reference detected filename, got %q", desc)+ })+ }+}++func TestReadFolderComposeTemplate_ReadsEnvExample(t *testing.T) {+ t.Parallel()++ baseDir := t.TempDir()+ folder := "envtest"+ folderPath := filepath.Join(baseDir, folder)+ require.NoError(t, os.MkdirAll(folderPath, 0o755))+ require.NoError(t, os.WriteFile(filepath.Join(folderPath, "compose.yml"), []byte("services: {}\n"), 0o600))+ require.NoError(t, os.WriteFile(filepath.Join(folderPath, ".env.example"), []byte("KEY=value\n"), 0o600))++ _, env, _, found, err := ReadFolderComposeTemplate(baseDir, folder)+ require.NoError(t, err)+ require.True(t, found)+ require.NotNil(t, env)+ assert.Equal(t, "KEY=value\n", *env)+}diff --git a/backend/pkg/projects/fs_util.go b/backend/pkg/projects/fs_util.goindex 2482981d2..7408b7401 100644--- a/backend/pkg/projects/fs_util.go+++ b/backend/pkg/projects/fs_util.go@@ -130,15 +130,15 @@ func HasComposeRootKeysInFile(path string) (bool, error) {return hasServices || hasInclude, nil}-func GetTemplatesDirectory(ctx context.Context) (string, error) {- templatesDir := filepath.Join("data", "templates")- if _, err := os.Stat(templatesDir); os.IsNotExist(err) {- if err := os.MkdirAll(templatesDir, pkgutils.DirPerm); err != nil {+func GetTemplatesDirectory(ctx context.Context, templatesDir string) (string, error) {+ resolved := ResolveConfiguredContainerDirectory(templatesDir, "/app/data/templates")+ if _, err := os.Stat(resolved); os.IsNotExist(err) {+ if err := os.MkdirAll(resolved, pkgutils.DirPerm); err != nil {return "", err}- slog.InfoContext(ctx, "Created templates directory", "path", templatesDir)+ slog.InfoContext(ctx, "Created templates directory", "path", resolved)}- return templatesDir, nil+ return resolved, nil}func ReadProjectDirectoryFiles(projectPath string, shownFiles map[string]bool, maxDepth int, skipDirectories string) ([]project.IncludeFile, error) {diff --git a/backend/pkg/projects/load.go b/backend/pkg/projects/load.goindex 9130fbfb1..bf636ee42 100644--- a/backend/pkg/projects/load.go+++ b/backend/pkg/projects/load.go@@ -2,6 +2,7 @@ package projectsimport ("context"+ "errors""fmt""log/slog""maps"@@ -17,6 +18,8 @@ import ("github.com/docker/compose/v5/pkg/api")+var errComposeFileNotFoundInternal = errors.New("no compose file found")+var ProjectFileCandidates = []string{"compose.yaml","compose.yml",@@ -133,7 +136,7 @@ func DetectComposeFile(dir string) (string, error) {case len(customCandidates) > 1:return "", fmt.Errorf("multiple custom compose files found in %q", dir)default:- return "", fmt.Errorf("no compose file found in %q", dir)+ return "", fmt.Errorf("%w in %q", errComposeFileNotFoundInternal, dir)}}diff --git a/backend/pkg/scheduler/filesystem_watcher_job.go b/backend/pkg/scheduler/filesystem_watcher_job.goindex 75f679105..0f375494d 100644--- a/backend/pkg/scheduler/filesystem_watcher_job.go+++ b/backend/pkg/scheduler/filesystem_watcher_job.go@@ -68,21 +68,28 @@ func (j *FilesystemWatcherJob) Start(ctx context.Context) error {j.projectsWatcher = sw- templatesDir, err := projects.GetTemplatesDirectory(ctx)+ templatesDir, err := projects.GetTemplatesDirectory(ctx, settings.TemplatesDirectory.Value)if err != nil {return err}if j.templateService != nil {- tw, err := fswatch.NewWatcher(templatesDir, fswatch.WatcherOptions{- Debounce: 3 * time.Second,- OnChange: j.handleTemplatesChange,- MaxDepth: 1,- })- if err != nil {- return err+ if directoriesOverlapInternal(projectsDirectory, templatesDir) {+ slog.ErrorContext(ctx,+ "Templates and projects directories overlap; templates watcher disabled to prevent compose files being treated as projects",+ "projectsDirectory", projectsDirectory,+ "templatesDirectory", templatesDir)+ } else {+ tw, err := fswatch.NewWatcher(templatesDir, fswatch.WatcherOptions{+ Debounce: 3 * time.Second,+ OnChange: j.handleTemplatesChange,+ MaxDepth: 1,+ })+ if err != nil {+ return err+ }+ j.templatesWatcher = tw}- j.templatesWatcher = tw}if err := j.projectsWatcher.Start(ctx); err != nil {@@ -242,3 +249,76 @@ func (j *FilesystemWatcherJob) projectWatcherOptionsInternal(followProjectSymlinFollowSymlinkDirs: followProjectSymlinks,}}++func (j *FilesystemWatcherJob) RestartTemplatesWatcher(ctx context.Context) error {+ if j.templateService == nil {+ return nil+ }+ slog.InfoContext(ctx, "Restarting templates filesystem watcher")++ j.mu.Lock()+ oldTemplatesWatcher := j.templatesWatcher+ j.templatesWatcher = nil+ j.mu.Unlock()++ if oldTemplatesWatcher != nil {+ if err := oldTemplatesWatcher.Stop(); err != nil {+ slog.WarnContext(ctx, "Failed to stop templates watcher during restart", "error", err)+ }+ }++ settings, err := j.settingsService.GetSettings(ctx)+ if err != nil {+ return err+ }+ projectsDirectory, err := projects.GetProjectsDirectory(ctx, settings.ProjectsDirectory.Value)+ if err != nil {+ return err+ }+ templatesDir, err := projects.GetTemplatesDirectory(ctx, settings.TemplatesDirectory.Value)+ if err != nil {+ return err+ }++ if directoriesOverlapInternal(projectsDirectory, templatesDir) {+ slog.ErrorContext(ctx,+ "Templates and projects directories overlap; templates watcher not restarted",+ "projectsDirectory", projectsDirectory,+ "templatesDirectory", templatesDir)+ return nil+ }++ tw, err := fswatch.NewWatcher(templatesDir, fswatch.WatcherOptions{+ Debounce: 3 * time.Second,+ OnChange: j.handleTemplatesChange,+ MaxDepth: 1,+ })+ if err != nil {+ return err+ }+ if err := tw.Start(ctx); err != nil {+ return err+ }++ j.mu.Lock()+ j.templatesWatcher = tw+ j.mu.Unlock()++ slog.InfoContext(ctx, "Templates filesystem watcher restarted", "path", templatesDir)++ if err := j.templateService.SyncLocalTemplatesFromFilesystem(ctx); err != nil {+ slog.ErrorContext(ctx, "Initial template sync after watcher restart failed", "error", err)+ }++ return nil+}++// directoriesOverlapInternal returns true when a or b is the same as or contained in the+// other. Used to refuse running both watchers against the same tree, which would+// cause local templates to be auto-imported as projects.+func directoriesOverlapInternal(a, b string) bool {+ if a == "" || b == "" {+ return false+ }+ return projects.IsSafeSubdirectory(a, b) || projects.IsSafeSubdirectory(b, a)+}diff --git a/frontend/messages/en.json b/frontend/messages/en.jsonindex 4d5a2798d..71e08ff22 100644--- a/frontend/messages/en.json+++ b/frontend/messages/en.json@@ -1379,6 +1379,8 @@"general_title": "General","general_projects_directory_label": "Projects Directory","general_projects_directory_help": "Directory where Docker Compose files are stored (this is inside the container). MUST be an absolute path!",+ "general_templates_directory_label": "Templates Directory",+ "general_templates_directory_help": "Directory where local compose template folders are discovered. Must not overlap with the Projects Directory. Supports the same container:host bind-mount format.","general_follow_project_symlinks_label": "Follow Project Symlinks","general_follow_project_symlinks_help": "Treat symlinked child directories inside the projects directory as Docker Compose projects.","general_base_url_label": "Base Server URL",diff --git a/frontend/src/lib/types/settings.type.ts b/frontend/src/lib/types/settings.type.tsindex 2c111ff89..10d1c4590 100644--- a/frontend/src/lib/types/settings.type.ts+++ b/frontend/src/lib/types/settings.type.ts@@ -4,6 +4,7 @@ export type ApplicationTheme = 'default' | 'graphite' | 'ocean' | 'amber' | 'gitexport type Settings = {projectsDirectory: string;+ templatesDirectory: string;followProjectSymlinks: boolean;swarmStackSourcesDirectory: string;diskUsagePath: string;diff --git a/frontend/src/routes/(app)/environments/[id]/+page.svelte b/frontend/src/routes/(app)/environments/[id]/+page.svelteindex a2479419c..238c650d4 100644--- a/frontend/src/routes/(app)/environments/[id]/+page.svelte+++ b/frontend/src/routes/(app)/environments/[id]/+page.svelte@@ -217,6 +217,7 @@defaultDeployPullPolicy: (settings?.defaultDeployPullPolicy as 'missing' | 'always' | 'never') || 'missing',defaultShell: settings?.defaultShell || '/bin/sh',projectsDirectory: settings?.projectsDirectory || '/app/data/projects',+ templatesDirectory: settings?.templatesDirectory || '/app/data/templates',swarmStackSourcesDirectory: settings?.swarmStackSourcesDirectory || '/app/data/swarm/sources',diskUsagePath: settings?.diskUsagePath || '/app/data/projects',maxImageUploadSize: settings?.maxImageUploadSize || 500,@@ -270,6 +271,7 @@defaultDeployPullPolicy: formData.defaultDeployPullPolicy,defaultShell: formData.defaultShell,projectsDirectory: formData.projectsDirectory,+ templatesDirectory: formData.templatesDirectory,swarmStackSourcesDirectory: formData.swarmStackSourcesDirectory,diskUsagePath: formData.diskUsagePath,maxImageUploadSize: formData.maxImageUploadSize,diff --git a/frontend/src/routes/(app)/environments/[id]/components/GeneralTab.svelte b/frontend/src/routes/(app)/environments/[id]/components/GeneralTab.svelteindex e90e56fa3..edb24c419 100644--- a/frontend/src/routes/(app)/environments/[id]/components/GeneralTab.svelte+++ b/frontend/src/routes/(app)/environments/[id]/components/GeneralTab.svelte@@ -30,6 +30,15 @@helpText={m.general_projects_directory_help()}/></div>+ <div class="space-y-2">+ <TextInputWithLabel+ id="templates-directory"+ label={m.general_templates_directory_label()}+ bind:value={$formInputs.templatesDirectory.value}+ error={$formInputs.templatesDirectory.error}+ helpText={m.general_templates_directory_help()}+ />+ </div><div class="space-y-2"><TextInputWithLabelid="disk-usage-path"diff --git a/frontend/src/routes/(app)/environments/[id]/components/environment-form-schema.ts b/frontend/src/routes/(app)/environments/[id]/components/environment-form-schema.tsindex 9464d2db0..dde8abb52 100644--- a/frontend/src/routes/(app)/environments/[id]/components/environment-form-schema.ts+++ b/frontend/src/routes/(app)/environments/[id]/components/environment-form-schema.ts@@ -12,6 +12,7 @@ export const environmentFormSchema = z.object({defaultDeployPullPolicy: z.enum(['missing', 'always', 'never']),defaultShell: z.string(),projectsDirectory: z.string(),+ templatesDirectory: z.string(),swarmStackSourcesDirectory: z.string(),diskUsagePath: z.string(),maxImageUploadSize: z.coerce.number(),diff --git a/types/settings/settings.go b/types/settings/settings.goindex 64b6fd899..50d3edbd1 100644--- a/types/settings/settings.go+++ b/types/settings/settings.go@@ -37,6 +37,12 @@ type Update struct {// Required: falseProjectsDirectory *string `json:"projectsDirectory,omitempty"`+ // TemplatesDirectory is the directory path where local compose template folders are discovered.+ // Must be an absolute path.+ //+ // Required: false+ TemplatesDirectory *string `json:"templatesDirectory,omitempty"`+// FollowProjectSymlinks controls whether symlinked child directories in the projects directory are discovered as projects.//// Required: false