Skip to content

Modern Docker Management, Designed for Everyone

fix: unable to use templates due to 'not found' error (#2634)

Kyle Mendell7da176b1585b
diff --git a/backend/api/handlers/templates.go b/backend/api/handlers/templates.go
index 7752e4d5a..66d26ed4a 100644
--- a/backend/api/handlers/templates.go
+++ b/backend/api/handlers/templates.go
@@ -4,6 +4,7 @@ import (
"context"
"encoding/json"
"net/http"
+ "net/url"
"github.com/danielgtaylor/huma/v2"
humamw "github.com/getarcaneapp/arcane/backend/api/middleware"
@@ -465,9 +466,17 @@ func (h *TemplateHandler) GetTemplate(ctx context.Context, input *GetTemplateInp
return nil, huma.Error400BadRequest((&common.TemplateIDRequiredError{}).Error())
}
- tmpl, err := h.templateService.GetTemplate(ctx, input.ID)
+ // Path parameter arrives URL-encoded (e.g. "remote%3Areg%3Aslug" for remote IDs that
+ // contain ':' separators). Chi/Huma do not auto-decode, so decode here before
+ // matching against cached / stored template IDs.
+ id, decodeErr := url.PathUnescape(input.ID)
+ if decodeErr != nil {
+ return nil, huma.Error400BadRequest((&common.TemplateIDRequiredError{}).Error())
+ }
+
+ tmpl, err := h.templateService.GetTemplate(ctx, id)
if err != nil {
- if err.Error() == "template not found" {
+ if common.IsTemplateNotFoundError(err) {
return nil, huma.Error404NotFound((&common.TemplateNotFoundError{}).Error())
}
return nil, huma.Error500InternalServerError((&common.TemplateRetrievalError{Err: err}).Error())
@@ -496,8 +505,16 @@ func (h *TemplateHandler) GetTemplateContent(ctx context.Context, input *GetTemp
return nil, huma.Error400BadRequest((&common.TemplateIDRequiredError{}).Error())
}
- contentData, err := h.templateService.GetTemplateContentWithParsedData(ctx, input.ID)
+ id, decodeErr := url.PathUnescape(input.ID)
+ if decodeErr != nil {
+ return nil, huma.Error400BadRequest((&common.TemplateIDRequiredError{}).Error())
+ }
+
+ contentData, err := h.templateService.GetTemplateContentWithParsedData(ctx, id)
if err != nil {
+ if common.IsTemplateNotFoundError(err) {
+ return nil, huma.Error404NotFound((&common.TemplateNotFoundError{}).Error())
+ }
return nil, huma.Error500InternalServerError((&common.TemplateContentError{Err: err}).Error())
}
@@ -553,6 +570,11 @@ func (h *TemplateHandler) UpdateTemplate(ctx context.Context, input *UpdateTempl
return nil, huma.Error400BadRequest((&common.TemplateIDRequiredError{}).Error())
}
+ id, decodeErr := url.PathUnescape(input.ID)
+ if decodeErr != nil {
+ return nil, huma.Error400BadRequest((&common.TemplateIDRequiredError{}).Error())
+ }
+
updates := &models.ComposeTemplate{
Name: input.Body.Name,
Description: input.Body.Description,
@@ -564,14 +586,14 @@ func (h *TemplateHandler) UpdateTemplate(ctx context.Context, input *UpdateTempl
updates.EnvContent = nil
}
- if err := h.templateService.UpdateTemplate(ctx, input.ID, updates); err != nil {
- if err.Error() == "template not found" {
+ if err := h.templateService.UpdateTemplate(ctx, id, updates); err != nil {
+ if common.IsTemplateNotFoundError(err) {
return nil, huma.Error404NotFound((&common.TemplateNotFoundError{}).Error())
}
return nil, huma.Error500InternalServerError((&common.TemplateUpdateError{Err: err}).Error())
}
- updated, err := h.templateService.GetTemplate(ctx, input.ID)
+ updated, err := h.templateService.GetTemplate(ctx, id)
if err != nil {
return nil, huma.Error500InternalServerError((&common.TemplateRetrievalError{Err: err}).Error())
}
@@ -599,8 +621,13 @@ func (h *TemplateHandler) DeleteTemplate(ctx context.Context, input *DeleteTempl
return nil, huma.Error400BadRequest((&common.TemplateIDRequiredError{}).Error())
}
- if err := h.templateService.DeleteTemplate(ctx, input.ID); err != nil {
- if err.Error() == "template not found" {
+ id, decodeErr := url.PathUnescape(input.ID)
+ if decodeErr != nil {
+ return nil, huma.Error400BadRequest((&common.TemplateIDRequiredError{}).Error())
+ }
+
+ if err := h.templateService.DeleteTemplate(ctx, id); err != nil {
+ if common.IsTemplateNotFoundError(err) {
return nil, huma.Error404NotFound((&common.TemplateNotFoundError{}).Error())
}
return nil, huma.Error500InternalServerError((&common.TemplateDeletionError{Err: err}).Error())
@@ -626,9 +653,17 @@ func (h *TemplateHandler) DownloadTemplate(ctx context.Context, input *DownloadT
return nil, huma.Error400BadRequest((&common.TemplateIDRequiredError{}).Error())
}
- tmpl, err := h.templateService.GetTemplate(ctx, input.ID)
+ id, decodeErr := url.PathUnescape(input.ID)
+ if decodeErr != nil {
+ return nil, huma.Error400BadRequest((&common.TemplateIDRequiredError{}).Error())
+ }
+
+ tmpl, err := h.templateService.GetTemplate(ctx, id)
if err != nil {
- return nil, huma.Error404NotFound((&common.TemplateNotFoundError{}).Error())
+ if common.IsTemplateNotFoundError(err) {
+ return nil, huma.Error404NotFound((&common.TemplateNotFoundError{}).Error())
+ }
+ return nil, huma.Error500InternalServerError((&common.TemplateDownloadError{Err: err}).Error())
}
if !tmpl.IsRemote {
return nil, huma.Error400BadRequest((&common.TemplateAlreadyLocalError{}).Error())
diff --git a/backend/internal/bootstrap/jobs_bootstrap.go b/backend/internal/bootstrap/jobs_bootstrap.go
index 7d169c203..429a1c3ce 100644
--- a/backend/internal/bootstrap/jobs_bootstrap.go
+++ b/backend/internal/bootstrap/jobs_bootstrap.go
@@ -84,6 +84,13 @@ func setupSettingsCallbacks(lifecycleCtx context.Context, appServices *Services,
}
}
}
+ appServices.Settings.OnTemplatesDirectoryChanged = func(_ context.Context) {
+ if fsWatcherJob != nil {
+ if err := fsWatcherJob.RestartTemplatesWatcher(lifecycleCtx); err != nil {
+ slog.WarnContext(lifecycleCtx, "Failed to restart templates filesystem watcher", "error", err)
+ }
+ }
+ }
appServices.Settings.OnScheduledPruneSettingsChanged = func(_ context.Context) {
if err := newScheduler.RescheduleJob(lifecycleCtx, scheduledPruneJob); err != nil {
slog.WarnContext(lifecycleCtx, "Failed to reschedule scheduled-prune job", "error", err)
diff --git a/backend/internal/common/errors.go b/backend/internal/common/errors.go
index 4c226a3c0..66b7d324e 100644
--- a/backend/internal/common/errors.go
+++ b/backend/internal/common/errors.go
@@ -1046,6 +1046,10 @@ func (e *TemplateNotFoundError) Error() string {
return "Template not found"
}
+func IsTemplateNotFoundError(err error) bool {
+ return isErrorTypeInternal[*TemplateNotFoundError](err)
+}
+
type TemplateRetrievalError struct {
Err error
}
diff --git a/backend/internal/config/config.go b/backend/internal/config/config.go
index 0883fef8e..caeb806ed 100644
--- a/backend/internal/config/config.go
+++ b/backend/internal/config/config.go
@@ -60,6 +60,7 @@ type Config struct {
DockerHost string `env:"DOCKER_HOST" default:"unix:///var/run/docker.sock"`
DockerConfig string `env:"DOCKER_CONFIG" default:""`
ProjectsDirectory string `env:"PROJECTS_DIRECTORY" default:"/app/data/projects"`
+ TemplatesDirectory string `env:"TEMPLATES_DIRECTORY" default:"/app/data/templates"`
ProjectScanMaxDepth int `env:"PROJECT_SCAN_MAX_DEPTH" default:"3"`
ProjectScanSkipDirs string `env:"PROJECT_SCAN_SKIP_DIRS" default:".git,node_modules,vendor,.venv,venv,__pycache__,.cache,dist,build,target,.next,.nuxt,.svelte-kit"`
LogJson bool `env:"LOG_JSON" default:"false"`
diff --git a/backend/internal/configschema/schema_test.go b/backend/internal/configschema/schema_test.go
index 1c172552d..617b7df3f 100644
--- a/backend/internal/configschema/schema_test.go
+++ b/backend/internal/configschema/schema_test.go
@@ -257,6 +257,7 @@ var expectedEnvConfigVars = []string{
"PROXY_REQUEST_TIMEOUT",
"PUID",
"REGISTRY_TIMEOUT",
+ "TEMPLATES_DIRECTORY",
"TLS_CERT_FILE",
"TLS_ENABLED",
"TLS_KEY_FILE",
@@ -353,6 +354,7 @@ var expectedSettingOverrideKeys = []string{
"scheduledPruneVolumes",
"sidebarHoverExpansion",
"swarmStackSourcesDirectory",
+ "templatesDirectory",
"trivyConcurrentScanContainers",
"trivyConfig",
"trivyCpuLimit",
diff --git a/backend/internal/models/settings.go b/backend/internal/models/settings.go
index 9e5b5c209..8e5be4809 100644
--- a/backend/internal/models/settings.go
+++ b/backend/internal/models/settings.go
@@ -70,6 +70,7 @@ func (s SettingVariable) AsDurationSeconds() time.Duration {
type Settings struct {
// General category
ProjectsDirectory SettingVariable `key:"projectsDirectory,envOverride" meta:"label=Projects Directory;type=text;keywords=projects,directory,path,folder,location,storage,files,compose,docker-compose;category=internal;description=Configure where project files are stored"`
+ TemplatesDirectory SettingVariable `key:"templatesDirectory,envOverride" meta:"label=Templates Directory;type=text;keywords=templates,directory,path,folder,location,storage,compose,docker-compose;category=internal;description=Configure where local compose template folders are discovered"`
FollowProjectSymlinks SettingVariable `key:"followProjectSymlinks,envOverride" meta:"label=Follow Project Symlinks;type=boolean;keywords=projects,symlink,symlinks,symbolic links,compose,directory,discovery;category=general;description=Treat symlinked child directories inside the projects directory as Docker Compose projects"`
SwarmStackSourcesDirectory SettingVariable `key:"swarmStackSourcesDirectory,envOverride" meta:"label=Swarm Stack Sources Directory;type=text;keywords=swarm,stacks,stack,source,sources,directory,path,folder,location,storage,compose,env;category=internal;description=Configure where swarm stack source files are stored"`
DiskUsagePath SettingVariable `key:"diskUsagePath" meta:"label=Disk Usage Path;type=text;keywords=disk,usage,path,storage,folder,files;category=general;description=Path used for disk usage calculations"`
diff --git a/backend/internal/services/settings_service.go b/backend/internal/services/settings_service.go
index 85b021fba..cc0a3b1b5 100644
--- a/backend/internal/services/settings_service.go
+++ b/backend/internal/services/settings_service.go
@@ -37,6 +37,7 @@ type SettingsService struct {
OnImagePollingSettingsChanged func(ctx context.Context)
OnAutoUpdateSettingsChanged func(ctx context.Context)
OnProjectsDirectoryChanged func(ctx context.Context)
+ OnTemplatesDirectoryChanged func(ctx context.Context)
OnScheduledPruneSettingsChanged func(ctx context.Context)
OnVulnerabilityScanSettingsChanged func(ctx context.Context)
OnAutoHealSettingsChanged func(ctx context.Context)
@@ -108,6 +109,7 @@ func (s *SettingsService) getDefaultSettings() *models.Settings {
func DefaultSettingsConfig() *models.Settings {
return &models.Settings{
ProjectsDirectory: models.SettingVariable{Value: "/app/data/projects"},
+ TemplatesDirectory: models.SettingVariable{Value: "/app/data/templates"},
FollowProjectSymlinks: models.SettingVariable{Value: "false"},
SwarmStackSourcesDirectory: models.SettingVariable{Value: "/app/data/swarm/sources"},
DiskUsagePath: models.SettingVariable{Value: "/app/data/projects"},
@@ -709,6 +711,11 @@ func (s *SettingsService) UpdateSettings(ctx context.Context, updates settings.U
}) && s.OnProjectsDirectoryChanged != nil {
s.OnProjectsDirectoryChanged(ctx)
}
+ if slices.ContainsFunc(valuesToUpdate, func(sv models.SettingVariable) bool {
+ return sv.Key == "templatesDirectory"
+ }) && s.OnTemplatesDirectoryChanged != nil {
+ s.OnTemplatesDirectoryChanged(ctx)
+ }
if len(changedTimeouts) > 0 && s.OnTimeoutSettingsChanged != nil {
s.OnTimeoutSettingsChanged(ctx, changedTimeouts)
}
diff --git a/backend/internal/services/template_service.go b/backend/internal/services/template_service.go
index 08509f04b..39b7f543c 100644
--- a/backend/internal/services/template_service.go
+++ b/backend/internal/services/template_service.go
@@ -82,10 +82,6 @@ func NewTemplateService(ctx context.Context, db *database.DB, httpClient *http.C
httpClient = http.DefaultClient
}
- if err := projects.EnsureDefaultTemplates(ctx); err != nil {
- slog.WarnContext(ctx, "failed to ensure default templates", "error", err)
- }
-
service := &TemplateService{
db: db,
httpClient: httpClient,
@@ -96,20 +92,44 @@ func NewTemplateService(ctx context.Context, db *database.DB, httpClient *http.C
registryErrors: make(map[string]string),
}
service.safeHTTPClient = service.newSafeHTTPClientInternal()
+
+ if err := projects.EnsureDefaultTemplates(ctx, service.configuredTemplatesDirSettingInternal(ctx)); err != nil {
+ slog.WarnContext(ctx, "failed to ensure default templates", "error", err)
+ }
+
return service
}
+// configuredTemplatesDirSettingInternal returns the raw user-configured templates
+// directory string (from the settingsService), without resolving it to an absolute
+// path. Use this when calling helpers like projects.GetTemplatesDirectory that
+// perform their own resolution.
+func (s *TemplateService) configuredTemplatesDirSettingInternal(ctx context.Context) string {
+ if s.settingsService == nil {
+ return ""
+ }
+ return s.settingsService.GetStringSetting(ctx, "templatesDirectory", "/app/data/templates")
+}
+
+// getTemplatesDirectoryInternal resolves the effective templates directory by
+// applying the same resolution rules used elsewhere for the projects directory.
+func (s *TemplateService) getTemplatesDirectoryInternal(ctx context.Context) (string, error) {
+ return projects.GetTemplatesDirectory(ctx, strings.TrimSpace(s.configuredTemplatesDirSettingInternal(ctx)))
+}
+
func (s *TemplateService) ensureRemoteTemplatesLoaded(ctx context.Context) error {
s.remoteMu.Lock()
- // If cache is valid, return
- if s.remoteCache.templates != nil && time.Since(s.remoteCache.lastFetch) < remoteCacheDuration {
+ // If cache has entries and is fresh, return.
+ // An empty slice is treated as "no cache" so callers always trigger a blocking
+ // refresh — otherwise a single failed/empty refresh would poison the cache.
+ if len(s.remoteCache.templates) > 0 && time.Since(s.remoteCache.lastFetch) < remoteCacheDuration {
s.remoteMu.Unlock()
return nil
}
- // If we have cache (even stale) and are not already refreshing, trigger background refresh
- if s.remoteCache.templates != nil {
+ // If we have a non-empty stale cache and nobody is refreshing, trigger background refresh.
+ if len(s.remoteCache.templates) > 0 {
if !s.remoteCache.refreshing {
s.remoteCache.refreshing = true
s.remoteMu.Unlock()
@@ -236,8 +256,6 @@ func (s *TemplateService) GetAllTemplatesPaginated(ctx context.Context, params p
return result.Items, paginationResp, nil
}
-var ErrTemplateNotFound = errors.New("template not found")
-
// envKeyPattern is the POSIX env-name shape used to validate global variable
// keys before they are persisted to .env.global. Keys that do not match are
// rejected with a common.InvalidEnvKeyError.
@@ -256,19 +274,48 @@ func (s *TemplateService) GetTemplate(ctx context.Context, id string) (*models.C
}
if err := s.ensureRemoteTemplatesLoaded(ctx); err != nil {
- return nil, fmt.Errorf("template not found (failed to load remote templates): %w", err)
+ return nil, fmt.Errorf("template %q lookup failed: registry refresh error: %w", id, err)
}
- s.remoteMu.RLock()
- copied := cloneRemoteTemplates(s.remoteCache.templates)
- s.remoteMu.RUnlock()
- for _, remoteTemplate := range copied {
- if remoteTemplate.ID == id {
- return new(remoteTemplate), nil
+ if found := s.lookupRemoteFromCacheInternal(id); found != nil {
+ return found, nil
+ }
+
+ // Remote IDs (remote:registry:slug) deserve a synchronous force-refresh on miss
+ // before we return "not found" — the cache may be stale or the previous refresh
+ // silently returned empty.
+ if strings.HasPrefix(id, remoteIDPrefix+":") {
+ slog.InfoContext(ctx, "remote template not in cache, forcing registry refresh", "templateID", id, "cacheSize", s.remoteCacheSizeInternal())
+ if _, refreshErr := s.refreshRemoteTemplates(ctx); refreshErr != nil {
+ return nil, fmt.Errorf("template %q not found and registry refresh failed: %w", id, refreshErr)
+ }
+ if found := s.lookupRemoteFromCacheInternal(id); found != nil {
+ return found, nil
+ }
+ return nil, &common.TemplateNotFoundError{
+ Err: fmt.Errorf("template %q not found in any registered registry (cache size=%d after refresh)", id, s.remoteCacheSizeInternal()),
}
}
- return nil, ErrTemplateNotFound
+ return nil, &common.TemplateNotFoundError{}
+}
+
+func (s *TemplateService) lookupRemoteFromCacheInternal(id string) *models.ComposeTemplate {
+ s.remoteMu.RLock()
+ defer s.remoteMu.RUnlock()
+ for i := range s.remoteCache.templates {
+ if s.remoteCache.templates[i].ID == id {
+ cloned := cloneRemoteTemplates(s.remoteCache.templates[i : i+1])
+ return &cloned[0]
+ }
+ }
+ return nil
+}
+
+func (s *TemplateService) remoteCacheSizeInternal() int {
+ s.remoteMu.RLock()
+ defer s.remoteMu.RUnlock()
+ return len(s.remoteCache.templates)
}
func (s *TemplateService) CreateTemplate(ctx context.Context, template *models.ComposeTemplate) error {
@@ -291,7 +338,7 @@ func (s *TemplateService) UpdateTemplate(ctx context.Context, id string, updates
var existing models.ComposeTemplate
if err := tx.Where("id = ?", id).First(&existing).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
- return fmt.Errorf("template not found")
+ return &common.TemplateNotFoundError{}
}
return fmt.Errorf("failed to find template: %w", err)
}
@@ -319,7 +366,7 @@ func (s *TemplateService) DeleteTemplate(ctx context.Context, id string) error {
var existing models.ComposeTemplate
if err := tx.Where("id = ?", id).First(&existing).Error; err != nil {
if errors.Is(err, gorm.ErrRecordNotFound) {
- return fmt.Errorf("template not found")
+ return &common.TemplateNotFoundError{}
}
return fmt.Errorf("failed to find template: %w", err)
}
@@ -328,15 +375,14 @@ func (s *TemplateService) DeleteTemplate(ctx context.Context, id string) error {
return fmt.Errorf("cannot delete remote template directly")
}
- baseDir, err := projects.GetTemplatesDirectory(ctx)
+ baseDir, err := s.getTemplatesDirectoryInternal(ctx)
if err != nil {
return fmt.Errorf("failed to get templates directory: %w", err)
} else {
templatePath := filepath.Join(baseDir, existing.Name)
if stat, err := os.Stat(templatePath); err == nil && stat.IsDir() {
- composeFile := filepath.Join(templatePath, "compose.yaml")
- if _, err := os.Stat(composeFile); err == nil {
+ if _, err := projects.DetectComposeFile(templatePath); err == nil {
if err := os.RemoveAll(templatePath); err != nil {
return fmt.Errorf("failed to delete template directory: %w", err)
}
@@ -707,8 +753,15 @@ func (s *TemplateService) convertRemoteToLocal(remote tmpl.RemoteTemplate, regis
}
func (s *TemplateService) FetchTemplateContent(ctx context.Context, template *models.ComposeTemplate) (string, string, error) {
- if !template.IsRemote || template.Metadata == nil || template.Metadata.RemoteURL == nil {
- return template.Content, "", fmt.Errorf("not a remote template or missing remote URL")
+ if !template.IsRemote {
+ envContent := ""
+ if template.EnvContent != nil {
+ envContent = *template.EnvContent
+ }
+ return template.Content, envContent, nil
+ }
+ if template.Metadata == nil || template.Metadata.RemoteURL == nil || strings.TrimSpace(*template.Metadata.RemoteURL) == "" {
+ return "", "", fmt.Errorf("remote template %q is missing compose_url in registry metadata", template.ID)
}
return s.fetchRemoteTemplateFiles(ctx, template)
@@ -809,7 +862,7 @@ func (s *TemplateService) DownloadTemplate(ctx context.Context, remoteTemplate *
base := s.templateBaseFromRemote(remoteTemplate)
- dir, composePath, envPath, err := projects.EnsureTemplateDir(ctx, base)
+ dir, composePath, envPath, err := projects.EnsureTemplateDir(ctx, s.configuredTemplatesDirSettingInternal(ctx), base)
if err != nil {
return nil, err
}
@@ -958,12 +1011,17 @@ func (s *TemplateService) upsertFilesystemTemplate(ctx context.Context, name, de
return s.db.WithContext(ctx).Transaction(func(tx *gorm.DB) error {
var existing models.ComposeTemplate
+ // Match on description (which encodes the absolute compose path) OR name (folder name).
+ // Matching on name as a fallback keeps a single row across templates-directory
+ // reconfigurations or compose-filename renames within the same folder, but
+ // only for templates previously imported from disk.
q := tx.
- Where("is_remote = ? AND registry_id IS NULL AND description = ?", false, desc).
+ Where("is_remote = ? AND registry_id IS NULL AND (description = ? OR (name = ? AND description LIKE ?))", false, desc, name, "Imported from %").
First(&existing)
if q.Error == nil {
existing.Name = name
+ existing.Description = desc
existing.Content = compose
existing.EnvContent = envPtr
existing.IsCustom = true
@@ -1014,7 +1072,7 @@ func (s *TemplateService) syncFilesystemTemplatesInternal(ctx context.Context) e
return nil
}
- dir, err := projects.GetTemplatesDirectory(ctx)
+ dir, err := s.getTemplatesDirectoryInternal(ctx)
if err != nil {
return fmt.Errorf("ensure templates dir: %w", err)
}
diff --git a/backend/internal/services/template_service_test.go b/backend/internal/services/template_service_test.go
index a97cd132f..dff9e5589 100644
--- a/backend/internal/services/template_service_test.go
+++ b/backend/internal/services/template_service_test.go
@@ -257,10 +257,14 @@ services:
client, lookupIP, baseURL := makePublicTestClient(t, server)
+ settingsSvc := minimalSettingsServiceForTest(t)
+ require.NoError(t, settingsSvc.UpdateSetting(context.Background(), "templatesDirectory", filepath.Join(tempDir, "templates")))
+
service := &TemplateService{
db: setupTemplateServiceTestDB(t),
httpClient: client,
lookupIP: lookupIP,
+ settingsService: settingsSvc,
registryFetchMeta: make(map[string]*registryFetchMeta),
}
@@ -383,9 +387,9 @@ func TestFetchRaw_BlocksUnsafeRemoteURL(t *testing.T) {
func TestSyncFilesystemTemplatesInternal_PopulatesIconURL(t *testing.T) {
tempDir := t.TempDir()
- setTestWorkingDir(t, tempDir)
- templateDir := filepath.Join(tempDir, "data", "templates", "example")
+ templatesRoot := filepath.Join(tempDir, "templates")
+ templateDir := filepath.Join(templatesRoot, "example")
require.NoError(t, os.MkdirAll(templateDir, 0o755))
require.NoError(t, os.WriteFile(filepath.Join(templateDir, "compose.yaml"), []byte(`x-arcane:
icon: https://cdn.example/local.png
@@ -394,9 +398,13 @@ services:
image: nginx:alpine
`), 0o644))
+ settingsSvc := minimalSettingsServiceForTest(t)
+ require.NoError(t, settingsSvc.UpdateSetting(context.Background(), "templatesDirectory", templatesRoot))
+
service := &TemplateService{
db: setupTemplateServiceTestDB(t),
httpClient: http.DefaultClient,
+ settingsService: settingsSvc,
registryFetchMeta: make(map[string]*registryFetchMeta),
}
@@ -409,6 +417,88 @@ services:
require.Equal(t, "https://cdn.example/local.png", *stored.Metadata.IconURL)
}
+func TestGetTemplate_ForceRefreshesRemoteCacheOnMiss(t *testing.T) {
+ tempDir := t.TempDir()
+ setTestWorkingDir(t, tempDir)
+
+ var registryHits atomic.Int32
+ server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+ if r.URL.Path != "/registry.json" {
+ http.NotFound(w, r)
+ return
+ }
+ registryHits.Add(1)
+ _, _ = w.Write([]byte(`{
+ "name": "Demo Registry",
+ "description": "Test",
+ "version": "1.0.0",
+ "author": "Arcane",
+ "templates": [
+ {
+ "id": "affine",
+ "name": "AFFiNE",
+ "description": "",
+ "version": "1.0.0",
+ "author": "Arcane",
+ "compose_url": "",
+ "env_url": "",
+ "documentation_url": "",
+ "tags": []
+ }
+ ]
+}`))
+ }))
+ defer server.Close()
+
+ client, lookupIP, baseURL := makePublicTestClient(t, server)
+ db := setupTemplateServiceTestDB(t)
+
+ registry := &models.TemplateRegistry{
+ BaseModel: models.BaseModel{ID: "reg-1"},
+ Name: "Demo",
+ URL: baseURL + "/registry.json",
+ Enabled: true,
+ }
+ require.NoError(t, db.WithContext(context.Background()).Create(registry).Error)
+
+ settingsSvc := minimalSettingsServiceForTest(t)
+ // Point templates+projects directories at tempDir so sync calls don't try to
+ // touch the real /app filesystem during tests.
+ require.NoError(t, settingsSvc.UpdateSetting(context.Background(), "templatesDirectory", filepath.Join(tempDir, "templates")))
+ require.NoError(t, settingsSvc.UpdateSetting(context.Background(), "projectsDirectory", filepath.Join(tempDir, "projects")))
+
+ service := &TemplateService{
+ db: db,
+ httpClient: client,
+ lookupIP: lookupIP,
+ settingsService: settingsSvc,
+ registryFetchMeta: make(map[string]*registryFetchMeta),
+ }
+
+ // Cache starts empty; GetTemplate for a remote ID should force a refresh and find the template.
+ got, err := service.GetTemplate(context.Background(), "remote:reg-1:affine")
+ require.NoError(t, err)
+ require.NotNil(t, got)
+ require.True(t, got.IsRemote)
+ require.Equal(t, "AFFiNE", got.Name)
+ require.GreaterOrEqual(t, registryHits.Load(), int32(1), "registry should be fetched at least once")
+
+ // Unknown remote ID still surfaces a clear error after the forced refresh.
+ _, err = service.GetTemplate(context.Background(), "remote:reg-1:does-not-exist")
+ require.Error(t, err)
+ require.Contains(t, err.Error(), "not found")
+}
+
+func minimalSettingsServiceForTest(t *testing.T) *SettingsService {
+ t.Helper()
+ db, err := gorm.Open(glsqlite.Open(":memory:"), &gorm.Config{})
+ require.NoError(t, err)
+ require.NoError(t, db.AutoMigrate(&models.SettingVariable{}))
+ svc, err := NewSettingsService(context.Background(), &database.DB{DB: db})
+ require.NoError(t, err)
+ return svc
+}
+
func TestUpdateGlobalVariables_RejectsNewlineInjectionKey(t *testing.T) {
projectsDir := t.TempDir()
diff --git a/backend/pkg/projects/fs_templates.go b/backend/pkg/projects/fs_templates.go
index 048142a04..20addb369 100644
--- a/backend/pkg/projects/fs_templates.go
+++ b/backend/pkg/projects/fs_templates.go
@@ -2,6 +2,7 @@ package projects
import (
"context"
+ "errors"
"fmt"
"os"
"path/filepath"
@@ -12,12 +13,13 @@ import (
)
func ReadFolderComposeTemplate(baseDir, folder string) (string, *string, string, bool, error) {
- composePath := filepath.Join(baseDir, folder, "compose.yaml")
- if _, err := os.Stat(composePath); err != nil {
- if os.IsNotExist(err) {
+ folderPath := filepath.Join(baseDir, folder)
+ composePath, err := DetectComposeFile(folderPath)
+ if err != nil {
+ if errors.Is(err, errComposeFileNotFoundInternal) {
return "", nil, "", false, nil
}
- return "", nil, "", false, fmt.Errorf("stat compose: %w", err)
+ return "", nil, "", false, fmt.Errorf("detect compose file: %w", err)
}
b, err := os.ReadFile(composePath)
@@ -27,14 +29,15 @@ func ReadFolderComposeTemplate(baseDir, folder string) (string, *string, string,
var envPtr *string
for _, envName := range []string{".env.example", ".env"} {
- envPath := filepath.Join(baseDir, folder, envName)
- if eb, err := os.ReadFile(envPath); err == nil {
- envPtr = new(string(eb))
+ envPath := filepath.Join(folderPath, envName)
+ if eb, rerr := os.ReadFile(envPath); rerr == nil {
+ s := string(eb)
+ envPtr = &s
break
}
}
- desc := fmt.Sprintf("Imported from %s/%s/compose.yaml", baseDir, folder)
+ desc := fmt.Sprintf("Imported from %s", composePath)
return string(b), envPtr, desc, true, nil
}
@@ -50,8 +53,8 @@ func Slugify(in string) string {
return strings.Trim(in, "-")
}
-func EnsureTemplateDir(ctx context.Context, base string) (dir, composePath, envPath string, err error) {
- baseDir, derr := GetTemplatesDirectory(ctx)
+func EnsureTemplateDir(ctx context.Context, templatesDir, base string) (dir, composePath, envPath string, err error) {
+ baseDir, derr := GetTemplatesDirectory(ctx, templatesDir)
if derr != nil {
return "", "", "", fmt.Errorf("ensure templates dir: %w", derr)
}
@@ -84,8 +87,8 @@ func WriteTemplateFiles(composePath, envPath, composeContent, envContent string)
return &envContent, nil
}
-func EnsureDefaultTemplates(ctx context.Context) error {
- templatesDir, err := GetTemplatesDirectory(ctx)
+func EnsureDefaultTemplates(ctx context.Context, configuredTemplatesDir string) error {
+ templatesDir, err := GetTemplatesDirectory(ctx, configuredTemplatesDir)
if err != nil {
return fmt.Errorf("get templates directory: %w", err)
}
diff --git a/backend/pkg/projects/fs_templates_test.go b/backend/pkg/projects/fs_templates_test.go
new file mode 100644
index 000000000..53efc4a16
--- /dev/null
+++ b/backend/pkg/projects/fs_templates_test.go
@@ -0,0 +1,62 @@
+package projects
+
+import (
+ "os"
+ "path/filepath"
+ "strings"
+ "testing"
+
+ "github.com/stretchr/testify/assert"
+ "github.com/stretchr/testify/require"
+)
+
+func TestReadFolderComposeTemplate_DetectsAllVariants(t *testing.T) {
+ t.Parallel()
+
+ composeContent := "services:\n app:\n image: nginx:alpine\n"
+
+ testCases := []struct {
+ name string
+ fileName string
+ }{
+ {name: "compose.yaml", fileName: "compose.yaml"},
+ {name: "compose.yml", fileName: "compose.yml"},
+ {name: "docker-compose.yml", fileName: "docker-compose.yml"},
+ }
+
+ for _, tc := range testCases {
+ t.Run(tc.name, func(t *testing.T) {
+ t.Parallel()
+
+ baseDir := t.TempDir()
+ folder := "demo"
+ folderPath := filepath.Join(baseDir, folder)
+ require.NoError(t, os.MkdirAll(folderPath, 0o755))
+ require.NoError(t, os.WriteFile(filepath.Join(folderPath, tc.fileName), []byte(composeContent), 0o600))
+
+ content, env, desc, found, err := ReadFolderComposeTemplate(baseDir, folder)
+ require.NoError(t, err)
+ require.True(t, found, "expected template to be detected for %s", tc.fileName)
+ assert.Equal(t, composeContent, content)
+ assert.Nil(t, env)
+ assert.True(t, strings.Contains(desc, tc.fileName), "description should reference detected filename, got %q", desc)
+ })
+ }
+}
+
+func TestReadFolderComposeTemplate_ReadsEnvExample(t *testing.T) {
+ t.Parallel()
+
+ baseDir := t.TempDir()
+ folder := "envtest"
+ folderPath := filepath.Join(baseDir, folder)
+ require.NoError(t, os.MkdirAll(folderPath, 0o755))
+ require.NoError(t, os.WriteFile(filepath.Join(folderPath, "compose.yml"), []byte("services: {}\n"), 0o600))
+ require.NoError(t, os.WriteFile(filepath.Join(folderPath, ".env.example"), []byte("KEY=value\n"), 0o600))
+
+ _, env, _, found, err := ReadFolderComposeTemplate(baseDir, folder)
+ require.NoError(t, err)
+ require.True(t, found)
+ require.NotNil(t, env)
+ assert.Equal(t, "KEY=value\n", *env)
+}
diff --git a/backend/pkg/projects/fs_util.go b/backend/pkg/projects/fs_util.go
index 2482981d2..7408b7401 100644
--- a/backend/pkg/projects/fs_util.go
+++ b/backend/pkg/projects/fs_util.go
@@ -130,15 +130,15 @@ func HasComposeRootKeysInFile(path string) (bool, error) {
return hasServices || hasInclude, nil
}
-func GetTemplatesDirectory(ctx context.Context) (string, error) {
- templatesDir := filepath.Join("data", "templates")
- if _, err := os.Stat(templatesDir); os.IsNotExist(err) {
- if err := os.MkdirAll(templatesDir, pkgutils.DirPerm); err != nil {
+func GetTemplatesDirectory(ctx context.Context, templatesDir string) (string, error) {
+ resolved := ResolveConfiguredContainerDirectory(templatesDir, "/app/data/templates")
+ if _, err := os.Stat(resolved); os.IsNotExist(err) {
+ if err := os.MkdirAll(resolved, pkgutils.DirPerm); err != nil {
return "", err
}
- slog.InfoContext(ctx, "Created templates directory", "path", templatesDir)
+ slog.InfoContext(ctx, "Created templates directory", "path", resolved)
}
- return templatesDir, nil
+ return resolved, nil
}
func ReadProjectDirectoryFiles(projectPath string, shownFiles map[string]bool, maxDepth int, skipDirectories string) ([]project.IncludeFile, error) {
diff --git a/backend/pkg/projects/load.go b/backend/pkg/projects/load.go
index 9130fbfb1..bf636ee42 100644
--- a/backend/pkg/projects/load.go
+++ b/backend/pkg/projects/load.go
@@ -2,6 +2,7 @@ package projects
import (
"context"
+ "errors"
"fmt"
"log/slog"
"maps"
@@ -17,6 +18,8 @@ import (
"github.com/docker/compose/v5/pkg/api"
)
+var errComposeFileNotFoundInternal = errors.New("no compose file found")
+
var ProjectFileCandidates = []string{
"compose.yaml",
"compose.yml",
@@ -133,7 +136,7 @@ func DetectComposeFile(dir string) (string, error) {
case len(customCandidates) > 1:
return "", fmt.Errorf("multiple custom compose files found in %q", dir)
default:
- return "", fmt.Errorf("no compose file found in %q", dir)
+ return "", fmt.Errorf("%w in %q", errComposeFileNotFoundInternal, dir)
}
}
diff --git a/backend/pkg/scheduler/filesystem_watcher_job.go b/backend/pkg/scheduler/filesystem_watcher_job.go
index 75f679105..0f375494d 100644
--- a/backend/pkg/scheduler/filesystem_watcher_job.go
+++ b/backend/pkg/scheduler/filesystem_watcher_job.go
@@ -68,21 +68,28 @@ func (j *FilesystemWatcherJob) Start(ctx context.Context) error {
j.projectsWatcher = sw
- templatesDir, err := projects.GetTemplatesDirectory(ctx)
+ templatesDir, err := projects.GetTemplatesDirectory(ctx, settings.TemplatesDirectory.Value)
if err != nil {
return err
}
if j.templateService != nil {
- tw, err := fswatch.NewWatcher(templatesDir, fswatch.WatcherOptions{
- Debounce: 3 * time.Second,
- OnChange: j.handleTemplatesChange,
- MaxDepth: 1,
- })
- if err != nil {
- return err
+ if directoriesOverlapInternal(projectsDirectory, templatesDir) {
+ slog.ErrorContext(ctx,
+ "Templates and projects directories overlap; templates watcher disabled to prevent compose files being treated as projects",
+ "projectsDirectory", projectsDirectory,
+ "templatesDirectory", templatesDir)
+ } else {
+ tw, err := fswatch.NewWatcher(templatesDir, fswatch.WatcherOptions{
+ Debounce: 3 * time.Second,
+ OnChange: j.handleTemplatesChange,
+ MaxDepth: 1,
+ })
+ if err != nil {
+ return err
+ }
+ j.templatesWatcher = tw
}
- j.templatesWatcher = tw
}
if err := j.projectsWatcher.Start(ctx); err != nil {
@@ -242,3 +249,76 @@ func (j *FilesystemWatcherJob) projectWatcherOptionsInternal(followProjectSymlin
FollowSymlinkDirs: followProjectSymlinks,
}
}
+
+func (j *FilesystemWatcherJob) RestartTemplatesWatcher(ctx context.Context) error {
+ if j.templateService == nil {
+ return nil
+ }
+ slog.InfoContext(ctx, "Restarting templates filesystem watcher")
+
+ j.mu.Lock()
+ oldTemplatesWatcher := j.templatesWatcher
+ j.templatesWatcher = nil
+ j.mu.Unlock()
+
+ if oldTemplatesWatcher != nil {
+ if err := oldTemplatesWatcher.Stop(); err != nil {
+ slog.WarnContext(ctx, "Failed to stop templates watcher during restart", "error", err)
+ }
+ }
+
+ settings, err := j.settingsService.GetSettings(ctx)
+ if err != nil {
+ return err
+ }
+ projectsDirectory, err := projects.GetProjectsDirectory(ctx, settings.ProjectsDirectory.Value)
+ if err != nil {
+ return err
+ }
+ templatesDir, err := projects.GetTemplatesDirectory(ctx, settings.TemplatesDirectory.Value)
+ if err != nil {
+ return err
+ }
+
+ if directoriesOverlapInternal(projectsDirectory, templatesDir) {
+ slog.ErrorContext(ctx,
+ "Templates and projects directories overlap; templates watcher not restarted",
+ "projectsDirectory", projectsDirectory,
+ "templatesDirectory", templatesDir)
+ return nil
+ }
+
+ tw, err := fswatch.NewWatcher(templatesDir, fswatch.WatcherOptions{
+ Debounce: 3 * time.Second,
+ OnChange: j.handleTemplatesChange,
+ MaxDepth: 1,
+ })
+ if err != nil {
+ return err
+ }
+ if err := tw.Start(ctx); err != nil {
+ return err
+ }
+
+ j.mu.Lock()
+ j.templatesWatcher = tw
+ j.mu.Unlock()
+
+ slog.InfoContext(ctx, "Templates filesystem watcher restarted", "path", templatesDir)
+
+ if err := j.templateService.SyncLocalTemplatesFromFilesystem(ctx); err != nil {
+ slog.ErrorContext(ctx, "Initial template sync after watcher restart failed", "error", err)
+ }
+
+ return nil
+}
+
+// directoriesOverlapInternal returns true when a or b is the same as or contained in the
+// other. Used to refuse running both watchers against the same tree, which would
+// cause local templates to be auto-imported as projects.
+func directoriesOverlapInternal(a, b string) bool {
+ if a == "" || b == "" {
+ return false
+ }
+ return projects.IsSafeSubdirectory(a, b) || projects.IsSafeSubdirectory(b, a)
+}
diff --git a/frontend/messages/en.json b/frontend/messages/en.json
index 4d5a2798d..71e08ff22 100644
--- a/frontend/messages/en.json
+++ b/frontend/messages/en.json
@@ -1379,6 +1379,8 @@
"general_title": "General",
"general_projects_directory_label": "Projects Directory",
"general_projects_directory_help": "Directory where Docker Compose files are stored (this is inside the container). MUST be an absolute path!",
+ "general_templates_directory_label": "Templates Directory",
+ "general_templates_directory_help": "Directory where local compose template folders are discovered. Must not overlap with the Projects Directory. Supports the same container:host bind-mount format.",
"general_follow_project_symlinks_label": "Follow Project Symlinks",
"general_follow_project_symlinks_help": "Treat symlinked child directories inside the projects directory as Docker Compose projects.",
"general_base_url_label": "Base Server URL",
diff --git a/frontend/src/lib/types/settings.type.ts b/frontend/src/lib/types/settings.type.ts
index 2c111ff89..10d1c4590 100644
--- a/frontend/src/lib/types/settings.type.ts
+++ b/frontend/src/lib/types/settings.type.ts
@@ -4,6 +4,7 @@ export type ApplicationTheme = 'default' | 'graphite' | 'ocean' | 'amber' | 'git
export type Settings = {
projectsDirectory: string;
+ templatesDirectory: string;
followProjectSymlinks: boolean;
swarmStackSourcesDirectory: string;
diskUsagePath: string;
diff --git a/frontend/src/routes/(app)/environments/[id]/+page.svelte b/frontend/src/routes/(app)/environments/[id]/+page.svelte
index a2479419c..238c650d4 100644
--- a/frontend/src/routes/(app)/environments/[id]/+page.svelte
+++ b/frontend/src/routes/(app)/environments/[id]/+page.svelte
@@ -217,6 +217,7 @@
defaultDeployPullPolicy: (settings?.defaultDeployPullPolicy as 'missing' | 'always' | 'never') || 'missing',
defaultShell: settings?.defaultShell || '/bin/sh',
projectsDirectory: settings?.projectsDirectory || '/app/data/projects',
+ templatesDirectory: settings?.templatesDirectory || '/app/data/templates',
swarmStackSourcesDirectory: settings?.swarmStackSourcesDirectory || '/app/data/swarm/sources',
diskUsagePath: settings?.diskUsagePath || '/app/data/projects',
maxImageUploadSize: settings?.maxImageUploadSize || 500,
@@ -270,6 +271,7 @@
defaultDeployPullPolicy: formData.defaultDeployPullPolicy,
defaultShell: formData.defaultShell,
projectsDirectory: formData.projectsDirectory,
+ templatesDirectory: formData.templatesDirectory,
swarmStackSourcesDirectory: formData.swarmStackSourcesDirectory,
diskUsagePath: formData.diskUsagePath,
maxImageUploadSize: formData.maxImageUploadSize,
diff --git a/frontend/src/routes/(app)/environments/[id]/components/GeneralTab.svelte b/frontend/src/routes/(app)/environments/[id]/components/GeneralTab.svelte
index e90e56fa3..edb24c419 100644
--- a/frontend/src/routes/(app)/environments/[id]/components/GeneralTab.svelte
+++ b/frontend/src/routes/(app)/environments/[id]/components/GeneralTab.svelte
@@ -30,6 +30,15 @@
helpText={m.general_projects_directory_help()}
/>
</div>
+ <div class="space-y-2">
+ <TextInputWithLabel
+ id="templates-directory"
+ label={m.general_templates_directory_label()}
+ bind:value={$formInputs.templatesDirectory.value}
+ error={$formInputs.templatesDirectory.error}
+ helpText={m.general_templates_directory_help()}
+ />
+ </div>
<div class="space-y-2">
<TextInputWithLabel
id="disk-usage-path"
diff --git a/frontend/src/routes/(app)/environments/[id]/components/environment-form-schema.ts b/frontend/src/routes/(app)/environments/[id]/components/environment-form-schema.ts
index 9464d2db0..dde8abb52 100644
--- a/frontend/src/routes/(app)/environments/[id]/components/environment-form-schema.ts
+++ b/frontend/src/routes/(app)/environments/[id]/components/environment-form-schema.ts
@@ -12,6 +12,7 @@ export const environmentFormSchema = z.object({
defaultDeployPullPolicy: z.enum(['missing', 'always', 'never']),
defaultShell: z.string(),
projectsDirectory: z.string(),
+ templatesDirectory: z.string(),
swarmStackSourcesDirectory: z.string(),
diskUsagePath: z.string(),
maxImageUploadSize: z.coerce.number(),
diff --git a/types/settings/settings.go b/types/settings/settings.go
index 64b6fd899..50d3edbd1 100644
--- a/types/settings/settings.go
+++ b/types/settings/settings.go
@@ -37,6 +37,12 @@ type Update struct {
// Required: false
ProjectsDirectory *string `json:"projectsDirectory,omitempty"`
+ // TemplatesDirectory is the directory path where local compose template folders are discovered.
+ // Must be an absolute path.
+ //
+ // Required: false
+ TemplatesDirectory *string `json:"templatesDirectory,omitempty"`
+
// FollowProjectSymlinks controls whether symlinked child directories in the projects directory are discovered as projects.
//
// Required: false