Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

Split TemplateService into mixin modules

Juan Denis3451d5ba2c2c
diff --git a/backend/app/services/template_catalog.py b/backend/app/services/template_catalog.py
new file mode 100644
index 00000000..d170baa4
--- /dev/null
+++ b/backend/app/services/template_catalog.py
@@ -0,0 +1,689 @@
+"""The template catalog: bundled, local, extension and remote-repository templates, their caches, and repository management.
+
+Mixed into TemplateService; callers keep using TemplateService."""
+import copy
+import hashlib
+import json
+import os
+import requests
+import time
+import yaml
+from typing import Any, Dict, List, Optional
+from datetime import datetime
+
+
+class TemplateCatalogMixin:
+ @classmethod
+ def extension_dirs(cls) -> List[str]:
+ """Where extension YAMLs are looked for, highest precedence first.
+
+ The synced/operator directory shadows the bundled one, exactly like
+ :meth:`list_local_templates` does for app templates.
+ """
+ return [os.path.join(cls.TEMPLATES_DIR, cls.EXTENSIONS_SUBDIR),
+ os.path.join(cls.LOCAL_TEMPLATES_DIR, cls.EXTENSIONS_SUBDIR)]
+
+ @classmethod
+ def validate_extension(cls, document: Dict) -> Dict:
+ """Validate an extension document (the whole file, not just the block)."""
+ errors = []
+ if not isinstance(document, dict):
+ return {'valid': False, 'errors': ['extension file must be a mapping']}
+ for field in ('name', 'version', 'description'):
+ if not document.get(field):
+ errors.append(f'Missing required field: {field}')
+ if 'extension' not in document:
+ errors.append("Missing required 'extension' block")
+ else:
+ errors.extend(cls._extension_block_errors(document['extension']))
+ if 'compose' in document or 'dockerfile' in document:
+ errors.append('An extension installs into a running engine and must '
+ 'not declare a compose/dockerfile of its own')
+ if errors:
+ return {'valid': False, 'errors': errors}
+ return {'valid': True, 'errors': []}
+
+ @classmethod
+ def _extension_block_errors(cls, extension: Any) -> List[str]:
+ """Hard errors in an ``extension:`` block (shape only, never opinions)."""
+ if not isinstance(extension, dict):
+ return ["'extension' must be a mapping"]
+ errors = []
+ protocol = extension.get('protocol')
+ if not protocol:
+ errors.append('extension.protocol is required (which engines can host it)')
+ elif protocol not in cls.ENGINE_PROTOCOLS:
+ errors.append(
+ f"extension.protocol '{protocol}' is not one of: "
+ f"{', '.join(cls.ENGINE_PROTOCOLS)}")
+ elif protocol == 'none':
+ errors.append("extension.protocol 'none' cannot host an extension: "
+ 'there is no client to execute the statement with')
+ statement = extension.get('statement')
+ if not statement or not isinstance(statement, str) or not statement.strip():
+ errors.append('extension.statement is required and must be a non-empty string')
+ for key in ('available_query', 'installed_query', 'image_hint', 'size', 'unit'):
+ value = extension.get(key)
+ if value is not None and not isinstance(value, str):
+ errors.append(f'extension.{key} must be a string')
+ for key in ('images', 'templates', 'versions'):
+ value = extension.get(key)
+ if value is None:
+ continue
+ if not isinstance(value, list) or not all(isinstance(v, str) for v in value):
+ errors.append(f'extension.{key} must be a list of strings')
+ return errors
+
+ @classmethod
+ def extension_metadata(cls, document: Dict, extension_id: str = None) -> Optional[Dict]:
+ """Normalized extension record, or ``None`` when the document is not one.
+
+ Everything a consumer needs is read off the YAML; nothing about any
+ particular extension is hardcoded here.
+ """
+ if not isinstance(document, dict) or not isinstance(document.get('extension'), dict):
+ return None
+ block = document['extension']
+ versions = block.get('versions')
+ if not isinstance(versions, list) or not versions:
+ versions = [document.get('version')] if document.get('version') else []
+ images = [i for i in (block.get('images') or []) if isinstance(i, str)]
+ return {
+ 'id': extension_id or document.get('id'),
+ 'name': document.get('name'),
+ 'version': document.get('version'),
+ 'description': document.get('description'),
+ 'icon': document.get('icon'),
+ 'website': document.get('website'),
+ 'documentation': document.get('documentation'),
+ 'family': cls.EXTENSION_FAMILY,
+ 'protocol': block.get('protocol'),
+ 'statement': block.get('statement'),
+ 'available_query': block.get('available_query'),
+ 'installed_query': block.get('installed_query'),
+ # [] means "any engine template speaking `protocol`".
+ 'templates': [t for t in (block.get('templates') or []) if isinstance(t, str)],
+ 'images': images,
+ 'image_hint': block.get('image_hint'),
+ # An extension with no declared images makes no image claim, so it
+ # is never blocked on one.
+ 'requires_image': bool(images),
+ 'size': block.get('size'),
+ 'unit': block.get('unit'),
+ 'versions': [str(v) for v in versions],
+ }
+
+ @classmethod
+ def list_extension_templates(cls) -> List[Dict]:
+ """Every extension declared under a ``templates/extensions/`` directory.
+
+ Derived from the files, like the engine catalog: dropping a YAML in
+ makes the extension appear, with no code change anywhere.
+ """
+ found: List[Dict] = []
+ seen = set()
+ for directory in cls.extension_dirs():
+ if not os.path.isdir(directory):
+ continue
+ for filename in sorted(os.listdir(directory)):
+ if not filename.endswith(('.yaml', '.yml')):
+ continue
+ extension_id = filename.rsplit('.', 1)[0]
+ if extension_id in seen:
+ continue
+ record = cls._load_extension_file(os.path.join(directory, filename),
+ extension_id)
+ if record:
+ seen.add(extension_id)
+ found.append(record)
+ return found
+
+ @classmethod
+ def _load_extension_file(cls, path: str, extension_id: str) -> Optional[Dict]:
+ """Parse + validate one extension file, or ``None`` (logged) if broken."""
+ try:
+ with open(path, 'r', encoding='utf-8') as fh:
+ document = yaml.safe_load(fh)
+ except Exception:
+ return None
+ validation = cls.validate_extension(document if isinstance(document, dict) else {})
+ if not validation['valid']:
+ return None
+ return cls.extension_metadata(document, extension_id=extension_id)
+
+ @classmethod
+ def get_extension_template(cls, extension_id: str) -> Optional[Dict]:
+ """One normalized extension by id, or ``None``."""
+ if not extension_id:
+ return None
+ for record in cls.list_extension_templates():
+ if record['id'] == extension_id:
+ return record
+ return None
+
+ @classmethod
+ def list_local_templates(cls) -> List[Dict]:
+ """List locally available templates.
+
+ Deliberately NOT provider-gated: this is the raw "what is bundled on
+ disk" view, which ``build_repo_index`` needs to describe the bundle
+ faithfully for publishing. Provider-owned templates are hidden from
+ the *catalog* instead — :meth:`list_all_templates` and
+ :meth:`get_template` apply the availability gate, so an
+ absent extension means hidden cards and refused installs even though
+ the YAML remains bundled.
+
+ Parsing is memoized per file (see the cache block near the top of the
+ class); a repeat listing stats each file instead of re-parsing it.
+ """
+ templates = []
+ seen_ids = set()
+ seen_paths = set()
+
+ for templates_dir in [cls.TEMPLATES_DIR, cls.LOCAL_TEMPLATES_DIR]:
+ if not os.path.isdir(templates_dir):
+ continue
+
+ try:
+ with os.scandir(templates_dir) as scan:
+ # Sorted so a same-id collision (foo.yaml vs foo.yml)
+ # resolves the same way on every platform; os.listdir order
+ # is filesystem-dependent.
+ dir_entries = sorted(scan, key=lambda e: e.name)
+ except OSError:
+ continue
+
+ for dir_entry in dir_entries:
+ if not dir_entry.name.endswith(('.yaml', '.yml')):
+ continue
+ seen_paths.add(dir_entry.path)
+
+ template_id = dir_entry.name.rsplit('.', 1)[0]
+ if template_id in seen_ids:
+ continue
+ try:
+ # DirEntry.stat() reuses the data the directory scan
+ # already returned where the platform provides it, so this
+ # is cheaper than a separate os.stat per file.
+ info = dir_entry.stat()
+ except OSError:
+ continue
+
+ entry = cls._local_entry(dir_entry.path, template_id,
+ (info.st_mtime_ns, info.st_size))
+ if entry is not None:
+ # A file that fails to parse deliberately does NOT claim
+ # the id, so a valid file of the same name in the fallback
+ # directory can still supply it.
+ seen_ids.add(template_id)
+ templates.append(entry)
+
+ cls._prune_local_cache(seen_paths)
+ return templates
+
+ @classmethod
+ def invalidate_local_cache(cls) -> None:
+ """Drop the parsed-template cache.
+
+ Ordinary writes do not need this -- the cache is keyed on
+ (mtime_ns, size), so syncing or editing a template invalidates exactly
+ that file on the next listing. Provided for tests and any out-of-band
+ mutation that could defeat mtime comparison."""
+ with cls._local_cache_lock:
+ cls._local_cache = {}
+
+ @classmethod
+ def _prune_local_cache(cls, seen_paths: set) -> None:
+ """Forget files that are no longer on disk, so a long-running panel
+ that churns templates does not grow the cache without bound."""
+ with cls._local_cache_lock:
+ stale = [p for p in cls._local_cache if p not in seen_paths]
+ for path in stale:
+ del cls._local_cache[path]
+
+ @classmethod
+ def _local_entry(cls, filepath: str, template_id: str,
+ stat_key: tuple) -> Optional[Dict]:
+ """Catalog projection for one bundled template, memoized on ``stat_key``.
+
+ Returns None when the file does not parse -- and caches that negative
+ result too, so one malformed YAML does not cost a parse on every
+ render. Returns a deep copy so a caller mutating the result (or its
+ nested ``repo``/``engine``/``categories``) cannot corrupt the cache."""
+ with cls._local_cache_lock:
+ cached = cls._local_cache.get(filepath)
+ if cached is not None and cached['key'] == stat_key:
+ return copy.deepcopy(cached['entry'])
+
+ entry = cls._parse_local_entry(filepath, template_id)
+
+ with cls._local_cache_lock:
+ cls._local_cache[filepath] = {'key': stat_key, 'entry': entry}
+ return copy.deepcopy(entry)
+
+ @classmethod
+ def _parse_local_entry(cls, filepath: str, template_id: str) -> Optional[Dict]:
+ """Parse one template file into its catalog projection, or None."""
+ result = cls.parse_template(filepath)
+ if not result.get('success'):
+ return None
+
+ template = result['template']
+ kind = template.get('kind', 'compose')
+ entry = {
+ 'id': template_id,
+ 'name': template.get('name'),
+ 'version': template.get('version'),
+ 'description': template.get('description'),
+ 'icon': template.get('icon'),
+ 'categories': template.get('categories', []),
+ 'kind': kind,
+ 'source': 'local',
+ 'filepath': filepath
+ }
+ # Repo templates surface a minimal repo summary so the catalog grid
+ # can badge them and link to the wizard without a second fetch.
+ if kind == 'repo' and isinstance(template.get('repo'), dict):
+ repo = template['repo']
+ entry['repo'] = {
+ 'url': repo.get('url'),
+ 'branch': repo.get('branch', 'main'),
+ }
+ # A database engine declares itself with a top-level `engine:` block.
+ # This listing is a fixed projection, so without carrying the block
+ # through, the Databases catalog could never see an engine without
+ # re-parsing every YAML file.
+ engine = cls.engine_metadata(template)
+ if engine:
+ entry['engine'] = engine
+ return entry
+
+ @classmethod
+ def list_engine_templates(cls) -> List[Dict]:
+ """Every available template carrying an ``engine:`` block.
+
+ This is what the Databases engine catalog renders. It is derived
+ entirely from the template sources -- drop a new engine YAML into
+ ``backend/templates/`` (or sync a repository that ships one) and it
+ appears here with no code change.
+ """
+ return [t for t in cls.list_all_templates()
+ if isinstance(t.get('engine'), dict)]
+
+ @classmethod
+ def build_repo_index(cls, repo_name: str = 'serverkit-official') -> Dict:
+ """Build the ``index.json`` document that describes the locally-bundled
+ templates as a publishable repository.
+
+ This is the exact shape :meth:`fetch_remote_templates` / :meth:`sync_templates`
+ consume from ``<repo_url>/index.json`` (templates served at
+ ``<repo_url>/templates/<id>.yaml``). Publishing a repo is then just:
+ host the ``templates/*.yaml`` files plus this ``index.json``. Lets
+ Prompture Hub & friends ship template updates without a panel release.
+ """
+ templates = []
+ for t in cls.list_local_templates():
+ entry = {
+ 'id': t['id'],
+ 'name': t.get('name'),
+ 'version': t.get('version'),
+ 'description': t.get('description'),
+ 'icon': t.get('icon'),
+ 'categories': t.get('categories', []),
+ }
+ # Carried in the index so a panel that syncs this repo can list the
+ # engine catalog without fetching all 100+ template files first.
+ if isinstance(t.get('engine'), dict):
+ entry['engine'] = t['engine']
+ templates.append(entry)
+ return {
+ 'name': repo_name,
+ 'schema_version': cls.SCHEMA_VERSION,
+ 'generated_at': datetime.now().isoformat(),
+ 'count': len(templates),
+ 'templates': templates,
+ }
+
+ @classmethod
+ def export_repo_index(cls, dest_path: str = None) -> Dict:
+ """Write :meth:`build_repo_index` to ``dest_path`` (defaults to
+ ``index.json`` alongside the bundled templates). Returns a status dict."""
+ index = cls.build_repo_index()
+ path = dest_path or os.path.join(cls.LOCAL_TEMPLATES_DIR, 'index.json')
+ try:
+ with open(path, 'w', encoding='utf-8') as f:
+ json.dump(index, f, indent=2)
+ return {'success': True, 'path': path, 'count': index['count']}
+ except Exception as e:
+ return {'success': False, 'error': str(e)}
+
+ @classmethod
+ def invalidate_remote_cache(cls, repo_url: str = None) -> None:
+ """Drop cached remote indexes -- all of them, or just ``repo_url``.
+
+ Called whenever the repo list changes so a freshly added repo appears
+ immediately instead of after the TTL, and a removed one stops being
+ served from cache."""
+ with cls._remote_cache_lock:
+ if repo_url is None:
+ cls._remote_cache = {}
+ else:
+ cls._remote_cache.pop(repo_url, None)
+
+ @classmethod
+ def _cached_remote(cls, repo_url: str) -> Optional[List[Dict]]:
+ """Unexpired cache entry for ``repo_url``, or None. Returns copies so a
+ caller mutating an entry cannot corrupt the cache."""
+ with cls._remote_cache_lock:
+ entry = cls._remote_cache.get(repo_url)
+ if entry and entry['expires'] > time.monotonic():
+ return [dict(t) for t in entry['templates']]
+ return None
+
+ @classmethod
+ def _last_good_remote(cls, repo_url: str) -> Optional[List[Dict]]:
+ """Last payload this repo successfully served, regardless of expiry."""
+ with cls._remote_cache_lock:
+ entry = cls._remote_cache.get(repo_url) or {}
+ last_good = entry.get('last_good')
+ return [dict(t) for t in last_good] if last_good else None
+
+ @classmethod
+ def _store_remote(cls, repo_url: str, templates: List[Dict], ok: bool) -> None:
+ """Cache ``templates`` for ``repo_url``. A successful payload also
+ becomes the last-good; a failure carries the previous last-good
+ forward, so repeated failures keep serving it instead of decaying to
+ an empty catalog on the second one."""
+ ttl = cls._REMOTE_TTL_SECONDS if ok else cls._REMOTE_ERROR_TTL_SECONDS
+ with cls._remote_cache_lock:
+ previous = cls._remote_cache.get(repo_url) or {}
+ cls._remote_cache[repo_url] = {
+ 'templates': [dict(t) for t in templates],
+ 'expires': time.monotonic() + ttl,
+ 'ok': ok,
+ 'last_good': ([dict(t) for t in templates] if ok
+ else previous.get('last_good')),
+ }
+
+ @classmethod
+ def fetch_remote_templates(cls, repo_url: str, force: bool = False) -> List[Dict]:
+ """Fetch a repository's template index, memoized (see the cache block
+ near the top of the class). ``force=True`` bypasses the cache, which is
+ what an explicit sync wants."""
+ if not force:
+ cached = cls._cached_remote(repo_url)
+ if cached is not None:
+ return cached
+
+ templates = []
+ try:
+ index_url = f"{repo_url}/index.json"
+ response = requests.get(index_url, timeout=cls._REMOTE_TIMEOUT_SECONDS)
+ response.raise_for_status()
+
+ index = response.json()
+ for template_info in index.get('templates', []):
+ template_info['source'] = 'remote'
+ template_info['repo_url'] = repo_url
+ templates.append(template_info)
+
+ except Exception as e:
+ print(f"Failed to fetch templates from {repo_url}: {e}")
+ # Keep serving the last good index rather than blanking a catalog
+ # the operator has already seen, but still back off so a dead repo
+ # is not retried on every single page load.
+ last_good = cls._last_good_remote(repo_url)
+ cls._store_remote(repo_url, last_good or [], ok=False)
+ return last_good or []
+
+ cls._store_remote(repo_url, templates, ok=True)
+ return [dict(t) for t in templates]
+
+ @classmethod
+ def list_all_templates(cls, category: str = None, search: str = None) -> List[Dict]:
+ """List all available templates from all sources, deduped by id.
+
+ Bundled and registry templates deliberately overlap -- the bundle is the
+ offline floor and the registry is the growth path (docs/REGISTRIES.md) --
+ so the same id legitimately arrives from both. Concatenating them listed
+ every bundled template twice. **Local wins**, matching
+ :meth:`get_template`, which already resolves local directories before
+ any repo; among repos, the first enabled one to claim an id keeps it.
+ """
+ by_id = {}
+
+ def _claim(entry):
+ """First source to claim an id keeps it."""
+ template_id = entry.get('id')
+ if template_id and template_id not in by_id:
+ by_id[template_id] = entry
+
+ # Local templates first -- they outrank every repo.
+ for entry in cls.list_local_templates():
+ _claim(entry)
+
+ # Remote templates fill in ids the bundle does not carry.
+ config = cls.get_config()
+ for repo in config.get('repos', []):
+ if repo.get('enabled', True):
+ for entry in cls.fetch_remote_templates(repo['url']):
+ _claim(entry)
+
+ templates = list(by_id.values())
+
+ # Filter by category
+ if category:
+ templates = [t for t in templates if category in t.get('categories', [])]
+
+ # Provider-owned templates from any source (incl. synced remote repos)
+ # are hidden while their extension is absent.
+ templates = [t for t in templates if cls.template_available(t.get('id'))]
+
+ # Search filter
+ if search:
+ search_lower = search.lower()
+ templates = [
+ t for t in templates
+ if search_lower in t.get('name', '').lower()
+ or search_lower in t.get('description', '').lower()
+ ]
+
+ return templates
+
+ @classmethod
+ def get_template(cls, template_id: str) -> Dict:
+ """Get full template details."""
+ if not cls.template_available(template_id):
+ owner = cls.PROVIDER_OWNED_TEMPLATES.get(template_id)
+ return {
+ 'success': False,
+ 'error': f"Template '{template_id}' is provided by the "
+ f"'{owner}' extension, which is not installed or not active",
+ }
+ # Check local directories (system dir, then bundled fallback)
+ for templates_dir in [cls.TEMPLATES_DIR, cls.LOCAL_TEMPLATES_DIR]:
+ for ext in ['.yaml', '.yml']:
+ filepath = os.path.join(templates_dir, f"{template_id}{ext}")
+ if os.path.exists(filepath):
+ result = cls.parse_template(filepath)
+ if result.get('success'):
+ template = result['template']
+ template['source'] = 'local'
+ template['filepath'] = filepath
+ return {'success': True, 'template': template}
+ return result
+
+ # Check remote repos
+ config = cls.get_config()
+ for repo in config.get('repos', []):
+ if not repo.get('enabled', True):
+ continue
+
+ try:
+ url = f"{repo['url']}/templates/{template_id}.yaml"
+ response = requests.get(url, timeout=30)
+ if response.status_code == 200:
+ template = yaml.safe_load(response.text)
+ validation = cls.validate_template(template)
+ if validation['valid']:
+ template['source'] = 'remote'
+ template['repo_url'] = repo['url']
+ return {'success': True, 'template': template}
+ except Exception:
+ continue
+
+ return {'success': False, 'error': 'Template not found'}
+
+ @classmethod
+ def add_repository(cls, name: str, url: str) -> Dict:
+ """Add a template repository."""
+ config = cls.get_config()
+
+ # Check if already exists
+ for repo in config.get('repos', []):
+ if repo['url'] == url:
+ return {'success': False, 'error': 'Repository already exists'}
+
+ config.setdefault('repos', []).append({
+ 'name': name,
+ 'url': url.rstrip('/'),
+ 'enabled': True,
+ 'added_at': datetime.now().isoformat()
+ })
+
+ return cls.save_config(config)
+
+ @classmethod
+ def remove_repository(cls, url: str) -> Dict:
+ """Remove a template repository."""
+ config = cls.get_config()
+ config['repos'] = [r for r in config.get('repos', []) if r['url'] != url]
+ return cls.save_config(config)
+
+ @classmethod
+ def list_repositories(cls) -> List[Dict]:
+ """List configured template repositories."""
+ config = cls.get_config()
+ return config.get('repos', cls.DEFAULT_REPOS)
+
+ @classmethod
+ def sync_templates(cls) -> Dict:
+ """Sync templates from all repositories."""
+ os.makedirs(cls.TEMPLATES_DIR, exist_ok=True)
+
+ config = cls.get_config()
+ synced = 0
+ unverified = 0 # saved, but the index pinned no sha256 for them
+ errors = []
+
+ for repo in config.get('repos', []):
+ if not repo.get('enabled', True):
+ continue
+
+ try:
+ # Fetch index
+ index_url = f"{repo['url']}/index.json"
+ response = requests.get(index_url, timeout=30)
+ response.raise_for_status()
+
+ index = response.json()
+
+ # Download each template
+ for template_info in index.get('templates', []):
+ template_id = template_info.get('id')
+ if not template_id:
+ continue
+
+ try:
+ template_url = f"{repo['url']}/templates/{template_id}.yaml"
+ response = requests.get(template_url, timeout=30)
+ response.raise_for_status()
+
+ # Verify against the checksum the index pinned for this
+ # entry. A template is a deploy definition -- images,
+ # ports, volumes, env -- so a swapped file is worth
+ # refusing outright, and the index already carries the
+ # hash for every official entry.
+ #
+ # Missing hash is allowed (third-party repos may not
+ # publish one) and counted, mirroring how extensions
+ # treat unsigned-vs-invalid: absent is a caveat, wrong
+ # is a hard stop.
+ expected = (template_info.get('sha256') or '').strip().lower()
+ if expected:
+ actual = hashlib.sha256(response.content).hexdigest()
+ if actual != expected:
+ errors.append(
+ f"Checksum mismatch for {template_id}: index pinned "
+ f"{expected[:12]}..., downloaded {actual[:12]}.... Not saved."
+ )
+ continue
+ else:
+ unverified += 1
+
+ # Written as bytes so what lands on disk is exactly the
+ # content that was hashed (text mode would rewrite line
+ # endings on Windows and no longer match).
+ filepath = os.path.join(cls.TEMPLATES_DIR, f"{template_id}.yaml")
+ with open(filepath, 'wb') as f:
+ f.write(response.content)
+
+ synced += 1
+ except Exception as e:
+ errors.append(f"Failed to sync {template_id}: {e}")
+
+ except Exception as e:
+ errors.append(f"Failed to sync from {repo['name']}: {e}")
+
+ config['last_sync'] = datetime.now().isoformat()
+ cls.save_config(config)
+
+ return {
+ 'success': True,
+ 'synced': synced,
+ 'unverified': unverified,
+ 'errors': errors if errors else None
+ }
+
+ @classmethod
+ def get_categories(cls) -> List[str]:
+ """Get all available template categories."""
+ templates = cls.list_all_templates()
+ categories = set()
+ for template in templates:
+ categories.update(template.get('categories', []))
+ return sorted(categories)
+
+ @classmethod
+ def create_local_template(cls, template_data: Dict) -> Dict:
+ """Create a local template."""
+ validation = cls.validate_template(template_data)
+ if not validation['valid']:
+ return {'success': False, 'errors': validation['errors']}
+
+ os.makedirs(cls.TEMPLATES_DIR, exist_ok=True)
+
+ template_id = template_data['name'].lower().replace(' ', '-')
+ filepath = os.path.join(cls.TEMPLATES_DIR, f"{template_id}.yaml")
+
+ if os.path.exists(filepath):
+ return {'success': False, 'error': 'Template with this name already exists'}
+
+ try:
+ with open(filepath, 'w') as f:
+ yaml.dump(template_data, f, default_flow_style=False, sort_keys=False)
+
+ return {'success': True, 'template_id': template_id, 'filepath': filepath}
+ except Exception as e:
+ return {'success': False, 'error': str(e)}
+
+ @classmethod
+ def delete_local_template(cls, template_id: str) -> Dict:
+ """Delete a local template."""
+ for ext in ['.yaml', '.yml']:
+ filepath = os.path.join(cls.TEMPLATES_DIR, f"{template_id}{ext}")
+ if os.path.exists(filepath):
+ os.remove(filepath)
+ return {'success': True}
+
+ return {'success': False, 'error': 'Template not found'}
diff --git a/backend/app/services/template_magic.py b/backend/app/services/template_magic.py
new file mode 100644
index 00000000..aa6c5032
--- /dev/null
+++ b/backend/app/services/template_magic.py
@@ -0,0 +1,249 @@
+"""Template variable values: generated secrets and ports, magic tokens (SERVICE_PASSWORD_*, SERVICE_URL_*, ...) and ${VAR} substitution.
+
+Mixed into TemplateService; callers keep using TemplateService."""
+import re
+import secrets
+import string
+from typing import Any, Dict
+
+
+class TemplateMagicMixin:
+ @classmethod
+ def generate_value(cls, var_config: Dict, force_generate: bool = False) -> str:
+ """Generate a value for a variable based on its configuration.
+
+ Args:
+ var_config: Variable configuration dict
+ force_generate: If True, always generate new value even for ports
+ """
+ var_type = var_config.get('type', 'string')
+ default = var_config.get('default', '')
+
+ if var_type == 'password':
+ length = var_config.get('length', 32)
+ chars = string.ascii_letters + string.digits
+ if var_config.get('special_chars', False):
+ chars += '!@#$%^&*'
+ return ''.join(secrets.choice(chars) for _ in range(length))
+
+ elif var_type == 'port':
+ # ALWAYS find an available port - never trust defaults
+ start_port = int(default) if default else 8000
+ # A global base-port setting (Settings > managed_app_base_port)
+ # overrides the per-template default when configured (non-zero).
+ base = cls._managed_app_base_port()
+ if base:
+ start_port = base
+ return str(cls._claim_port(start_port))
+
+ elif var_type == 'uuid':
+ import uuid
+ return str(uuid.uuid4())
+
+ elif var_type == 'random':
+ length = var_config.get('length', 16)
+ return secrets.token_hex(length // 2)
+
+ return str(default)
+
+ @classmethod
+ def _classify_magic_token(cls, token: str):
+ """Return ``(kind, name)`` for a bare magic token (no ``${}``), or
+ ``(None, None)`` if it is not a recognized magic variable."""
+ if token in cls.MAGIC_EXACT_TOKENS:
+ return cls.MAGIC_EXACT_TOKENS[token], ''
+ for prefix, kind in cls.MAGIC_PREFIXES:
+ if token.startswith(prefix):
+ return kind, token[len(prefix):]
+ return None, None
+
+ @classmethod
+ def _generate_magic_password(cls) -> str:
+ """Strong password for a SERVICE_PASSWORD_* token (alnum, no special
+ chars to stay shell/compose-safe). Reuses the same primitive as
+ ``generate_value(type=password)``."""
+ chars = string.ascii_letters + string.digits
+ return ''.join(secrets.choice(chars) for _ in range(cls.MAGIC_PASSWORD_LENGTH))
+
+ @classmethod
+ def _generate_magic_user(cls, name: str) -> str:
+ """Service username for a SERVICE_USER_* token: ``svc_<name>_<rand>``,
+ lowercased and reduced to ``[a-z0-9_]`` so it is safe as a DB/app user."""
+ base = re.sub(r'[^a-z0-9]+', '_', (name or 'service').lower()).strip('_') or 'service'
+ suffix = secrets.token_hex(2) # 4 hex chars, keeps it short but unique
+ return f'svc_{base}_{suffix}'
+
+ @classmethod
+ def _resolve_magic_value(cls, kind: str, name: str, context: Dict) -> str:
+ """Resolve a single magic token to a value.
+
+ ``context`` may carry ``fqdn`` / ``scheme`` (and ``app_name``); when an
+ FQDN is not known yet the FQDN/URL forms degrade to a documented
+ placeholder (``localhost``) that the install finalizer can later fill —
+ this keeps resolution best-effort and non-fatal.
+ """
+ context = context or {}
+ if kind == 'password':
+ return cls._generate_magic_password()
+ if kind == 'user':
+ return cls._generate_magic_user(name)
+ if kind == 'base64':
+ import base64
+ secret = secrets.token_bytes(24)
+ return base64.b64encode(secret).decode('ascii')
+ if kind == 'fqdn':
+ return str(context.get('fqdn') or context.get('app_name') or 'localhost')
+ if kind == 'url':
+ scheme = str(context.get('scheme') or 'http')
+ host = context.get('fqdn') or context.get('app_name') or 'localhost'
+ return f'{scheme}://{host}'
+ if kind == 'metrics_token':
+ from app.services import metrics_token_service
+ return metrics_token_service.get_or_create()
+ if kind == 'server_public_ip':
+ ip = context.get('server_public_ip')
+ if not ip:
+ try:
+ from app.services.site_domain_service import SiteDomainService
+ ip = SiteDomainService.server_ip()
+ except Exception:
+ ip = None
+ return str(ip or '')
+ return ''
+
+ @classmethod
+ def resolve_magic_variables(cls, content: Any, context: Dict = None):
+ """Scan ``content`` for ``${SERVICE_*}`` magic tokens, generate a value
+ once per unique token, substitute, and return ``(substituted, generated)``.
+
+ Args:
+ content: A string, or a (possibly nested) dict/list — e.g. a parsed
+ compose section. Returned with the same shape.
+ context: Optional dict with ``app_name`` / ``fqdn`` / ``scheme`` used
+ to resolve FQDN/URL tokens. No Docker/network access is performed.
+
+ Returns:
+ ``(substituted_content, generated_vars)`` where ``generated_vars`` maps
+ each magic variable name (without ``${}``) to its generated value, so
+ callers can persist them as env vars / surface them post-install.
+ Pure and idempotent for a given ``content`` within one call (the same
+ token always maps to the same generated value here).
+ """
+ context = context or {}
+ generated: Dict[str, str] = {}
+
+ def _collect(text: str):
+ for match in re.finditer(cls.MAGIC_TOKEN_PATTERN, text):
+ token = match.group(1)
+ if token in generated:
+ continue
+ kind, name = cls._classify_magic_token(token)
+ if kind is None:
+ continue
+ generated[token] = cls._resolve_magic_value(kind, name, context)
+
+ def _walk_collect(node: Any):
+ if isinstance(node, str):
+ _collect(node)
+ elif isinstance(node, dict):
+ for value in node.values():
+ _walk_collect(value)
+ elif isinstance(node, list):
+ for item in node:
+ _walk_collect(item)
+
+ # Pass 1: discover every unique token and generate a stable value.
+ _walk_collect(content)
+
+ # Pass 2: substitute using the generated values. Reuse the existing
+ # ${VAR} substitution so behavior is identical to normal variables.
+ if isinstance(content, str):
+ substituted = cls.substitute_variables(content, generated)
+ else:
+ substituted = cls.substitute_in_dict(content, generated)
+
+ return substituted, generated
+
+ @classmethod
+ def collect_magic_variables(cls, template: Dict, context: Dict = None) -> Dict[str, str]:
+ """Generate the magic variables a template uses, given its declared
+ ``compose`` / ``files`` / ``scripts`` sections, WITHOUT mutating the
+ template. Returns ``{name: value}`` for every ``${SERVICE_*}`` token found.
+
+ This is the wiring entry point for the install flow: the returned dict is
+ merged into the install ``variables`` so the existing ``${VAR}``
+ substitution renders the tokens, and so the secrets land in ``.env`` /
+ post-install output. Templates with no magic tokens get ``{}`` and behave
+ exactly as before.
+ """
+ # Scan the parts that the installer substitutes against.
+ scan_target = {
+ 'compose': template.get('compose', {}),
+ 'files': template.get('files', []),
+ 'scripts': template.get('scripts', {}),
+ }
+ _, generated = cls.resolve_magic_variables(scan_target, context)
+ return generated
+
+ @classmethod
+ def _install_magic_context(cls, template: Dict, app_name: str,
+ variables: Dict = None) -> Dict:
+ """Build the best-effort ``context`` for magic-variable resolution at
+ install time.
+
+ Resolves the would-be FQDN from the managed-sites base domain when one is
+ configured (the same ``<slug>.<base>`` the finalizer publishes), and the
+ scheme from whether wildcard HTTPS is on. All of this is best-effort and
+ non-fatal: if site routing isn't set up (or we're outside an app context),
+ FQDN/URL tokens fall back to a documented ``localhost`` placeholder that
+ the install finalizer can later fill in.
+ """
+ context = {'app_name': app_name}
+ try:
+ from app.services.site_domain_service import SiteDomainService
+ base = SiteDomainService.base_domain()
+ # Only assign an FQDN when the template opts into auto-domain and a
+ # base domain exists — mirrors the finalizer's publish condition.
+ if base and template.get('auto_domain'):
+ host = SiteDomainService.subdomain_for(app_name)
+ if host:
+ context['fqdn'] = host
+ context['scheme'] = 'https' if (
+ SiteDomainService.https_enabled()
+ and SiteDomainService.covers(host)
+ ) else 'http'
+ except Exception:
+ # No app context / site routing not available — leave FQDN unset so
+ # tokens degrade to the localhost placeholder.
+ pass
+ return context
+
+ @classmethod
+ def _collect_magic_for_install(cls, template: Dict, app_name: str,
+ variables: Dict = None) -> Dict[str, str]:
+ """Resolve a template's magic variables for an install, using the
+ best-effort FQDN context. Thin wrapper over :meth:`collect_magic_variables`."""
+ context = cls._install_magic_context(template, app_name, variables)
+ return cls.collect_magic_variables(template, context)
+
+ @classmethod
+ def substitute_variables(cls, content: str, variables: Dict) -> str:
+ """Substitute variables in content using ${VAR} syntax."""
+ def replace_var(match):
+ var_name = match.group(1)
+ return str(variables.get(var_name, match.group(0)))
+
+ # Replace ${VAR} patterns
+ pattern = r'\$\{([A-Z_][A-Z0-9_]*)\}'
+ return re.sub(pattern, replace_var, content)
+
+ @classmethod
+ def substitute_in_dict(cls, data: Any, variables: Dict) -> Any:
+ """Recursively substitute variables in a dictionary."""
+ if isinstance(data, str):
+ return cls.substitute_variables(data, variables)
+ elif isinstance(data, dict):
+ return {k: cls.substitute_in_dict(v, variables) for k, v in data.items()}
+ elif isinstance(data, list):
+ return [cls.substitute_in_dict(item, variables) for item in data]
+ return data
diff --git a/backend/app/services/template_ports.py b/backend/app/services/template_ports.py
new file mode 100644
index 00000000..a98dd034
--- /dev/null
+++ b/backend/app/services/template_ports.py
@@ -0,0 +1,166 @@
+"""Host-port allocation for template installs: scans app rows, containers and live sockets, and holds a claim until the install records its port.
+
+Mixed into TemplateService; callers keep using TemplateService."""
+import re
+import threading
+import time
+from app.exceptions import ConflictError
+from app.utils.system import run_checked
+
+# Ports handed to an install that has not written its app row (or started its
+# container) yet. Without this, two installs resolving a port variable at the
+# same time both saw the same port free. Single-process panel, so in-memory.
+_PORT_CLAIM_TTL = 15 * 60
+_claimed_ports = {}
+_claimed_ports_lock = threading.Lock()
+
+
+class TemplatePortsMixin:
+ @classmethod
+ def _managed_app_base_port(cls) -> int:
+ """Return the admin-configured base port for managed apps, or 0 if unset.
+
+ Reads the ``managed_app_base_port`` system setting. Returns 0 (meaning
+ "use each template's own default") on any error, e.g. if the settings
+ table isn't ready yet during early startup.
+ """
+ try:
+ from app.services.settings_service import SettingsService
+ return int(SettingsService.get('managed_app_base_port', 0) or 0)
+ except Exception:
+ return 0
+
+ @classmethod
+ def _find_available_port(cls, start_port: int = 8000, max_attempts: int = 1000) -> int:
+ """Find an available port that's not in use by the system, Docker, or database.
+
+ Checks:
+ 1. Ports assigned to existing applications in the database
+ 2. Docker container port mappings
+ 3. Socket binding test
+ """
+ import socket
+
+ # Get ports from database (assigned to apps)
+ db_ports = cls._get_database_used_ports()
+
+ # Get ports currently used by Docker containers
+ docker_ports = cls._get_docker_used_ports()
+
+ # Combine all used ports
+ used_ports = db_ports | docker_ports | cls._live_claims()
+
+ for port in range(start_port, start_port + max_attempts):
+ # Skip reserved/common ports
+ if port < 1024:
+ continue
+
+ # Skip if already assigned in DB or Docker
+ if port in used_ports:
+ continue
+
+ # Check if port is available on localhost (where Docker binds)
+ try:
+ # Try to bind - most reliable check
+ test_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ test_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ test_sock.bind(('127.0.0.1', port))
+ test_sock.close()
+ return port
+ except OSError:
+ continue
+ except Exception:
+ continue
+
+ # A random port here could collide with anything; say so instead.
+ raise ConflictError(
+ f'No free port between {start_port} and {start_port + max_attempts - 1}')
+
+ @staticmethod
+ def _live_claims() -> set:
+ now = time.monotonic()
+ with _claimed_ports_lock:
+ for port, at in list(_claimed_ports.items()):
+ if now - at > _PORT_CLAIM_TTL:
+ del _claimed_ports[port]
+ return set(_claimed_ports)
+
+ @classmethod
+ def _claim_port(cls, start_port: int = 8000) -> int:
+ """Find a free port and hold it for the install about to use it."""
+ while True:
+ port = cls._find_available_port(start_port)
+ with _claimed_ports_lock:
+ if port not in _claimed_ports:
+ _claimed_ports[port] = time.monotonic()
+ return port
+ start_port = port + 1
+
+ @classmethod
+ def _port_is_free(cls, port) -> bool:
+ """True when nothing -- app row, container, or live socket -- holds
+ ``port``. Same three sources as :meth:`_find_available_port`, asked
+ about one specific port instead of scanning."""
+ import socket
+ try:
+ port = int(port)
+ except (TypeError, ValueError):
+ return False
+ if not 1 <= port <= 65535:
+ return False
+ if port in (cls._get_database_used_ports() | cls._get_docker_used_ports()
+ | cls._live_claims()):
+ return False
+ try:
+ test_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+ test_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+ test_sock.bind(('127.0.0.1', port))
+ test_sock.close()
+ return True
+ except OSError:
+ return False
+ except Exception:
+ return False
+
+ @classmethod
+ def _get_database_used_ports(cls) -> set:
+ """Get all ports assigned to applications in the database."""
+ used_ports = set()
+ try:
+ from app.models import Application
+ # Deliberately unfiltered: a soft-deleted app must KEEP reserving its
+ # port, or a new app takes it and restoring from the recycle bin collides.
+ apps = Application.query.filter(Application.port.isnot(None)).all()
+ for app in apps:
+ if app.port:
+ used_ports.add(app.port)
+ # A slot app holds two ports: the live one is app.port,
+ # the standby's stays reserved for the switch back.
+ from app.models.app_slot import AppSlot
+ used_ports.update(row.host_port for row in
+ AppSlot.query.filter(AppSlot.host_port.isnot(None)).all())
+ except Exception:
+ pass
+ return used_ports
+
+ @classmethod
+ def _get_docker_used_ports(cls) -> set:
+ """Get all ports currently mapped by Docker containers."""
+ used_ports = set()
+ try:
+ result = run_checked(['docker', 'ps', '--format', '{{.Ports}}'], timeout=10)
+ if result['success']:
+ # Parse port mappings like "0.0.0.0:8080->80/tcp, 127.0.0.1:3306->3306/tcp"
+ import re
+ for line in result['output'].strip().split('\n'):
+ if line:
+ # Find all host ports in the format "host:port->container"
+ matches = re.findall(r'(?:[\d.]+:)?(\d+)->', line)
+ for port_str in matches:
+ try:
+ used_ports.add(int(port_str))
+ except ValueError:
+ pass
+ except Exception:
+ pass
+ return used_ports
diff --git a/backend/app/services/template_service.py b/backend/app/services/template_service.py
index 70d648f1..c110db63 100644
--- a/backend/app/services/template_service.py
+++ b/backend/app/services/template_service.py
@@ -12,35 +12,24 @@ Supports:
import os
import re
-import copy
import time
import yaml
import json
import shutil
-import secrets
-import string
-import hashlib
import threading
import subprocess
from datetime import datetime
from typing import Callable, Dict, List, Optional, Any
-from pathlib import Path
-import requests
from app import paths
from app.utils.system import app_process_env, run_checked
from app.services.deploy_lock import with_deploy_lock
-from app.exceptions import ConflictError
+from app.services.template_catalog import TemplateCatalogMixin
+from app.services.template_magic import TemplateMagicMixin
+from app.services.template_ports import TemplatePortsMixin
-# Ports handed to an install that has not written its app row (or started its
-# container) yet. Without this, two installs resolving a port variable at the
-# same time both saw the same port free. Single-process panel, so in-memory.
-_PORT_CLAIM_TTL = 15 * 60
-_claimed_ports = {}
-_claimed_ports_lock = threading.Lock()
-
-class TemplateService:
+class TemplateService(TemplateCatalogMixin, TemplateMagicMixin, TemplatePortsMixin):
"""Service for managing and deploying application templates."""
CONFIG_DIR = paths.SERVERKIT_CONFIG_DIR
@@ -441,153 +430,6 @@ class TemplateService:
# nothing can declare it wrong.
EXTENSION_FAMILY = 'Extension'
- @classmethod
- def extension_dirs(cls) -> List[str]:
- """Where extension YAMLs are looked for, highest precedence first.
-
- The synced/operator directory shadows the bundled one, exactly like
- :meth:`list_local_templates` does for app templates.
- """
- return [os.path.join(cls.TEMPLATES_DIR, cls.EXTENSIONS_SUBDIR),
- os.path.join(cls.LOCAL_TEMPLATES_DIR, cls.EXTENSIONS_SUBDIR)]
-
- @classmethod
- def validate_extension(cls, document: Dict) -> Dict:
- """Validate an extension document (the whole file, not just the block)."""
- errors = []
- if not isinstance(document, dict):
- return {'valid': False, 'errors': ['extension file must be a mapping']}
- for field in ('name', 'version', 'description'):
- if not document.get(field):
- errors.append(f'Missing required field: {field}')
- if 'extension' not in document:
- errors.append("Missing required 'extension' block")
- else:
- errors.extend(cls._extension_block_errors(document['extension']))
- if 'compose' in document or 'dockerfile' in document:
- errors.append('An extension installs into a running engine and must '
- 'not declare a compose/dockerfile of its own')
- if errors:
- return {'valid': False, 'errors': errors}
- return {'valid': True, 'errors': []}
-
- @classmethod
- def _extension_block_errors(cls, extension: Any) -> List[str]:
- """Hard errors in an ``extension:`` block (shape only, never opinions)."""
- if not isinstance(extension, dict):
- return ["'extension' must be a mapping"]
- errors = []
- protocol = extension.get('protocol')
- if not protocol:
- errors.append('extension.protocol is required (which engines can host it)')
- elif protocol not in cls.ENGINE_PROTOCOLS:
- errors.append(
- f"extension.protocol '{protocol}' is not one of: "
- f"{', '.join(cls.ENGINE_PROTOCOLS)}")
- elif protocol == 'none':
- errors.append("extension.protocol 'none' cannot host an extension: "
- 'there is no client to execute the statement with')
- statement = extension.get('statement')
- if not statement or not isinstance(statement, str) or not statement.strip():
- errors.append('extension.statement is required and must be a non-empty string')
- for key in ('available_query', 'installed_query', 'image_hint', 'size', 'unit'):
- value = extension.get(key)
- if value is not None and not isinstance(value, str):
- errors.append(f'extension.{key} must be a string')
- for key in ('images', 'templates', 'versions'):
- value = extension.get(key)
- if value is None:
- continue
- if not isinstance(value, list) or not all(isinstance(v, str) for v in value):
- errors.append(f'extension.{key} must be a list of strings')
- return errors
-
- @classmethod
- def extension_metadata(cls, document: Dict, extension_id: str = None) -> Optional[Dict]:
- """Normalized extension record, or ``None`` when the document is not one.
-
- Everything a consumer needs is read off the YAML; nothing about any
- particular extension is hardcoded here.
- """
- if not isinstance(document, dict) or not isinstance(document.get('extension'), dict):
- return None
- block = document['extension']
- versions = block.get('versions')
- if not isinstance(versions, list) or not versions:
- versions = [document.get('version')] if document.get('version') else []
- images = [i for i in (block.get('images') or []) if isinstance(i, str)]
- return {
- 'id': extension_id or document.get('id'),
- 'name': document.get('name'),
- 'version': document.get('version'),
- 'description': document.get('description'),
- 'icon': document.get('icon'),
- 'website': document.get('website'),
- 'documentation': document.get('documentation'),
- 'family': cls.EXTENSION_FAMILY,
- 'protocol': block.get('protocol'),
- 'statement': block.get('statement'),
- 'available_query': block.get('available_query'),
- 'installed_query': block.get('installed_query'),
- # [] means "any engine template speaking `protocol`".
- 'templates': [t for t in (block.get('templates') or []) if isinstance(t, str)],
- 'images': images,
- 'image_hint': block.get('image_hint'),
- # An extension with no declared images makes no image claim, so it
- # is never blocked on one.
- 'requires_image': bool(images),
- 'size': block.get('size'),
- 'unit': block.get('unit'),
- 'versions': [str(v) for v in versions],
- }
-
- @classmethod
- def list_extension_templates(cls) -> List[Dict]:
- """Every extension declared under a ``templates/extensions/`` directory.
-
- Derived from the files, like the engine catalog: dropping a YAML in
- makes the extension appear, with no code change anywhere.
- """
- found: List[Dict] = []
- seen = set()
- for directory in cls.extension_dirs():
- if not os.path.isdir(directory):
- continue
- for filename in sorted(os.listdir(directory)):
- if not filename.endswith(('.yaml', '.yml')):
- continue
- extension_id = filename.rsplit('.', 1)[0]
- if extension_id in seen:
- continue
- record = cls._load_extension_file(os.path.join(directory, filename),
- extension_id)
- if record:
- seen.add(extension_id)
- found.append(record)
- return found
-
- @classmethod
- def _load_extension_file(cls, path: str, extension_id: str) -> Optional[Dict]:
- """Parse + validate one extension file, or ``None`` (logged) if broken."""
- try:
- with open(path, 'r', encoding='utf-8') as fh:
- document = yaml.safe_load(fh)
- except Exception:
- return None
- validation = cls.validate_extension(document if isinstance(document, dict) else {})
- if not validation['valid']:
- return None
- return cls.extension_metadata(document, extension_id=extension_id)
-
- @classmethod
- def get_extension_template(cls, extension_id: str) -> Optional[Dict]:
- """One normalized extension by id, or ``None``."""
- if not extension_id:
- return None
- for record in cls.list_extension_templates():
- if record['id'] == extension_id:
- return record
- return None
@classmethod
def parse_template(cls, template_path: str) -> Dict:
@@ -606,43 +448,6 @@ class TemplateService:
except Exception as e:
return {'success': False, 'error': str(e)}
- @classmethod
- def generate_value(cls, var_config: Dict, force_generate: bool = False) -> str:
- """Generate a value for a variable based on its configuration.
-
- Args:
- var_config: Variable configuration dict
- force_generate: If True, always generate new value even for ports
- """
- var_type = var_config.get('type', 'string')
- default = var_config.get('default', '')
-
- if var_type == 'password':
- length = var_config.get('length', 32)
- chars = string.ascii_letters + string.digits
- if var_config.get('special_chars', False):
- chars += '!@#$%^&*'
- return ''.join(secrets.choice(chars) for _ in range(length))
-
- elif var_type == 'port':
- # ALWAYS find an available port - never trust defaults
- start_port = int(default) if default else 8000
- # A global base-port setting (Settings > managed_app_base_port)
- # overrides the per-template default when configured (non-zero).
- base = cls._managed_app_base_port()
- if base:
- start_port = base
- return str(cls._claim_port(start_port))
-
- elif var_type == 'uuid':
- import uuid
- return str(uuid.uuid4())
-
- elif var_type == 'random':
- length = var_config.get('length', 16)
- return secrets.token_hex(length // 2)
-
- return str(default)
# ==================================================================
# Magic variables
@@ -697,185 +502,6 @@ class TemplateService:
# Default password length for magic SERVICE_PASSWORD_* tokens.
MAGIC_PASSWORD_LENGTH = 32
- @classmethod
- def _classify_magic_token(cls, token: str):
- """Return ``(kind, name)`` for a bare magic token (no ``${}``), or
- ``(None, None)`` if it is not a recognized magic variable."""
- if token in cls.MAGIC_EXACT_TOKENS:
- return cls.MAGIC_EXACT_TOKENS[token], ''
- for prefix, kind in cls.MAGIC_PREFIXES:
- if token.startswith(prefix):
- return kind, token[len(prefix):]
- return None, None
-
- @classmethod
- def _generate_magic_password(cls) -> str:
- """Strong password for a SERVICE_PASSWORD_* token (alnum, no special
- chars to stay shell/compose-safe). Reuses the same primitive as
- ``generate_value(type=password)``."""
- chars = string.ascii_letters + string.digits
- return ''.join(secrets.choice(chars) for _ in range(cls.MAGIC_PASSWORD_LENGTH))
-
- @classmethod
- def _generate_magic_user(cls, name: str) -> str:
- """Service username for a SERVICE_USER_* token: ``svc_<name>_<rand>``,
- lowercased and reduced to ``[a-z0-9_]`` so it is safe as a DB/app user."""
- base = re.sub(r'[^a-z0-9]+', '_', (name or 'service').lower()).strip('_') or 'service'
- suffix = secrets.token_hex(2) # 4 hex chars, keeps it short but unique
- return f'svc_{base}_{suffix}'
-
- @classmethod
- def _resolve_magic_value(cls, kind: str, name: str, context: Dict) -> str:
- """Resolve a single magic token to a value.
-
- ``context`` may carry ``fqdn`` / ``scheme`` (and ``app_name``); when an
- FQDN is not known yet the FQDN/URL forms degrade to a documented
- placeholder (``localhost``) that the install finalizer can later fill —
- this keeps resolution best-effort and non-fatal.
- """
- context = context or {}
- if kind == 'password':
- return cls._generate_magic_password()
- if kind == 'user':
- return cls._generate_magic_user(name)
- if kind == 'base64':
- import base64
- secret = secrets.token_bytes(24)
- return base64.b64encode(secret).decode('ascii')
- if kind == 'fqdn':
- return str(context.get('fqdn') or context.get('app_name') or 'localhost')
- if kind == 'url':
- scheme = str(context.get('scheme') or 'http')
- host = context.get('fqdn') or context.get('app_name') or 'localhost'
- return f'{scheme}://{host}'
- if kind == 'metrics_token':
- from app.services import metrics_token_service
- return metrics_token_service.get_or_create()
- if kind == 'server_public_ip':
- ip = context.get('server_public_ip')
- if not ip:
- try:
- from app.services.site_domain_service import SiteDomainService
- ip = SiteDomainService.server_ip()
- except Exception:
- ip = None
- return str(ip or '')
- return ''
-
- @classmethod
- def resolve_magic_variables(cls, content: Any, context: Dict = None):
- """Scan ``content`` for ``${SERVICE_*}`` magic tokens, generate a value
- once per unique token, substitute, and return ``(substituted, generated)``.
-
- Args:
- content: A string, or a (possibly nested) dict/list — e.g. a parsed
- compose section. Returned with the same shape.
- context: Optional dict with ``app_name`` / ``fqdn`` / ``scheme`` used
- to resolve FQDN/URL tokens. No Docker/network access is performed.
-
- Returns:
- ``(substituted_content, generated_vars)`` where ``generated_vars`` maps
- each magic variable name (without ``${}``) to its generated value, so
- callers can persist them as env vars / surface them post-install.
- Pure and idempotent for a given ``content`` within one call (the same
- token always maps to the same generated value here).
- """
- context = context or {}
- generated: Dict[str, str] = {}
-
- def _collect(text: str):
- for match in re.finditer(cls.MAGIC_TOKEN_PATTERN, text):
- token = match.group(1)
- if token in generated:
- continue
- kind, name = cls._classify_magic_token(token)
- if kind is None:
- continue
- generated[token] = cls._resolve_magic_value(kind, name, context)
-
- def _walk_collect(node: Any):
- if isinstance(node, str):
- _collect(node)
- elif isinstance(node, dict):
- for value in node.values():
- _walk_collect(value)
- elif isinstance(node, list):
- for item in node:
- _walk_collect(item)
-
- # Pass 1: discover every unique token and generate a stable value.
- _walk_collect(content)
-
- # Pass 2: substitute using the generated values. Reuse the existing
- # ${VAR} substitution so behavior is identical to normal variables.
- if isinstance(content, str):
- substituted = cls.substitute_variables(content, generated)
- else:
- substituted = cls.substitute_in_dict(content, generated)
-
- return substituted, generated
-
- @classmethod
- def collect_magic_variables(cls, template: Dict, context: Dict = None) -> Dict[str, str]:
- """Generate the magic variables a template uses, given its declared
- ``compose`` / ``files`` / ``scripts`` sections, WITHOUT mutating the
- template. Returns ``{name: value}`` for every ``${SERVICE_*}`` token found.
-
- This is the wiring entry point for the install flow: the returned dict is
- merged into the install ``variables`` so the existing ``${VAR}``
- substitution renders the tokens, and so the secrets land in ``.env`` /
- post-install output. Templates with no magic tokens get ``{}`` and behave
- exactly as before.
- """
- # Scan the parts that the installer substitutes against.
- scan_target = {
- 'compose': template.get('compose', {}),
- 'files': template.get('files', []),
- 'scripts': template.get('scripts', {}),
- }
- _, generated = cls.resolve_magic_variables(scan_target, context)
- return generated
-
- @classmethod
- def _install_magic_context(cls, template: Dict, app_name: str,
- variables: Dict = None) -> Dict:
- """Build the best-effort ``context`` for magic-variable resolution at
- install time.
-
- Resolves the would-be FQDN from the managed-sites base domain when one is
- configured (the same ``<slug>.<base>`` the finalizer publishes), and the
- scheme from whether wildcard HTTPS is on. All of this is best-effort and
- non-fatal: if site routing isn't set up (or we're outside an app context),
- FQDN/URL tokens fall back to a documented ``localhost`` placeholder that
- the install finalizer can later fill in.
- """
- context = {'app_name': app_name}
- try:
- from app.services.site_domain_service import SiteDomainService
- base = SiteDomainService.base_domain()
- # Only assign an FQDN when the template opts into auto-domain and a
- # base domain exists — mirrors the finalizer's publish condition.
- if base and template.get('auto_domain'):
- host = SiteDomainService.subdomain_for(app_name)
- if host:
- context['fqdn'] = host
- context['scheme'] = 'https' if (
- SiteDomainService.https_enabled()
- and SiteDomainService.covers(host)
- ) else 'http'
- except Exception:
- # No app context / site routing not available — leave FQDN unset so
- # tokens degrade to the localhost placeholder.
- pass
- return context
-
- @classmethod
- def _collect_magic_for_install(cls, template: Dict, app_name: str,
- variables: Dict = None) -> Dict[str, str]:
- """Resolve a template's magic variables for an install, using the
- best-effort FQDN context. Thin wrapper over :meth:`collect_magic_variables`."""
- context = cls._install_magic_context(template, app_name, variables)
- return cls.collect_magic_variables(template, context)
@classmethod
def validate_catalog_entry(cls, entry: Dict) -> Dict:
@@ -1023,176 +649,6 @@ class TemplateService:
'error': f'Connection check failed: {str(e)}'
}
- @classmethod
- def _managed_app_base_port(cls) -> int:
- """Return the admin-configured base port for managed apps, or 0 if unset.
-
- Reads the ``managed_app_base_port`` system setting. Returns 0 (meaning
- "use each template's own default") on any error, e.g. if the settings
- table isn't ready yet during early startup.
- """
- try:
- from app.services.settings_service import SettingsService
- return int(SettingsService.get('managed_app_base_port', 0) or 0)
- except Exception:
- return 0
-
- @classmethod
- def _find_available_port(cls, start_port: int = 8000, max_attempts: int = 1000) -> int:
- """Find an available port that's not in use by the system, Docker, or database.
-
- Checks:
- 1. Ports assigned to existing applications in the database
- 2. Docker container port mappings
- 3. Socket binding test
- """
- import socket
-
- # Get ports from database (assigned to apps)
- db_ports = cls._get_database_used_ports()
-
- # Get ports currently used by Docker containers
- docker_ports = cls._get_docker_used_ports()
-
- # Combine all used ports
- used_ports = db_ports | docker_ports | cls._live_claims()
-
- for port in range(start_port, start_port + max_attempts):
- # Skip reserved/common ports
- if port < 1024:
- continue
-
- # Skip if already assigned in DB or Docker
- if port in used_ports:
- continue
-
- # Check if port is available on localhost (where Docker binds)
- try:
- # Try to bind - most reliable check
- test_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- test_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
- test_sock.bind(('127.0.0.1', port))
- test_sock.close()
- return port
- except OSError:
- continue
- except Exception:
- continue
-
- # A random port here could collide with anything; say so instead.
- raise ConflictError(
- f'No free port between {start_port} and {start_port + max_attempts - 1}')
-
- @staticmethod
- def _live_claims() -> set:
- now = time.monotonic()
- with _claimed_ports_lock:
- for port, at in list(_claimed_ports.items()):
- if now - at > _PORT_CLAIM_TTL:
- del _claimed_ports[port]
- return set(_claimed_ports)
-
- @classmethod
- def _claim_port(cls, start_port: int = 8000) -> int:
- """Find a free port and hold it for the install about to use it."""
- while True:
- port = cls._find_available_port(start_port)
- with _claimed_ports_lock:
- if port not in _claimed_ports:
- _claimed_ports[port] = time.monotonic()
- return port
- start_port = port + 1
-
- @classmethod
- def _port_is_free(cls, port) -> bool:
- """True when nothing -- app row, container, or live socket -- holds
- ``port``. Same three sources as :meth:`_find_available_port`, asked
- about one specific port instead of scanning."""
- import socket
- try:
- port = int(port)
- except (TypeError, ValueError):
- return False
- if not 1 <= port <= 65535:
- return False
- if port in (cls._get_database_used_ports() | cls._get_docker_used_ports()
- | cls._live_claims()):
- return False
- try:
- test_sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
- test_sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
- test_sock.bind(('127.0.0.1', port))
- test_sock.close()
- return True
- except OSError:
- return False
- except Exception:
- return False
-
- @classmethod
- def _get_database_used_ports(cls) -> set:
- """Get all ports assigned to applications in the database."""
- used_ports = set()
- try:
- from app.models import Application
- # Deliberately unfiltered: a soft-deleted app must KEEP reserving its
- # port, or a new app takes it and restoring from the recycle bin collides.
- apps = Application.query.filter(Application.port.isnot(None)).all()
- for app in apps:
- if app.port:
- used_ports.add(app.port)
- # A slot app holds two ports: the live one is app.port,
- # the standby's stays reserved for the switch back.
- from app.models.app_slot import AppSlot
- used_ports.update(row.host_port for row in
- AppSlot.query.filter(AppSlot.host_port.isnot(None)).all())
- except Exception:
- pass
- return used_ports
-
- @classmethod
- def _get_docker_used_ports(cls) -> set:
- """Get all ports currently mapped by Docker containers."""
- used_ports = set()
- try:
- result = run_checked(['docker', 'ps', '--format', '{{.Ports}}'], timeout=10)
- if result['success']:
- # Parse port mappings like "0.0.0.0:8080->80/tcp, 127.0.0.1:3306->3306/tcp"
- import re
- for line in result['output'].strip().split('\n'):
- if line:
- # Find all host ports in the format "host:port->container"
- matches = re.findall(r'(?:[\d.]+:)?(\d+)->', line)
- for port_str in matches:
- try:
- used_ports.add(int(port_str))
- except ValueError:
- pass
- except Exception:
- pass
- return used_ports
-
- @classmethod
- def substitute_variables(cls, content: str, variables: Dict) -> str:
- """Substitute variables in content using ${VAR} syntax."""
- def replace_var(match):
- var_name = match.group(1)
- return str(variables.get(var_name, match.group(0)))
-
- # Replace ${VAR} patterns
- pattern = r'\$\{([A-Z_][A-Z0-9_]*)\}'
- return re.sub(pattern, replace_var, content)
-
- @classmethod
- def substitute_in_dict(cls, data: Any, variables: Dict) -> Any:
- """Recursively substitute variables in a dictionary."""
- if isinstance(data, str):
- return cls.substitute_variables(data, variables)
- elif isinstance(data, dict):
- return {k: cls.substitute_in_dict(v, variables) for k, v in data.items()}
- elif isinstance(data, list):
- return [cls.substitute_in_dict(item, variables) for item in data]
- return data
@classmethod
def generate_compose(cls, template: Dict, variables: Dict) -> str:
@@ -1208,378 +664,6 @@ class TemplateService:
return yaml.dump(compose, default_flow_style=False, sort_keys=False)
- @classmethod
- def list_local_templates(cls) -> List[Dict]:
- """List locally available templates.
-
- Deliberately NOT provider-gated: this is the raw "what is bundled on
- disk" view, which ``build_repo_index`` needs to describe the bundle
- faithfully for publishing. Provider-owned templates are hidden from
- the *catalog* instead — :meth:`list_all_templates` and
- :meth:`get_template` apply the availability gate, so an
- absent extension means hidden cards and refused installs even though
- the YAML remains bundled.
-
- Parsing is memoized per file (see the cache block near the top of the
- class); a repeat listing stats each file instead of re-parsing it.
- """
- templates = []
- seen_ids = set()
- seen_paths = set()
-
- for templates_dir in [cls.TEMPLATES_DIR, cls.LOCAL_TEMPLATES_DIR]:
- if not os.path.isdir(templates_dir):
- continue
-
- try:
- with os.scandir(templates_dir) as scan:
- # Sorted so a same-id collision (foo.yaml vs foo.yml)
- # resolves the same way on every platform; os.listdir order
- # is filesystem-dependent.
- dir_entries = sorted(scan, key=lambda e: e.name)
- except OSError:
- continue
-
- for dir_entry in dir_entries:
- if not dir_entry.name.endswith(('.yaml', '.yml')):
- continue
- seen_paths.add(dir_entry.path)
-
- template_id = dir_entry.name.rsplit('.', 1)[0]
- if template_id in seen_ids:
- continue
- try:
- # DirEntry.stat() reuses the data the directory scan
- # already returned where the platform provides it, so this
- # is cheaper than a separate os.stat per file.
- info = dir_entry.stat()
- except OSError:
- continue
-
- entry = cls._local_entry(dir_entry.path, template_id,
- (info.st_mtime_ns, info.st_size))
- if entry is not None:
- # A file that fails to parse deliberately does NOT claim
- # the id, so a valid file of the same name in the fallback
- # directory can still supply it.
- seen_ids.add(template_id)
- templates.append(entry)
-
- cls._prune_local_cache(seen_paths)
- return templates
-
- @classmethod
- def invalidate_local_cache(cls) -> None:
- """Drop the parsed-template cache.
-
- Ordinary writes do not need this -- the cache is keyed on
- (mtime_ns, size), so syncing or editing a template invalidates exactly
- that file on the next listing. Provided for tests and any out-of-band
- mutation that could defeat mtime comparison."""
- with cls._local_cache_lock:
- cls._local_cache = {}
-
- @classmethod
- def _prune_local_cache(cls, seen_paths: set) -> None:
- """Forget files that are no longer on disk, so a long-running panel
- that churns templates does not grow the cache without bound."""
- with cls._local_cache_lock:
- stale = [p for p in cls._local_cache if p not in seen_paths]
- for path in stale:
- del cls._local_cache[path]
-
- @classmethod
- def _local_entry(cls, filepath: str, template_id: str,
- stat_key: tuple) -> Optional[Dict]:
- """Catalog projection for one bundled template, memoized on ``stat_key``.
-
- Returns None when the file does not parse -- and caches that negative
- result too, so one malformed YAML does not cost a parse on every
- render. Returns a deep copy so a caller mutating the result (or its
- nested ``repo``/``engine``/``categories``) cannot corrupt the cache."""
- with cls._local_cache_lock:
- cached = cls._local_cache.get(filepath)
- if cached is not None and cached['key'] == stat_key:
- return copy.deepcopy(cached['entry'])
-
- entry = cls._parse_local_entry(filepath, template_id)
-
- with cls._local_cache_lock:
- cls._local_cache[filepath] = {'key': stat_key, 'entry': entry}
- return copy.deepcopy(entry)
-
- @classmethod
- def _parse_local_entry(cls, filepath: str, template_id: str) -> Optional[Dict]:
- """Parse one template file into its catalog projection, or None."""
- result = cls.parse_template(filepath)
- if not result.get('success'):
- return None
-
- template = result['template']
- kind = template.get('kind', 'compose')
- entry = {
- 'id': template_id,
- 'name': template.get('name'),
- 'version': template.get('version'),
- 'description': template.get('description'),
- 'icon': template.get('icon'),
- 'categories': template.get('categories', []),
- 'kind': kind,
- 'source': 'local',
- 'filepath': filepath
- }
- # Repo templates surface a minimal repo summary so the catalog grid
- # can badge them and link to the wizard without a second fetch.
- if kind == 'repo' and isinstance(template.get('repo'), dict):
- repo = template['repo']
- entry['repo'] = {
- 'url': repo.get('url'),
- 'branch': repo.get('branch', 'main'),
- }
- # A database engine declares itself with a top-level `engine:` block.
- # This listing is a fixed projection, so without carrying the block
- # through, the Databases catalog could never see an engine without
- # re-parsing every YAML file.
- engine = cls.engine_metadata(template)
- if engine:
- entry['engine'] = engine
- return entry
-
- @classmethod
- def list_engine_templates(cls) -> List[Dict]:
- """Every available template carrying an ``engine:`` block.
-
- This is what the Databases engine catalog renders. It is derived
- entirely from the template sources -- drop a new engine YAML into
- ``backend/templates/`` (or sync a repository that ships one) and it
- appears here with no code change.
- """
- return [t for t in cls.list_all_templates()
- if isinstance(t.get('engine'), dict)]
-
- @classmethod
- def build_repo_index(cls, repo_name: str = 'serverkit-official') -> Dict:
- """Build the ``index.json`` document that describes the locally-bundled
- templates as a publishable repository.
-
- This is the exact shape :meth:`fetch_remote_templates` / :meth:`sync_templates`
- consume from ``<repo_url>/index.json`` (templates served at
- ``<repo_url>/templates/<id>.yaml``). Publishing a repo is then just:
- host the ``templates/*.yaml`` files plus this ``index.json``. Lets
- Prompture Hub & friends ship template updates without a panel release.
- """
- templates = []
- for t in cls.list_local_templates():
- entry = {
- 'id': t['id'],
- 'name': t.get('name'),
- 'version': t.get('version'),
- 'description': t.get('description'),
- 'icon': t.get('icon'),
- 'categories': t.get('categories', []),
- }
- # Carried in the index so a panel that syncs this repo can list the
- # engine catalog without fetching all 100+ template files first.
- if isinstance(t.get('engine'), dict):
- entry['engine'] = t['engine']
- templates.append(entry)
- return {
- 'name': repo_name,
- 'schema_version': cls.SCHEMA_VERSION,
- 'generated_at': datetime.now().isoformat(),
- 'count': len(templates),
- 'templates': templates,
- }
-
- @classmethod
- def export_repo_index(cls, dest_path: str = None) -> Dict:
- """Write :meth:`build_repo_index` to ``dest_path`` (defaults to
- ``index.json`` alongside the bundled templates). Returns a status dict."""
- index = cls.build_repo_index()
- path = dest_path or os.path.join(cls.LOCAL_TEMPLATES_DIR, 'index.json')
- try:
- with open(path, 'w', encoding='utf-8') as f:
- json.dump(index, f, indent=2)
- return {'success': True, 'path': path, 'count': index['count']}
- except Exception as e:
- return {'success': False, 'error': str(e)}
-
- @classmethod
- def invalidate_remote_cache(cls, repo_url: str = None) -> None:
- """Drop cached remote indexes -- all of them, or just ``repo_url``.
-
- Called whenever the repo list changes so a freshly added repo appears
- immediately instead of after the TTL, and a removed one stops being
- served from cache."""
- with cls._remote_cache_lock:
- if repo_url is None:
- cls._remote_cache = {}
- else:
- cls._remote_cache.pop(repo_url, None)
-
- @classmethod
- def _cached_remote(cls, repo_url: str) -> Optional[List[Dict]]:
- """Unexpired cache entry for ``repo_url``, or None. Returns copies so a
- caller mutating an entry cannot corrupt the cache."""
- with cls._remote_cache_lock:
- entry = cls._remote_cache.get(repo_url)
- if entry and entry['expires'] > time.monotonic():
- return [dict(t) for t in entry['templates']]
- return None
-
- @classmethod
- def _last_good_remote(cls, repo_url: str) -> Optional[List[Dict]]:
- """Last payload this repo successfully served, regardless of expiry."""
- with cls._remote_cache_lock:
- entry = cls._remote_cache.get(repo_url) or {}
- last_good = entry.get('last_good')
- return [dict(t) for t in last_good] if last_good else None
-
- @classmethod
- def _store_remote(cls, repo_url: str, templates: List[Dict], ok: bool) -> None:
- """Cache ``templates`` for ``repo_url``. A successful payload also
- becomes the last-good; a failure carries the previous last-good
- forward, so repeated failures keep serving it instead of decaying to
- an empty catalog on the second one."""
- ttl = cls._REMOTE_TTL_SECONDS if ok else cls._REMOTE_ERROR_TTL_SECONDS
- with cls._remote_cache_lock:
- previous = cls._remote_cache.get(repo_url) or {}
- cls._remote_cache[repo_url] = {
- 'templates': [dict(t) for t in templates],
- 'expires': time.monotonic() + ttl,
- 'ok': ok,
- 'last_good': ([dict(t) for t in templates] if ok
- else previous.get('last_good')),
- }
-
- @classmethod
- def fetch_remote_templates(cls, repo_url: str, force: bool = False) -> List[Dict]:
- """Fetch a repository's template index, memoized (see the cache block
- near the top of the class). ``force=True`` bypasses the cache, which is
- what an explicit sync wants."""
- if not force:
- cached = cls._cached_remote(repo_url)
- if cached is not None:
- return cached
-
- templates = []
- try:
- index_url = f"{repo_url}/index.json"
- response = requests.get(index_url, timeout=cls._REMOTE_TIMEOUT_SECONDS)
- response.raise_for_status()
-
- index = response.json()
- for template_info in index.get('templates', []):
- template_info['source'] = 'remote'
- template_info['repo_url'] = repo_url
- templates.append(template_info)
-
- except Exception as e:
- print(f"Failed to fetch templates from {repo_url}: {e}")
- # Keep serving the last good index rather than blanking a catalog
- # the operator has already seen, but still back off so a dead repo
- # is not retried on every single page load.
- last_good = cls._last_good_remote(repo_url)
- cls._store_remote(repo_url, last_good or [], ok=False)
- return last_good or []
-
- cls._store_remote(repo_url, templates, ok=True)
- return [dict(t) for t in templates]
-
- @classmethod
- def list_all_templates(cls, category: str = None, search: str = None) -> List[Dict]:
- """List all available templates from all sources, deduped by id.
-
- Bundled and registry templates deliberately overlap -- the bundle is the
- offline floor and the registry is the growth path (docs/REGISTRIES.md) --
- so the same id legitimately arrives from both. Concatenating them listed
- every bundled template twice. **Local wins**, matching
- :meth:`get_template`, which already resolves local directories before
- any repo; among repos, the first enabled one to claim an id keeps it.
- """
- by_id = {}
-
- def _claim(entry):
- """First source to claim an id keeps it."""
- template_id = entry.get('id')
- if template_id and template_id not in by_id:
- by_id[template_id] = entry
-
- # Local templates first -- they outrank every repo.
- for entry in cls.list_local_templates():
- _claim(entry)
-
- # Remote templates fill in ids the bundle does not carry.
- config = cls.get_config()
- for repo in config.get('repos', []):
- if repo.get('enabled', True):
- for entry in cls.fetch_remote_templates(repo['url']):
- _claim(entry)
-
- templates = list(by_id.values())
-
- # Filter by category
- if category:
- templates = [t for t in templates if category in t.get('categories', [])]
-
- # Provider-owned templates from any source (incl. synced remote repos)
- # are hidden while their extension is absent.
- templates = [t for t in templates if cls.template_available(t.get('id'))]
-
- # Search filter
- if search:
- search_lower = search.lower()
- templates = [
- t for t in templates
- if search_lower in t.get('name', '').lower()
- or search_lower in t.get('description', '').lower()
- ]
-
- return templates
-
- @classmethod
- def get_template(cls, template_id: str) -> Dict:
- """Get full template details."""
- if not cls.template_available(template_id):
- owner = cls.PROVIDER_OWNED_TEMPLATES.get(template_id)
- return {
- 'success': False,
- 'error': f"Template '{template_id}' is provided by the "
- f"'{owner}' extension, which is not installed or not active",
- }
- # Check local directories (system dir, then bundled fallback)
- for templates_dir in [cls.TEMPLATES_DIR, cls.LOCAL_TEMPLATES_DIR]:
- for ext in ['.yaml', '.yml']:
- filepath = os.path.join(templates_dir, f"{template_id}{ext}")
- if os.path.exists(filepath):
- result = cls.parse_template(filepath)
- if result.get('success'):
- template = result['template']
- template['source'] = 'local'
- template['filepath'] = filepath
- return {'success': True, 'template': template}
- return result
-
- # Check remote repos
- config = cls.get_config()
- for repo in config.get('repos', []):
- if not repo.get('enabled', True):
- continue
-
- try:
- url = f"{repo['url']}/templates/{template_id}.yaml"
- response = requests.get(url, timeout=30)
- if response.status_code == 200:
- template = yaml.safe_load(response.text)
- validation = cls.validate_template(template)
- if validation['valid']:
- template['source'] = 'remote'
- template['repo_url'] = repo['url']
- return {'success': True, 'template': template}
- except Exception:
- continue
-
- return {'success': False, 'error': 'Template not found'}
@classmethod
def build_template_hints(cls, template: Dict) -> Dict:
@@ -2652,156 +1736,4 @@ class TemplateService:
except Exception as e:
return {'success': False, 'error': str(e)}
- @classmethod
- def add_repository(cls, name: str, url: str) -> Dict:
- """Add a template repository."""
- config = cls.get_config()
-
- # Check if already exists
- for repo in config.get('repos', []):
- if repo['url'] == url:
- return {'success': False, 'error': 'Repository already exists'}
-
- config.setdefault('repos', []).append({
- 'name': name,
- 'url': url.rstrip('/'),
- 'enabled': True,
- 'added_at': datetime.now().isoformat()
- })
-
- return cls.save_config(config)
-
- @classmethod
- def remove_repository(cls, url: str) -> Dict:
- """Remove a template repository."""
- config = cls.get_config()
- config['repos'] = [r for r in config.get('repos', []) if r['url'] != url]
- return cls.save_config(config)
-
- @classmethod
- def list_repositories(cls) -> List[Dict]:
- """List configured template repositories."""
- config = cls.get_config()
- return config.get('repos', cls.DEFAULT_REPOS)
-
- @classmethod
- def sync_templates(cls) -> Dict:
- """Sync templates from all repositories."""
- os.makedirs(cls.TEMPLATES_DIR, exist_ok=True)
-
- config = cls.get_config()
- synced = 0
- unverified = 0 # saved, but the index pinned no sha256 for them
- errors = []
-
- for repo in config.get('repos', []):
- if not repo.get('enabled', True):
- continue
-
- try:
- # Fetch index
- index_url = f"{repo['url']}/index.json"
- response = requests.get(index_url, timeout=30)
- response.raise_for_status()
-
- index = response.json()
-
- # Download each template
- for template_info in index.get('templates', []):
- template_id = template_info.get('id')
- if not template_id:
- continue
-
- try:
- template_url = f"{repo['url']}/templates/{template_id}.yaml"
- response = requests.get(template_url, timeout=30)
- response.raise_for_status()
-
- # Verify against the checksum the index pinned for this
- # entry. A template is a deploy definition -- images,
- # ports, volumes, env -- so a swapped file is worth
- # refusing outright, and the index already carries the
- # hash for every official entry.
- #
- # Missing hash is allowed (third-party repos may not
- # publish one) and counted, mirroring how extensions
- # treat unsigned-vs-invalid: absent is a caveat, wrong
- # is a hard stop.
- expected = (template_info.get('sha256') or '').strip().lower()
- if expected:
- actual = hashlib.sha256(response.content).hexdigest()
- if actual != expected:
- errors.append(
- f"Checksum mismatch for {template_id}: index pinned "
- f"{expected[:12]}..., downloaded {actual[:12]}.... Not saved."
- )
- continue
- else:
- unverified += 1
-
- # Written as bytes so what lands on disk is exactly the
- # content that was hashed (text mode would rewrite line
- # endings on Windows and no longer match).
- filepath = os.path.join(cls.TEMPLATES_DIR, f"{template_id}.yaml")
- with open(filepath, 'wb') as f:
- f.write(response.content)
-
- synced += 1
- except Exception as e:
- errors.append(f"Failed to sync {template_id}: {e}")
-
- except Exception as e:
- errors.append(f"Failed to sync from {repo['name']}: {e}")
-
- config['last_sync'] = datetime.now().isoformat()
- cls.save_config(config)
-
- return {
- 'success': True,
- 'synced': synced,
- 'unverified': unverified,
- 'errors': errors if errors else None
- }
-
- @classmethod
- def get_categories(cls) -> List[str]:
- """Get all available template categories."""
- templates = cls.list_all_templates()
- categories = set()
- for template in templates:
- categories.update(template.get('categories', []))
- return sorted(categories)
-
- @classmethod
- def create_local_template(cls, template_data: Dict) -> Dict:
- """Create a local template."""
- validation = cls.validate_template(template_data)
- if not validation['valid']:
- return {'success': False, 'errors': validation['errors']}
-
- os.makedirs(cls.TEMPLATES_DIR, exist_ok=True)
-
- template_id = template_data['name'].lower().replace(' ', '-')
- filepath = os.path.join(cls.TEMPLATES_DIR, f"{template_id}.yaml")
-
- if os.path.exists(filepath):
- return {'success': False, 'error': 'Template with this name already exists'}
-
- try:
- with open(filepath, 'w') as f:
- yaml.dump(template_data, f, default_flow_style=False, sort_keys=False)
-
- return {'success': True, 'template_id': template_id, 'filepath': filepath}
- except Exception as e:
- return {'success': False, 'error': str(e)}
-
- @classmethod
- def delete_local_template(cls, template_id: str) -> Dict:
- """Delete a local template."""
- for ext in ['.yaml', '.yml']:
- filepath = os.path.join(cls.TEMPLATES_DIR, f"{template_id}{ext}")
- if os.path.exists(filepath):
- os.remove(filepath)
- return {'success': True}
- return {'success': False, 'error': 'Template not found'}
diff --git a/backend/tests/test_port_claims.py b/backend/tests/test_port_claims.py
index 3a7b10bc..c40d4f08 100644
--- a/backend/tests/test_port_claims.py
+++ b/backend/tests/test_port_claims.py
@@ -10,7 +10,7 @@ import threading
import pytest
from app.exceptions import ConflictError
-from app.services import template_service
+from app.services import template_ports
from app.services.template_service import TemplateService
@@ -18,7 +18,7 @@ from app.services.template_service import TemplateService
def nothing_in_use(monkeypatch):
monkeypatch.setattr(TemplateService, '_get_database_used_ports', classmethod(lambda cls: set()))
monkeypatch.setattr(TemplateService, '_get_docker_used_ports', classmethod(lambda cls: set()))
- monkeypatch.setattr(template_service, '_claimed_ports', {})
+ monkeypatch.setattr(template_ports, '_claimed_ports', {})
def test_concurrent_claims_get_distinct_ports(nothing_in_use):
@@ -44,7 +44,7 @@ def test_claimed_port_is_not_suggested_or_free(nothing_in_use):
def test_claims_expire(nothing_in_use, monkeypatch):
claimed = TemplateService._claim_port(41200)
- monkeypatch.setattr(template_service, '_PORT_CLAIM_TTL', -1)
+ monkeypatch.setattr(template_ports, '_PORT_CLAIM_TTL', -1)
assert TemplateService._find_available_port(41200) == claimed
diff --git a/backend/tests/test_template_registry_cache.py b/backend/tests/test_template_registry_cache.py
index fa0e21a4..c21ee1e5 100644
--- a/backend/tests/test_template_registry_cache.py
+++ b/backend/tests/test_template_registry_cache.py
@@ -66,7 +66,7 @@ def _serve(monkeypatch, payload=None, fail=False):
raise RuntimeError("connection refused")
return _FakeResponse(payload or {"templates": []})
- monkeypatch.setattr("app.services.template_service.requests.get", fake_get)
+ monkeypatch.setattr("app.services.template_catalog.requests.get", fake_get)
return calls
@@ -121,7 +121,7 @@ def test_first_enabled_repo_wins_among_remotes(monkeypatch):
def fake_get(url, **kwargs):
return _FakeResponse(_index("shared-id"))
- monkeypatch.setattr("app.services.template_service.requests.get", fake_get)
+ monkeypatch.setattr("app.services.template_catalog.requests.get", fake_get)
by_id = {t["id"]: t for t in TemplateService.list_all_templates()}
assert by_id["shared-id"]["repo_url"] == "https://a.test/t"
diff --git a/backend/tests/test_template_repo_sync.py b/backend/tests/test_template_repo_sync.py
index ed4a8118..39e5fa3d 100644
--- a/backend/tests/test_template_repo_sync.py
+++ b/backend/tests/test_template_repo_sync.py
@@ -134,7 +134,7 @@ def _run_sync(tmp_path, index_entry, body):
with patch.object(TemplateService, 'TEMPLATES_DIR', str(templates_dir)), \
patch.object(TemplateService, 'TEMPLATE_CONFIG', str(config_path)), \
patch.object(TemplateService, 'CONFIG_DIR', str(tmp_path)), \
- patch('app.services.template_service.requests.get', side_effect=fake_get):
+ patch('app.services.template_catalog.requests.get', side_effect=fake_get):
result = TemplateService.sync_templates()
return result, templates_dir / 'demo.yaml'
@@ -203,7 +203,7 @@ def test_one_bad_template_does_not_abort_the_rest(tmp_path):
with patch.object(TemplateService, 'TEMPLATES_DIR', str(templates_dir)), \
patch.object(TemplateService, 'TEMPLATE_CONFIG', str(config_path)), \
patch.object(TemplateService, 'CONFIG_DIR', str(tmp_path)), \
- patch('app.services.template_service.requests.get', side_effect=fake_get):
+ patch('app.services.template_catalog.requests.get', side_effect=fake_get):
result = TemplateService.sync_templates()
assert result['synced'] == 1