Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

perf(templates): cache parsed local templates on file mtime

Juan Denis6e03c0a523c8
diff --git a/backend/app/services/template_service.py b/backend/app/services/template_service.py
index 4f4eb764..9fd55a48 100644
--- a/backend/app/services/template_service.py
+++ b/backend/app/services/template_service.py
@@ -12,6 +12,7 @@ Supports:
import os
import re
+import copy
import time
import yaml
import json
@@ -73,6 +74,23 @@ class TemplateService:
_remote_cache = {}
_remote_cache_lock = threading.Lock()
+ # Parsed-template cache for the bundled catalog.
+ #
+ # With the remote fetch memoized above, the remaining cost of a Templates
+ # page load was `list_local_templates()` re-reading and re-parsing every
+ # bundled YAML on every call (118 files, ~0.4s). Entries are memoized per
+ # file and keyed on (st_mtime_ns, st_size), so a sync or an edit
+ # invalidates exactly the file that changed and nothing else -- which is
+ # why the writers need no explicit invalidation call. A deleted template
+ # disappears because the listing only returns what the directory scan
+ # finds. (Caveat: a filesystem with coarse mtime granularity could serve a
+ # stale card if a file were rewritten to the identical size within the
+ # same clock tick; ns-resolution local filesystems do not have this
+ # problem.)
+ # filepath -> {'key': (mtime_ns, size), 'entry': dict|None}
+ _local_cache = {}
+ _local_cache_lock = threading.Lock()
+
# Repo URLs that never worked and should be healed on read rather than
# left to rot in an operator's templates.json. `serverkit/templates` was a
# guess at the org name -- the registry is `jhd3197/serverkit-templates` --
@@ -1154,57 +1172,132 @@ class TemplateService:
:meth:`get_template` apply the availability gate (plan 52 D4), so an
absent extension means hidden cards and refused installs even though
the YAML remains bundled.
+
+ Parsing is memoized per file (see the cache block near the top of the
+ class); a repeat listing stats each file instead of re-parsing it.
"""
templates = []
seen_ids = set()
+ seen_paths = set()
for templates_dir in [cls.TEMPLATES_DIR, cls.LOCAL_TEMPLATES_DIR]:
- if not os.path.exists(templates_dir):
+ if not os.path.isdir(templates_dir):
continue
- for filename in os.listdir(templates_dir):
- if filename.endswith('.yaml') or filename.endswith('.yml'):
- template_id = filename.rsplit('.', 1)[0]
- if template_id in seen_ids:
- continue
- filepath = os.path.join(templates_dir, filename)
- result = cls.parse_template(filepath)
- if result.get('success'):
- template = result['template']
- seen_ids.add(template_id)
- kind = template.get('kind', 'compose')
- entry = {
- 'id': template_id,
- 'name': template.get('name'),
- 'version': template.get('version'),
- 'description': template.get('description'),
- 'icon': template.get('icon'),
- 'categories': template.get('categories', []),
- 'kind': kind,
- 'source': 'local',
- 'filepath': filepath
- }
- # Repo templates surface a minimal repo summary so the
- # catalog grid can badge them and link to the wizard
- # without a second fetch.
- if kind == 'repo' and isinstance(template.get('repo'), dict):
- repo = template['repo']
- entry['repo'] = {
- 'url': repo.get('url'),
- 'branch': repo.get('branch', 'main'),
- }
- # A database engine declares itself with a top-level
- # `engine:` block. This listing is a fixed projection,
- # so without carrying the block through, the Databases
- # catalog could never see an engine without re-parsing
- # every YAML file.
- engine = cls.engine_metadata(template)
- if engine:
- entry['engine'] = engine
- templates.append(entry)
+ try:
+ with os.scandir(templates_dir) as scan:
+ # Sorted so a same-id collision (foo.yaml vs foo.yml)
+ # resolves the same way on every platform; os.listdir order
+ # is filesystem-dependent.
+ dir_entries = sorted(scan, key=lambda e: e.name)
+ except OSError:
+ continue
+
+ for dir_entry in dir_entries:
+ if not dir_entry.name.endswith(('.yaml', '.yml')):
+ continue
+ seen_paths.add(dir_entry.path)
+ template_id = dir_entry.name.rsplit('.', 1)[0]
+ if template_id in seen_ids:
+ continue
+ try:
+ # DirEntry.stat() reuses the data the directory scan
+ # already returned where the platform provides it, so this
+ # is cheaper than a separate os.stat per file.
+ info = dir_entry.stat()
+ except OSError:
+ continue
+
+ entry = cls._local_entry(dir_entry.path, template_id,
+ (info.st_mtime_ns, info.st_size))
+ if entry is not None:
+ # A file that fails to parse deliberately does NOT claim
+ # the id, so a valid file of the same name in the fallback
+ # directory can still supply it.
+ seen_ids.add(template_id)
+ templates.append(entry)
+
+ cls._prune_local_cache(seen_paths)
return templates
+ @classmethod
+ def invalidate_local_cache(cls) -> None:
+ """Drop the parsed-template cache.
+
+ Ordinary writes do not need this -- the cache is keyed on
+ (mtime_ns, size), so syncing or editing a template invalidates exactly
+ that file on the next listing. Provided for tests and any out-of-band
+ mutation that could defeat mtime comparison."""
+ with cls._local_cache_lock:
+ cls._local_cache = {}
+
+ @classmethod
+ def _prune_local_cache(cls, seen_paths: set) -> None:
+ """Forget files that are no longer on disk, so a long-running panel
+ that churns templates does not grow the cache without bound."""
+ with cls._local_cache_lock:
+ stale = [p for p in cls._local_cache if p not in seen_paths]
+ for path in stale:
+ del cls._local_cache[path]
+
+ @classmethod
+ def _local_entry(cls, filepath: str, template_id: str,
+ stat_key: tuple) -> Optional[Dict]:
+ """Catalog projection for one bundled template, memoized on ``stat_key``.
+
+ Returns None when the file does not parse -- and caches that negative
+ result too, so one malformed YAML does not cost a parse on every
+ render. Returns a deep copy so a caller mutating the result (or its
+ nested ``repo``/``engine``/``categories``) cannot corrupt the cache."""
+ with cls._local_cache_lock:
+ cached = cls._local_cache.get(filepath)
+ if cached is not None and cached['key'] == stat_key:
+ return copy.deepcopy(cached['entry'])
+
+ entry = cls._parse_local_entry(filepath, template_id)
+
+ with cls._local_cache_lock:
+ cls._local_cache[filepath] = {'key': stat_key, 'entry': entry}
+ return copy.deepcopy(entry)
+
+ @classmethod
+ def _parse_local_entry(cls, filepath: str, template_id: str) -> Optional[Dict]:
+ """Parse one template file into its catalog projection, or None."""
+ result = cls.parse_template(filepath)
+ if not result.get('success'):
+ return None
+
+ template = result['template']
+ kind = template.get('kind', 'compose')
+ entry = {
+ 'id': template_id,
+ 'name': template.get('name'),
+ 'version': template.get('version'),
+ 'description': template.get('description'),
+ 'icon': template.get('icon'),
+ 'categories': template.get('categories', []),
+ 'kind': kind,
+ 'source': 'local',
+ 'filepath': filepath
+ }
+ # Repo templates surface a minimal repo summary so the catalog grid
+ # can badge them and link to the wizard without a second fetch.
+ if kind == 'repo' and isinstance(template.get('repo'), dict):
+ repo = template['repo']
+ entry['repo'] = {
+ 'url': repo.get('url'),
+ 'branch': repo.get('branch', 'main'),
+ }
+ # A database engine declares itself with a top-level `engine:` block.
+ # This listing is a fixed projection, so without carrying the block
+ # through, the Databases catalog could never see an engine without
+ # re-parsing every YAML file.
+ engine = cls.engine_metadata(template)
+ if engine:
+ entry['engine'] = engine
+ return entry
+
@classmethod
def list_engine_templates(cls) -> List[Dict]:
"""Every available template carrying an ``engine:`` block.
diff --git a/backend/tests/test_template_registry_cache.py b/backend/tests/test_template_registry_cache.py
index 56fa1533..fa0e21a4 100644
--- a/backend/tests/test_template_registry_cache.py
+++ b/backend/tests/test_template_registry_cache.py
@@ -10,6 +10,8 @@ Two bugs motivated these, both only latent while ``DEFAULT_REPOS`` pointed at a
timeout and no memoization, so one unreachable repo stalled the Templates
page on every render.
"""
+import os
+
import pytest
from app.services.template_service import TemplateService
@@ -36,10 +38,12 @@ class _FakeResponse:
@pytest.fixture(autouse=True)
def _clean_cache():
- """The cache is class-level state; never leak it between tests."""
+ """Both caches are class-level state; never leak them between tests."""
TemplateService.invalidate_remote_cache()
+ TemplateService.invalidate_local_cache()
yield
TemplateService.invalidate_remote_cache()
+ TemplateService.invalidate_local_cache()
@pytest.fixture
@@ -204,3 +208,159 @@ def test_cached_entries_are_copies(monkeypatch, one_repo):
second = TemplateService.fetch_remote_templates(REPO_URL)
assert second[0]["name"] == "remote-a"
+
+
+# ---------------------------------------------------------------------------
+# Local parse cache
+#
+# With the remote fetch memoized, re-parsing every bundled YAML on every call
+# became the whole cost of a Templates page load (118 files, ~0.4s).
+# ---------------------------------------------------------------------------
+MINIMAL = """\
+name: {name}
+version: "1.0"
+description: A fixture template
+categories:
+ - test
+compose:
+ services:
+ app:
+ image: nginx:alpine
+"""
+
+
+def template_dirs(monkeypatch, tmp_path):
+ """Point both template directories at a temp tree.
+
+ ``TEMPLATES_DIR`` is the primary (synced/user) directory and
+ ``LOCAL_TEMPLATES_DIR`` the bundled fallback -- the same precedence
+ ``list_local_templates`` walks in production."""
+ primary = tmp_path / "primary"
+ fallback = tmp_path / "fallback"
+ primary.mkdir()
+ fallback.mkdir()
+ monkeypatch.setattr(TemplateService, "TEMPLATES_DIR", str(primary))
+ monkeypatch.setattr(TemplateService, "LOCAL_TEMPLATES_DIR", str(fallback))
+ TemplateService.invalidate_local_cache()
+ return primary, fallback
+
+
+def parse_calls(monkeypatch):
+ """Record every real parse so the tests assert on work done, not wall time."""
+ calls = []
+ original = TemplateService.parse_template
+
+ def counting(template_path):
+ calls.append(template_path)
+ return original(template_path)
+
+ monkeypatch.setattr(TemplateService, "parse_template", staticmethod(counting))
+ return calls
+
+
+def test_repeat_listing_does_not_reparse(template_dirs, parse_calls):
+ """The bug: every catalog render re-read and re-parsed the whole bundle."""
+ primary, _ = template_dirs
+ for i in range(3):
+ (primary / f"t{i}.yaml").write_text(MINIMAL.format(name=f"T{i}"))
+
+ assert len(TemplateService.list_local_templates()) == 3
+ assert len(parse_calls) == 3
+
+ TemplateService.list_local_templates()
+ TemplateService.list_local_templates()
+
+ assert len(parse_calls) == 3, "repeat listing re-parsed unchanged files"
+
+
+def test_editing_one_template_reparses_only_that_file(template_dirs, parse_calls):
+ """Per-file keying: a sync that rewrites one template costs one parse."""
+ primary, _ = template_dirs
+ for i in range(3):
+ (primary / f"t{i}.yaml").write_text(MINIMAL.format(name=f"T{i}"))
+ TemplateService.list_local_templates()
+ parse_calls.clear()
+
+ # Changes both mtime and size, so this holds even on a filesystem with
+ # coarse mtime granularity.
+ (primary / "t1.yaml").write_text(MINIMAL.format(name="Renamed") + "\n# edited\n")
+
+ listed = {t["id"]: t for t in TemplateService.list_local_templates()}
+
+ assert [os.path.basename(p) for p in parse_calls] == ["t1.yaml"]
+ assert listed["t1"]["name"] == "Renamed"
+ assert listed["t0"]["name"] == "T0"
+
+
+def test_deleted_template_disappears_and_is_pruned(template_dirs):
+ """A removed file leaves the catalog and does not linger in the cache."""
+ primary, _ = template_dirs
+ path = primary / "gone.yaml"
+ path.write_text(MINIMAL.format(name="Gone"))
+
+ assert [t["id"] for t in TemplateService.list_local_templates()] == ["gone"]
+ assert str(path) in TemplateService._local_cache
+
+ path.unlink()
+
+ assert TemplateService.list_local_templates() == []
+ assert str(path) not in TemplateService._local_cache
+
+
+def test_unparseable_template_is_not_retried_until_it_changes(template_dirs, parse_calls):
+ """One malformed YAML must not cost a parse on every single render."""
+ primary, _ = template_dirs
+ broken = primary / "broken.yaml"
+ broken.write_text("name: [unclosed\n")
+
+ assert TemplateService.list_local_templates() == []
+ assert len(parse_calls) == 1
+
+ TemplateService.list_local_templates()
+ TemplateService.list_local_templates()
+ assert len(parse_calls) == 1, "malformed template re-parsed on every listing"
+
+ broken.write_text(MINIMAL.format(name="Fixed"))
+
+ assert [t["id"] for t in TemplateService.list_local_templates()] == ["broken"]
+ assert len(parse_calls) == 2
+
+
+def test_unparseable_primary_does_not_block_the_fallback(template_dirs):
+ """A file that fails to parse must not claim its id, so a valid file of the
+ same name in the fallback directory still supplies it."""
+ primary, fallback = template_dirs
+ (primary / "dup.yaml").write_text("name: [unclosed\n")
+ (fallback / "dup.yaml").write_text(MINIMAL.format(name="FromFallback"))
+
+ assert [t["name"] for t in TemplateService.list_local_templates()] == ["FromFallback"]
+
+
+def test_primary_directory_wins_over_fallback(template_dirs):
+ """Precedence is unchanged by caching: the primary directory shadows the bundle."""
+ primary, fallback = template_dirs
+ (primary / "dup.yaml").write_text(MINIMAL.format(name="FromPrimary"))
+ (fallback / "dup.yaml").write_text(MINIMAL.format(name="FromFallback"))
+
+ listed = TemplateService.list_local_templates()
+
+ assert len(listed) == 1
+ assert listed[0]["name"] == "FromPrimary"
+
+
+def test_local_entries_are_copies(template_dirs):
+ """A caller mutating an entry -- or its nested categories list -- must not
+ corrupt the cache."""
+ primary, _ = template_dirs
+ (primary / "c.yaml").write_text(MINIMAL.format(name="C"))
+
+ first = TemplateService.list_local_templates()[0]
+ first["categories"].append("mutated")
+ first["name"] = "mutated"
+
+ second = TemplateService.list_local_templates()[0]
+
+ assert second["categories"] == ["test"]
+ assert second["name"] == "C"