ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
perf(templates): cache parsed local templates on file mtime
Juan Denis6e03c0a523c8
diff --git a/backend/app/services/template_service.py b/backend/app/services/template_service.pyindex 4f4eb764..9fd55a48 100644--- a/backend/app/services/template_service.py+++ b/backend/app/services/template_service.py@@ -12,6 +12,7 @@ Supports:import osimport re+import copyimport timeimport yamlimport json@@ -73,6 +74,23 @@ class TemplateService:_remote_cache = {}_remote_cache_lock = threading.Lock()+ # Parsed-template cache for the bundled catalog.+ #+ # With the remote fetch memoized above, the remaining cost of a Templates+ # page load was `list_local_templates()` re-reading and re-parsing every+ # bundled YAML on every call (118 files, ~0.4s). Entries are memoized per+ # file and keyed on (st_mtime_ns, st_size), so a sync or an edit+ # invalidates exactly the file that changed and nothing else -- which is+ # why the writers need no explicit invalidation call. A deleted template+ # disappears because the listing only returns what the directory scan+ # finds. (Caveat: a filesystem with coarse mtime granularity could serve a+ # stale card if a file were rewritten to the identical size within the+ # same clock tick; ns-resolution local filesystems do not have this+ # problem.)+ # filepath -> {'key': (mtime_ns, size), 'entry': dict|None}+ _local_cache = {}+ _local_cache_lock = threading.Lock()+# Repo URLs that never worked and should be healed on read rather than# left to rot in an operator's templates.json. `serverkit/templates` was a# guess at the org name -- the registry is `jhd3197/serverkit-templates` --@@ -1154,57 +1172,132 @@ class TemplateService::meth:`get_template` apply the availability gate (plan 52 D4), so anabsent extension means hidden cards and refused installs even thoughthe YAML remains bundled.++ Parsing is memoized per file (see the cache block near the top of the+ class); a repeat listing stats each file instead of re-parsing it."""templates = []seen_ids = set()+ seen_paths = set()for templates_dir in [cls.TEMPLATES_DIR, cls.LOCAL_TEMPLATES_DIR]:- if not os.path.exists(templates_dir):+ if not os.path.isdir(templates_dir):continue- for filename in os.listdir(templates_dir):- if filename.endswith('.yaml') or filename.endswith('.yml'):- template_id = filename.rsplit('.', 1)[0]- if template_id in seen_ids:- continue- filepath = os.path.join(templates_dir, filename)- result = cls.parse_template(filepath)- if result.get('success'):- template = result['template']- seen_ids.add(template_id)- kind = template.get('kind', 'compose')- entry = {- 'id': template_id,- 'name': template.get('name'),- 'version': template.get('version'),- 'description': template.get('description'),- 'icon': template.get('icon'),- 'categories': template.get('categories', []),- 'kind': kind,- 'source': 'local',- 'filepath': filepath- }- # Repo templates surface a minimal repo summary so the- # catalog grid can badge them and link to the wizard- # without a second fetch.- if kind == 'repo' and isinstance(template.get('repo'), dict):- repo = template['repo']- entry['repo'] = {- 'url': repo.get('url'),- 'branch': repo.get('branch', 'main'),- }- # A database engine declares itself with a top-level- # `engine:` block. This listing is a fixed projection,- # so without carrying the block through, the Databases- # catalog could never see an engine without re-parsing- # every YAML file.- engine = cls.engine_metadata(template)- if engine:- entry['engine'] = engine- templates.append(entry)+ try:+ with os.scandir(templates_dir) as scan:+ # Sorted so a same-id collision (foo.yaml vs foo.yml)+ # resolves the same way on every platform; os.listdir order+ # is filesystem-dependent.+ dir_entries = sorted(scan, key=lambda e: e.name)+ except OSError:+ continue++ for dir_entry in dir_entries:+ if not dir_entry.name.endswith(('.yaml', '.yml')):+ continue+ seen_paths.add(dir_entry.path)+ template_id = dir_entry.name.rsplit('.', 1)[0]+ if template_id in seen_ids:+ continue+ try:+ # DirEntry.stat() reuses the data the directory scan+ # already returned where the platform provides it, so this+ # is cheaper than a separate os.stat per file.+ info = dir_entry.stat()+ except OSError:+ continue++ entry = cls._local_entry(dir_entry.path, template_id,+ (info.st_mtime_ns, info.st_size))+ if entry is not None:+ # A file that fails to parse deliberately does NOT claim+ # the id, so a valid file of the same name in the fallback+ # directory can still supply it.+ seen_ids.add(template_id)+ templates.append(entry)++ cls._prune_local_cache(seen_paths)return templates+ @classmethod+ def invalidate_local_cache(cls) -> None:+ """Drop the parsed-template cache.++ Ordinary writes do not need this -- the cache is keyed on+ (mtime_ns, size), so syncing or editing a template invalidates exactly+ that file on the next listing. Provided for tests and any out-of-band+ mutation that could defeat mtime comparison."""+ with cls._local_cache_lock:+ cls._local_cache = {}++ @classmethod+ def _prune_local_cache(cls, seen_paths: set) -> None:+ """Forget files that are no longer on disk, so a long-running panel+ that churns templates does not grow the cache without bound."""+ with cls._local_cache_lock:+ stale = [p for p in cls._local_cache if p not in seen_paths]+ for path in stale:+ del cls._local_cache[path]++ @classmethod+ def _local_entry(cls, filepath: str, template_id: str,+ stat_key: tuple) -> Optional[Dict]:+ """Catalog projection for one bundled template, memoized on ``stat_key``.++ Returns None when the file does not parse -- and caches that negative+ result too, so one malformed YAML does not cost a parse on every+ render. Returns a deep copy so a caller mutating the result (or its+ nested ``repo``/``engine``/``categories``) cannot corrupt the cache."""+ with cls._local_cache_lock:+ cached = cls._local_cache.get(filepath)+ if cached is not None and cached['key'] == stat_key:+ return copy.deepcopy(cached['entry'])++ entry = cls._parse_local_entry(filepath, template_id)++ with cls._local_cache_lock:+ cls._local_cache[filepath] = {'key': stat_key, 'entry': entry}+ return copy.deepcopy(entry)++ @classmethod+ def _parse_local_entry(cls, filepath: str, template_id: str) -> Optional[Dict]:+ """Parse one template file into its catalog projection, or None."""+ result = cls.parse_template(filepath)+ if not result.get('success'):+ return None++ template = result['template']+ kind = template.get('kind', 'compose')+ entry = {+ 'id': template_id,+ 'name': template.get('name'),+ 'version': template.get('version'),+ 'description': template.get('description'),+ 'icon': template.get('icon'),+ 'categories': template.get('categories', []),+ 'kind': kind,+ 'source': 'local',+ 'filepath': filepath+ }+ # Repo templates surface a minimal repo summary so the catalog grid+ # can badge them and link to the wizard without a second fetch.+ if kind == 'repo' and isinstance(template.get('repo'), dict):+ repo = template['repo']+ entry['repo'] = {+ 'url': repo.get('url'),+ 'branch': repo.get('branch', 'main'),+ }+ # A database engine declares itself with a top-level `engine:` block.+ # This listing is a fixed projection, so without carrying the block+ # through, the Databases catalog could never see an engine without+ # re-parsing every YAML file.+ engine = cls.engine_metadata(template)+ if engine:+ entry['engine'] = engine+ return entry+@classmethoddef list_engine_templates(cls) -> List[Dict]:"""Every available template carrying an ``engine:`` block.diff --git a/backend/tests/test_template_registry_cache.py b/backend/tests/test_template_registry_cache.pyindex 56fa1533..fa0e21a4 100644--- a/backend/tests/test_template_registry_cache.py+++ b/backend/tests/test_template_registry_cache.py@@ -10,6 +10,8 @@ Two bugs motivated these, both only latent while ``DEFAULT_REPOS`` pointed at atimeout and no memoization, so one unreachable repo stalled the Templatespage on every render."""+import os+import pytestfrom app.services.template_service import TemplateService@@ -36,10 +38,12 @@ class _FakeResponse:@pytest.fixture(autouse=True)def _clean_cache():- """The cache is class-level state; never leak it between tests."""+ """Both caches are class-level state; never leak them between tests."""TemplateService.invalidate_remote_cache()+ TemplateService.invalidate_local_cache()yieldTemplateService.invalidate_remote_cache()+ TemplateService.invalidate_local_cache()@pytest.fixture@@ -204,3 +208,159 @@ def test_cached_entries_are_copies(monkeypatch, one_repo):second = TemplateService.fetch_remote_templates(REPO_URL)assert second[0]["name"] == "remote-a"+++# ---------------------------------------------------------------------------+# Local parse cache+#+# With the remote fetch memoized, re-parsing every bundled YAML on every call+# became the whole cost of a Templates page load (118 files, ~0.4s).+# ---------------------------------------------------------------------------+MINIMAL = """\+name: {name}+version: "1.0"+description: A fixture template+categories:+ - test+compose:+ services:+ app:+ image: nginx:alpine+"""+++def template_dirs(monkeypatch, tmp_path):+ """Point both template directories at a temp tree.++ ``TEMPLATES_DIR`` is the primary (synced/user) directory and+ ``LOCAL_TEMPLATES_DIR`` the bundled fallback -- the same precedence+ ``list_local_templates`` walks in production."""+ primary = tmp_path / "primary"+ fallback = tmp_path / "fallback"+ primary.mkdir()+ fallback.mkdir()+ monkeypatch.setattr(TemplateService, "TEMPLATES_DIR", str(primary))+ monkeypatch.setattr(TemplateService, "LOCAL_TEMPLATES_DIR", str(fallback))+ TemplateService.invalidate_local_cache()+ return primary, fallback+++def parse_calls(monkeypatch):+ """Record every real parse so the tests assert on work done, not wall time."""+ calls = []+ original = TemplateService.parse_template++ def counting(template_path):+ calls.append(template_path)+ return original(template_path)++ monkeypatch.setattr(TemplateService, "parse_template", staticmethod(counting))+ return calls+++def test_repeat_listing_does_not_reparse(template_dirs, parse_calls):+ """The bug: every catalog render re-read and re-parsed the whole bundle."""+ primary, _ = template_dirs+ for i in range(3):+ (primary / f"t{i}.yaml").write_text(MINIMAL.format(name=f"T{i}"))++ assert len(TemplateService.list_local_templates()) == 3+ assert len(parse_calls) == 3++ TemplateService.list_local_templates()+ TemplateService.list_local_templates()++ assert len(parse_calls) == 3, "repeat listing re-parsed unchanged files"+++def test_editing_one_template_reparses_only_that_file(template_dirs, parse_calls):+ """Per-file keying: a sync that rewrites one template costs one parse."""+ primary, _ = template_dirs+ for i in range(3):+ (primary / f"t{i}.yaml").write_text(MINIMAL.format(name=f"T{i}"))+ TemplateService.list_local_templates()+ parse_calls.clear()++ # Changes both mtime and size, so this holds even on a filesystem with+ # coarse mtime granularity.+ (primary / "t1.yaml").write_text(MINIMAL.format(name="Renamed") + "\n# edited\n")++ listed = {t["id"]: t for t in TemplateService.list_local_templates()}++ assert [os.path.basename(p) for p in parse_calls] == ["t1.yaml"]+ assert listed["t1"]["name"] == "Renamed"+ assert listed["t0"]["name"] == "T0"+++def test_deleted_template_disappears_and_is_pruned(template_dirs):+ """A removed file leaves the catalog and does not linger in the cache."""+ primary, _ = template_dirs+ path = primary / "gone.yaml"+ path.write_text(MINIMAL.format(name="Gone"))++ assert [t["id"] for t in TemplateService.list_local_templates()] == ["gone"]+ assert str(path) in TemplateService._local_cache++ path.unlink()++ assert TemplateService.list_local_templates() == []+ assert str(path) not in TemplateService._local_cache+++def test_unparseable_template_is_not_retried_until_it_changes(template_dirs, parse_calls):+ """One malformed YAML must not cost a parse on every single render."""+ primary, _ = template_dirs+ broken = primary / "broken.yaml"+ broken.write_text("name: [unclosed\n")++ assert TemplateService.list_local_templates() == []+ assert len(parse_calls) == 1++ TemplateService.list_local_templates()+ TemplateService.list_local_templates()+ assert len(parse_calls) == 1, "malformed template re-parsed on every listing"++ broken.write_text(MINIMAL.format(name="Fixed"))++ assert [t["id"] for t in TemplateService.list_local_templates()] == ["broken"]+ assert len(parse_calls) == 2+++def test_unparseable_primary_does_not_block_the_fallback(template_dirs):+ """A file that fails to parse must not claim its id, so a valid file of the+ same name in the fallback directory still supplies it."""+ primary, fallback = template_dirs+ (primary / "dup.yaml").write_text("name: [unclosed\n")+ (fallback / "dup.yaml").write_text(MINIMAL.format(name="FromFallback"))++ assert [t["name"] for t in TemplateService.list_local_templates()] == ["FromFallback"]+++def test_primary_directory_wins_over_fallback(template_dirs):+ """Precedence is unchanged by caching: the primary directory shadows the bundle."""+ primary, fallback = template_dirs+ (primary / "dup.yaml").write_text(MINIMAL.format(name="FromPrimary"))+ (fallback / "dup.yaml").write_text(MINIMAL.format(name="FromFallback"))++ listed = TemplateService.list_local_templates()++ assert len(listed) == 1+ assert listed[0]["name"] == "FromPrimary"+++def test_local_entries_are_copies(template_dirs):+ """A caller mutating an entry -- or its nested categories list -- must not+ corrupt the cache."""+ primary, _ = template_dirs+ (primary / "c.yaml").write_text(MINIMAL.format(name="C"))++ first = TemplateService.list_local_templates()[0]+ first["categories"].append("mutated")+ first["name"] = "mutated"++ second = TemplateService.list_local_templates()[0]++ assert second["categories"] == ["test"]+ assert second["name"] == "C"