Skip to content

ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.

fix(templates): dedupe the catalog by id and cache remote indexes

Juan Denis9d64b4ab4d48
diff --git a/backend/app/services/template_service.py b/backend/app/services/template_service.py
index 22cb9b50..4f4eb764 100644
--- a/backend/app/services/template_service.py
+++ b/backend/app/services/template_service.py
@@ -12,12 +12,14 @@ Supports:
import os
import re
+import time
import yaml
import json
import shutil
import secrets
import string
import hashlib
+import threading
import subprocess
from datetime import datetime
from typing import Dict, List, Optional, Any
@@ -52,6 +54,25 @@ class TemplateService:
}
]
+ # Remote index cache.
+ #
+ # `list_all_templates()` runs on every Templates page load and used to call
+ # `fetch_remote_templates()` inline for every enabled repo, with a 30s
+ # timeout and no memoization -- so a single unreachable repo stalled the
+ # whole catalog on every render, and a reachable one was refetched for no
+ # reason. Successes are cached for _REMOTE_TTL. Failures fall back to the
+ # last good payload when there is one (a transient blip must never blank a
+ # catalog the panel has already shown) and are otherwise negatively cached
+ # for the shorter _REMOTE_ERROR_TTL, so a dead repo is retried soon but not
+ # hammered. Same TTL + last-good shape as the serverkit.ai proxy that
+ # DEFAULT_REPOS points at.
+ _REMOTE_TTL_SECONDS = 300
+ _REMOTE_ERROR_TTL_SECONDS = 60
+ _REMOTE_TIMEOUT_SECONDS = 10
+ # url -> {'templates': [...], 'expires': monotonic float, 'ok': bool}
+ _remote_cache = {}
+ _remote_cache_lock = threading.Lock()
+
# Repo URLs that never worked and should be healed on read rather than
# left to rot in an operator's templates.json. `serverkit/templates` was a
# guess at the org name -- the registry is `jhd3197/serverkit-templates` --
@@ -218,11 +239,16 @@ class TemplateService:
@classmethod
def save_config(cls, config: Dict) -> Dict:
- """Save template configuration."""
+ """Save template configuration.
+
+ This is the single choke point for repo mutations (add/remove/enable),
+ so the remote index cache is dropped here -- a repo added now must show
+ up on the next listing, not after the TTL."""
try:
os.makedirs(cls.CONFIG_DIR, exist_ok=True)
with open(cls.TEMPLATE_CONFIG, 'w') as f:
json.dump(config, f, indent=2)
+ cls.invalidate_remote_cache()
return {'success': True}
except Exception as e:
return {'success': False, 'error': str(e)}
@@ -1239,14 +1265,67 @@ class TemplateService:
return {'success': False, 'error': str(e)}
@classmethod
- def fetch_remote_templates(cls, repo_url: str) -> List[Dict]:
- """Fetch templates from a remote repository."""
- templates = []
+ def invalidate_remote_cache(cls, repo_url: str = None) -> None:
+ """Drop cached remote indexes -- all of them, or just ``repo_url``.
+
+ Called whenever the repo list changes so a freshly added repo appears
+ immediately instead of after the TTL, and a removed one stops being
+ served from cache."""
+ with cls._remote_cache_lock:
+ if repo_url is None:
+ cls._remote_cache = {}
+ else:
+ cls._remote_cache.pop(repo_url, None)
+
+ @classmethod
+ def _cached_remote(cls, repo_url: str) -> Optional[List[Dict]]:
+ """Unexpired cache entry for ``repo_url``, or None. Returns copies so a
+ caller mutating an entry cannot corrupt the cache."""
+ with cls._remote_cache_lock:
+ entry = cls._remote_cache.get(repo_url)
+ if entry and entry['expires'] > time.monotonic():
+ return [dict(t) for t in entry['templates']]
+ return None
+
+ @classmethod
+ def _last_good_remote(cls, repo_url: str) -> Optional[List[Dict]]:
+ """Last payload this repo successfully served, regardless of expiry."""
+ with cls._remote_cache_lock:
+ entry = cls._remote_cache.get(repo_url) or {}
+ last_good = entry.get('last_good')
+ return [dict(t) for t in last_good] if last_good else None
+
+ @classmethod
+ def _store_remote(cls, repo_url: str, templates: List[Dict], ok: bool) -> None:
+ """Cache ``templates`` for ``repo_url``. A successful payload also
+ becomes the last-good; a failure carries the previous last-good
+ forward, so repeated failures keep serving it instead of decaying to
+ an empty catalog on the second one."""
+ ttl = cls._REMOTE_TTL_SECONDS if ok else cls._REMOTE_ERROR_TTL_SECONDS
+ with cls._remote_cache_lock:
+ previous = cls._remote_cache.get(repo_url) or {}
+ cls._remote_cache[repo_url] = {
+ 'templates': [dict(t) for t in templates],
+ 'expires': time.monotonic() + ttl,
+ 'ok': ok,
+ 'last_good': ([dict(t) for t in templates] if ok
+ else previous.get('last_good')),
+ }
+ @classmethod
+ def fetch_remote_templates(cls, repo_url: str, force: bool = False) -> List[Dict]:
+ """Fetch a repository's template index, memoized (see the cache block
+ near the top of the class). ``force=True`` bypasses the cache, which is
+ what an explicit sync wants."""
+ if not force:
+ cached = cls._cached_remote(repo_url)
+ if cached is not None:
+ return cached
+
+ templates = []
try:
- # Fetch index.json from repo
index_url = f"{repo_url}/index.json"
- response = requests.get(index_url, timeout=30)
+ response = requests.get(index_url, timeout=cls._REMOTE_TIMEOUT_SECONDS)
response.raise_for_status()
index = response.json()
@@ -1257,22 +1336,47 @@ class TemplateService:
except Exception as e:
print(f"Failed to fetch templates from {repo_url}: {e}")
+ # Keep serving the last good index rather than blanking a catalog
+ # the operator has already seen, but still back off so a dead repo
+ # is not retried on every single page load.
+ last_good = cls._last_good_remote(repo_url)
+ cls._store_remote(repo_url, last_good or [], ok=False)
+ return last_good or []
- return templates
+ cls._store_remote(repo_url, templates, ok=True)
+ return [dict(t) for t in templates]
@classmethod
def list_all_templates(cls, category: str = None, search: str = None) -> List[Dict]:
- """List all available templates from all sources."""
- templates = []
+ """List all available templates from all sources, deduped by id.
+
+ Bundled and registry templates deliberately overlap -- the bundle is the
+ offline floor and the registry is the growth path (docs/REGISTRIES.md) --
+ so the same id legitimately arrives from both. Concatenating them listed
+ every bundled template twice. **Local wins**, matching
+ :meth:`get_template`, which already resolves local directories before
+ any repo; among repos, the first enabled one to claim an id keeps it.
+ """
+ by_id = {}
- # Local templates
- templates.extend(cls.list_local_templates())
+ def _claim(entry):
+ """First source to claim an id keeps it."""
+ template_id = entry.get('id')
+ if template_id and template_id not in by_id:
+ by_id[template_id] = entry
- # Remote templates
+ # Local templates first -- they outrank every repo.
+ for entry in cls.list_local_templates():
+ _claim(entry)
+
+ # Remote templates fill in ids the bundle does not carry.
config = cls.get_config()
for repo in config.get('repos', []):
if repo.get('enabled', True):
- templates.extend(cls.fetch_remote_templates(repo['url']))
+ for entry in cls.fetch_remote_templates(repo['url']):
+ _claim(entry)
+
+ templates = list(by_id.values())
# Filter by category
if category:
diff --git a/backend/tests/test_template_registry_cache.py b/backend/tests/test_template_registry_cache.py
new file mode 100644
index 00000000..56fa1533
--- /dev/null
+++ b/backend/tests/test_template_registry_cache.py
@@ -0,0 +1,206 @@
+"""Proving tests for the template catalog's dedupe + remote-index cache.
+
+Two bugs motivated these, both only latent while ``DEFAULT_REPOS`` pointed at a
+404:
+
+1. ``list_all_templates()`` concatenated local + remote with no id merge, so
+ every bundled template appeared TWICE the moment a real registry answered
+ (measured: 106 bundled + 106 remote = 213 entries for 107 unique ids).
+2. It called ``fetch_remote_templates()`` inline on EVERY listing with a 30s
+ timeout and no memoization, so one unreachable repo stalled the Templates
+ page on every render.
+"""
+import pytest
+
+from app.services.template_service import TemplateService
+
+REPO_URL = "https://example.test/templates"
+
+# Ids that really are bundled in backend/templates/, so the dedupe test
+# exercises a genuine local-vs-remote collision rather than a synthetic one.
+BUNDLED_IDS = ["ollama-webui", "litellm"]
+
+
+class _FakeResponse:
+ def __init__(self, payload, status=200):
+ self._payload = payload
+ self.status_code = status
+
+ def raise_for_status(self):
+ if self.status_code >= 400:
+ raise RuntimeError(f"HTTP {self.status_code}")
+
+ def json(self):
+ return self._payload
+
+
[email protected](autouse=True)
+def _clean_cache():
+ """The cache is class-level state; never leak it between tests."""
+ TemplateService.invalidate_remote_cache()
+ yield
+ TemplateService.invalidate_remote_cache()
+
+
+def one_repo(monkeypatch):
+ """Pin the repo list so these tests never read the operator's config."""
+ monkeypatch.setattr(
+ TemplateService, "get_config",
+ classmethod(lambda cls: {"repos": [{"name": "t", "url": REPO_URL,
+ "enabled": True}]}),
+ )
+
+
+def _serve(monkeypatch, payload=None, fail=False):
+ """Install a fake requests.get and return its call counter."""
+ calls = []
+
+ def fake_get(url, **kwargs):
+ calls.append(url)
+ if fail:
+ raise RuntimeError("connection refused")
+ return _FakeResponse(payload or {"templates": []})
+
+ monkeypatch.setattr("app.services.template_service.requests.get", fake_get)
+ return calls
+
+
+def _index(*ids):
+ return {"templates": [{"id": i, "name": i, "categories": []} for i in ids]}
+
+
+# ---------------------------------------------------------------------------
+# Dedupe
+# ---------------------------------------------------------------------------
+def test_bundled_template_is_not_listed_twice(monkeypatch, one_repo):
+ """A repo serving ids that are also bundled must not double the catalog."""
+ _serve(monkeypatch, _index(*BUNDLED_IDS))
+
+ listed = TemplateService.list_all_templates()
+ ids = [t["id"] for t in listed]
+
+ assert len(ids) == len(set(ids)), "catalog contains duplicate ids"
+ for bundled in BUNDLED_IDS:
+ assert ids.count(bundled) == 1
+
+
+def test_local_entry_wins_over_remote(monkeypatch, one_repo):
+ """Local outranks every repo, matching get_template()'s resolution order."""
+ _serve(monkeypatch, _index(*BUNDLED_IDS))
+
+ by_id = {t["id"]: t for t in TemplateService.list_all_templates()}
+
+ for bundled in BUNDLED_IDS:
+ assert by_id[bundled]["source"] == "local"
+
+
+def test_remote_only_template_is_still_listed(monkeypatch, one_repo):
+ """Dedupe must not swallow ids the bundle does not carry."""
+ _serve(monkeypatch, _index("not-a-bundled-template"))
+
+ by_id = {t["id"]: t for t in TemplateService.list_all_templates()}
+
+ assert by_id["not-a-bundled-template"]["source"] == "remote"
+
+
+def test_first_enabled_repo_wins_among_remotes(monkeypatch):
+ """Two repos claiming one id: the earlier repo keeps it."""
+ monkeypatch.setattr(
+ TemplateService, "get_config",
+ classmethod(lambda cls: {"repos": [
+ {"name": "a", "url": "https://a.test/t", "enabled": True},
+ {"name": "b", "url": "https://b.test/t", "enabled": True},
+ ]}),
+ )
+
+ def fake_get(url, **kwargs):
+ return _FakeResponse(_index("shared-id"))
+
+ monkeypatch.setattr("app.services.template_service.requests.get", fake_get)
+
+ by_id = {t["id"]: t for t in TemplateService.list_all_templates()}
+ assert by_id["shared-id"]["repo_url"] == "https://a.test/t"
+
+
+# ---------------------------------------------------------------------------
+# Caching
+# ---------------------------------------------------------------------------
+def test_remote_index_fetched_once_across_listings(monkeypatch, one_repo):
+ """Rendering the catalog twice must not hit the network twice."""
+ calls = _serve(monkeypatch, _index("remote-a"))
+
+ TemplateService.list_all_templates()
+ TemplateService.list_all_templates()
+ TemplateService.list_all_templates()
+
+ assert len(calls) == 1
+
+
+def test_dead_repo_is_not_refetched_on_every_listing(monkeypatch, one_repo):
+ """The original bug: a 404 repo cost a request (and a timeout) per render."""
+ calls = _serve(monkeypatch, fail=True)
+
+ for _ in range(5):
+ TemplateService.list_all_templates()
+
+ assert len(calls) == 1
+
+
+def test_failure_serves_last_good_index_repeatedly(monkeypatch, one_repo):
+ """A transient failure must not blank a catalog the panel already showed --
+ and the SECOND failure must still serve it, not decay to empty."""
+ monkeypatch.setattr(TemplateService, "_REMOTE_TTL_SECONDS", 0)
+ monkeypatch.setattr(TemplateService, "_REMOTE_ERROR_TTL_SECONDS", 0)
+
+ _serve(monkeypatch, _index("remote-a"))
+ assert TemplateService.fetch_remote_templates(REPO_URL)[0]["id"] == "remote-a"
+
+ _serve(monkeypatch, fail=True)
+ for attempt in range(3):
+ served = TemplateService.fetch_remote_templates(REPO_URL)
+ assert [t["id"] for t in served] == ["remote-a"], f"lost on attempt {attempt}"
+
+
+def test_failure_with_no_last_good_returns_empty(monkeypatch, one_repo):
+ """No prior success means an honest empty list, not an exception."""
+ _serve(monkeypatch, fail=True)
+ assert TemplateService.fetch_remote_templates(REPO_URL) == []
+
+
+def test_force_bypasses_the_cache(monkeypatch, one_repo):
+ """An explicit sync must see current data, not a cached index."""
+ calls = _serve(monkeypatch, _index("remote-a"))
+
+ TemplateService.fetch_remote_templates(REPO_URL)
+ TemplateService.fetch_remote_templates(REPO_URL, force=True)
+
+ assert len(calls) == 2
+
+
+def test_saving_config_invalidates_the_cache(monkeypatch, one_repo, tmp_path):
+ """Adding or removing a repo must take effect on the next listing, not
+ after the TTL. save_config is the choke point for every repo mutation."""
+ calls = _serve(monkeypatch, _index("remote-a"))
+ monkeypatch.setattr(TemplateService, "CONFIG_DIR", str(tmp_path))
+ monkeypatch.setattr(TemplateService, "TEMPLATE_CONFIG",
+ str(tmp_path / "templates.json"))
+
+ TemplateService.list_all_templates()
+ assert len(calls) == 1
+
+ TemplateService.save_config({"repos": [], "installed": {}})
+
+ TemplateService.list_all_templates()
+ assert len(calls) == 2
+
+
+def test_cached_entries_are_copies(monkeypatch, one_repo):
+ """A caller mutating a returned entry must not corrupt the cache."""
+ _serve(monkeypatch, _index("remote-a"))
+
+ first = TemplateService.fetch_remote_templates(REPO_URL)
+ first[0]["name"] = "mutated"
+
+ second = TemplateService.fetch_remote_templates(REPO_URL)
+ assert second[0]["name"] == "remote-a"