ServerKit is a lightweight, modern server control panel for managing web applications, databases, and services on your VPS or dedicated server. Built with Python/Flask backend and React frontend.
Python astracloudflarecontrol-panelcyberpaneldnsfablehosting-control-panelhosting-panelkimi-k3server-managementserverkitubuntu-web-panelwebserver
fix(templates): dedupe the catalog by id and cache remote indexes
Juan Denis9d64b4ab4d48
diff --git a/backend/app/services/template_service.py b/backend/app/services/template_service.pyindex 22cb9b50..4f4eb764 100644--- a/backend/app/services/template_service.py+++ b/backend/app/services/template_service.py@@ -12,12 +12,14 @@ Supports:import osimport re+import timeimport yamlimport jsonimport shutilimport secretsimport stringimport hashlib+import threadingimport subprocessfrom datetime import datetimefrom typing import Dict, List, Optional, Any@@ -52,6 +54,25 @@ class TemplateService:}]+ # Remote index cache.+ #+ # `list_all_templates()` runs on every Templates page load and used to call+ # `fetch_remote_templates()` inline for every enabled repo, with a 30s+ # timeout and no memoization -- so a single unreachable repo stalled the+ # whole catalog on every render, and a reachable one was refetched for no+ # reason. Successes are cached for _REMOTE_TTL. Failures fall back to the+ # last good payload when there is one (a transient blip must never blank a+ # catalog the panel has already shown) and are otherwise negatively cached+ # for the shorter _REMOTE_ERROR_TTL, so a dead repo is retried soon but not+ # hammered. Same TTL + last-good shape as the serverkit.ai proxy that+ # DEFAULT_REPOS points at.+ _REMOTE_TTL_SECONDS = 300+ _REMOTE_ERROR_TTL_SECONDS = 60+ _REMOTE_TIMEOUT_SECONDS = 10+ # url -> {'templates': [...], 'expires': monotonic float, 'ok': bool}+ _remote_cache = {}+ _remote_cache_lock = threading.Lock()+# Repo URLs that never worked and should be healed on read rather than# left to rot in an operator's templates.json. `serverkit/templates` was a# guess at the org name -- the registry is `jhd3197/serverkit-templates` --@@ -218,11 +239,16 @@ class TemplateService:@classmethoddef save_config(cls, config: Dict) -> Dict:- """Save template configuration."""+ """Save template configuration.++ This is the single choke point for repo mutations (add/remove/enable),+ so the remote index cache is dropped here -- a repo added now must show+ up on the next listing, not after the TTL."""try:os.makedirs(cls.CONFIG_DIR, exist_ok=True)with open(cls.TEMPLATE_CONFIG, 'w') as f:json.dump(config, f, indent=2)+ cls.invalidate_remote_cache()return {'success': True}except Exception as e:return {'success': False, 'error': str(e)}@@ -1239,14 +1265,67 @@ class TemplateService:return {'success': False, 'error': str(e)}@classmethod- def fetch_remote_templates(cls, repo_url: str) -> List[Dict]:- """Fetch templates from a remote repository."""- templates = []+ def invalidate_remote_cache(cls, repo_url: str = None) -> None:+ """Drop cached remote indexes -- all of them, or just ``repo_url``.++ Called whenever the repo list changes so a freshly added repo appears+ immediately instead of after the TTL, and a removed one stops being+ served from cache."""+ with cls._remote_cache_lock:+ if repo_url is None:+ cls._remote_cache = {}+ else:+ cls._remote_cache.pop(repo_url, None)++ @classmethod+ def _cached_remote(cls, repo_url: str) -> Optional[List[Dict]]:+ """Unexpired cache entry for ``repo_url``, or None. Returns copies so a+ caller mutating an entry cannot corrupt the cache."""+ with cls._remote_cache_lock:+ entry = cls._remote_cache.get(repo_url)+ if entry and entry['expires'] > time.monotonic():+ return [dict(t) for t in entry['templates']]+ return None++ @classmethod+ def _last_good_remote(cls, repo_url: str) -> Optional[List[Dict]]:+ """Last payload this repo successfully served, regardless of expiry."""+ with cls._remote_cache_lock:+ entry = cls._remote_cache.get(repo_url) or {}+ last_good = entry.get('last_good')+ return [dict(t) for t in last_good] if last_good else None++ @classmethod+ def _store_remote(cls, repo_url: str, templates: List[Dict], ok: bool) -> None:+ """Cache ``templates`` for ``repo_url``. A successful payload also+ becomes the last-good; a failure carries the previous last-good+ forward, so repeated failures keep serving it instead of decaying to+ an empty catalog on the second one."""+ ttl = cls._REMOTE_TTL_SECONDS if ok else cls._REMOTE_ERROR_TTL_SECONDS+ with cls._remote_cache_lock:+ previous = cls._remote_cache.get(repo_url) or {}+ cls._remote_cache[repo_url] = {+ 'templates': [dict(t) for t in templates],+ 'expires': time.monotonic() + ttl,+ 'ok': ok,+ 'last_good': ([dict(t) for t in templates] if ok+ else previous.get('last_good')),+ }+ @classmethod+ def fetch_remote_templates(cls, repo_url: str, force: bool = False) -> List[Dict]:+ """Fetch a repository's template index, memoized (see the cache block+ near the top of the class). ``force=True`` bypasses the cache, which is+ what an explicit sync wants."""+ if not force:+ cached = cls._cached_remote(repo_url)+ if cached is not None:+ return cached++ templates = []try:- # Fetch index.json from repoindex_url = f"{repo_url}/index.json"- response = requests.get(index_url, timeout=30)+ response = requests.get(index_url, timeout=cls._REMOTE_TIMEOUT_SECONDS)response.raise_for_status()index = response.json()@@ -1257,22 +1336,47 @@ class TemplateService:except Exception as e:print(f"Failed to fetch templates from {repo_url}: {e}")+ # Keep serving the last good index rather than blanking a catalog+ # the operator has already seen, but still back off so a dead repo+ # is not retried on every single page load.+ last_good = cls._last_good_remote(repo_url)+ cls._store_remote(repo_url, last_good or [], ok=False)+ return last_good or []- return templates+ cls._store_remote(repo_url, templates, ok=True)+ return [dict(t) for t in templates]@classmethoddef list_all_templates(cls, category: str = None, search: str = None) -> List[Dict]:- """List all available templates from all sources."""- templates = []+ """List all available templates from all sources, deduped by id.++ Bundled and registry templates deliberately overlap -- the bundle is the+ offline floor and the registry is the growth path (docs/REGISTRIES.md) --+ so the same id legitimately arrives from both. Concatenating them listed+ every bundled template twice. **Local wins**, matching+ :meth:`get_template`, which already resolves local directories before+ any repo; among repos, the first enabled one to claim an id keeps it.+ """+ by_id = {}- # Local templates- templates.extend(cls.list_local_templates())+ def _claim(entry):+ """First source to claim an id keeps it."""+ template_id = entry.get('id')+ if template_id and template_id not in by_id:+ by_id[template_id] = entry- # Remote templates+ # Local templates first -- they outrank every repo.+ for entry in cls.list_local_templates():+ _claim(entry)++ # Remote templates fill in ids the bundle does not carry.config = cls.get_config()for repo in config.get('repos', []):if repo.get('enabled', True):- templates.extend(cls.fetch_remote_templates(repo['url']))+ for entry in cls.fetch_remote_templates(repo['url']):+ _claim(entry)++ templates = list(by_id.values())# Filter by categoryif category:diff --git a/backend/tests/test_template_registry_cache.py b/backend/tests/test_template_registry_cache.pynew file mode 100644index 00000000..56fa1533--- /dev/null+++ b/backend/tests/test_template_registry_cache.py@@ -0,0 +1,206 @@+"""Proving tests for the template catalog's dedupe + remote-index cache.++Two bugs motivated these, both only latent while ``DEFAULT_REPOS`` pointed at a+404:++1. ``list_all_templates()`` concatenated local + remote with no id merge, so+ every bundled template appeared TWICE the moment a real registry answered+ (measured: 106 bundled + 106 remote = 213 entries for 107 unique ids).+2. It called ``fetch_remote_templates()`` inline on EVERY listing with a 30s+ timeout and no memoization, so one unreachable repo stalled the Templates+ page on every render.+"""+import pytest++from app.services.template_service import TemplateService++REPO_URL = "https://example.test/templates"++# Ids that really are bundled in backend/templates/, so the dedupe test+# exercises a genuine local-vs-remote collision rather than a synthetic one.+BUNDLED_IDS = ["ollama-webui", "litellm"]+++class _FakeResponse:+ def __init__(self, payload, status=200):+ self._payload = payload+ self.status_code = status++ def raise_for_status(self):+ if self.status_code >= 400:+ raise RuntimeError(f"HTTP {self.status_code}")++ def json(self):+ return self._payload++[email protected](autouse=True)+def _clean_cache():+ """The cache is class-level state; never leak it between tests."""+ TemplateService.invalidate_remote_cache()+ yield+ TemplateService.invalidate_remote_cache()+++def one_repo(monkeypatch):+ """Pin the repo list so these tests never read the operator's config."""+ monkeypatch.setattr(+ TemplateService, "get_config",+ classmethod(lambda cls: {"repos": [{"name": "t", "url": REPO_URL,+ "enabled": True}]}),+ )+++def _serve(monkeypatch, payload=None, fail=False):+ """Install a fake requests.get and return its call counter."""+ calls = []++ def fake_get(url, **kwargs):+ calls.append(url)+ if fail:+ raise RuntimeError("connection refused")+ return _FakeResponse(payload or {"templates": []})++ monkeypatch.setattr("app.services.template_service.requests.get", fake_get)+ return calls+++def _index(*ids):+ return {"templates": [{"id": i, "name": i, "categories": []} for i in ids]}+++# ---------------------------------------------------------------------------+# Dedupe+# ---------------------------------------------------------------------------+def test_bundled_template_is_not_listed_twice(monkeypatch, one_repo):+ """A repo serving ids that are also bundled must not double the catalog."""+ _serve(monkeypatch, _index(*BUNDLED_IDS))++ listed = TemplateService.list_all_templates()+ ids = [t["id"] for t in listed]++ assert len(ids) == len(set(ids)), "catalog contains duplicate ids"+ for bundled in BUNDLED_IDS:+ assert ids.count(bundled) == 1+++def test_local_entry_wins_over_remote(monkeypatch, one_repo):+ """Local outranks every repo, matching get_template()'s resolution order."""+ _serve(monkeypatch, _index(*BUNDLED_IDS))++ by_id = {t["id"]: t for t in TemplateService.list_all_templates()}++ for bundled in BUNDLED_IDS:+ assert by_id[bundled]["source"] == "local"+++def test_remote_only_template_is_still_listed(monkeypatch, one_repo):+ """Dedupe must not swallow ids the bundle does not carry."""+ _serve(monkeypatch, _index("not-a-bundled-template"))++ by_id = {t["id"]: t for t in TemplateService.list_all_templates()}++ assert by_id["not-a-bundled-template"]["source"] == "remote"+++def test_first_enabled_repo_wins_among_remotes(monkeypatch):+ """Two repos claiming one id: the earlier repo keeps it."""+ monkeypatch.setattr(+ TemplateService, "get_config",+ classmethod(lambda cls: {"repos": [+ {"name": "a", "url": "https://a.test/t", "enabled": True},+ {"name": "b", "url": "https://b.test/t", "enabled": True},+ ]}),+ )++ def fake_get(url, **kwargs):+ return _FakeResponse(_index("shared-id"))++ monkeypatch.setattr("app.services.template_service.requests.get", fake_get)++ by_id = {t["id"]: t for t in TemplateService.list_all_templates()}+ assert by_id["shared-id"]["repo_url"] == "https://a.test/t"+++# ---------------------------------------------------------------------------+# Caching+# ---------------------------------------------------------------------------+def test_remote_index_fetched_once_across_listings(monkeypatch, one_repo):+ """Rendering the catalog twice must not hit the network twice."""+ calls = _serve(monkeypatch, _index("remote-a"))++ TemplateService.list_all_templates()+ TemplateService.list_all_templates()+ TemplateService.list_all_templates()++ assert len(calls) == 1+++def test_dead_repo_is_not_refetched_on_every_listing(monkeypatch, one_repo):+ """The original bug: a 404 repo cost a request (and a timeout) per render."""+ calls = _serve(monkeypatch, fail=True)++ for _ in range(5):+ TemplateService.list_all_templates()++ assert len(calls) == 1+++def test_failure_serves_last_good_index_repeatedly(monkeypatch, one_repo):+ """A transient failure must not blank a catalog the panel already showed --+ and the SECOND failure must still serve it, not decay to empty."""+ monkeypatch.setattr(TemplateService, "_REMOTE_TTL_SECONDS", 0)+ monkeypatch.setattr(TemplateService, "_REMOTE_ERROR_TTL_SECONDS", 0)++ _serve(monkeypatch, _index("remote-a"))+ assert TemplateService.fetch_remote_templates(REPO_URL)[0]["id"] == "remote-a"++ _serve(monkeypatch, fail=True)+ for attempt in range(3):+ served = TemplateService.fetch_remote_templates(REPO_URL)+ assert [t["id"] for t in served] == ["remote-a"], f"lost on attempt {attempt}"+++def test_failure_with_no_last_good_returns_empty(monkeypatch, one_repo):+ """No prior success means an honest empty list, not an exception."""+ _serve(monkeypatch, fail=True)+ assert TemplateService.fetch_remote_templates(REPO_URL) == []+++def test_force_bypasses_the_cache(monkeypatch, one_repo):+ """An explicit sync must see current data, not a cached index."""+ calls = _serve(monkeypatch, _index("remote-a"))++ TemplateService.fetch_remote_templates(REPO_URL)+ TemplateService.fetch_remote_templates(REPO_URL, force=True)++ assert len(calls) == 2+++def test_saving_config_invalidates_the_cache(monkeypatch, one_repo, tmp_path):+ """Adding or removing a repo must take effect on the next listing, not+ after the TTL. save_config is the choke point for every repo mutation."""+ calls = _serve(monkeypatch, _index("remote-a"))+ monkeypatch.setattr(TemplateService, "CONFIG_DIR", str(tmp_path))+ monkeypatch.setattr(TemplateService, "TEMPLATE_CONFIG",+ str(tmp_path / "templates.json"))++ TemplateService.list_all_templates()+ assert len(calls) == 1++ TemplateService.save_config({"repos": [], "installed": {}})++ TemplateService.list_all_templates()+ assert len(calls) == 2+++def test_cached_entries_are_copies(monkeypatch, one_repo):+ """A caller mutating a returned entry must not corrupt the cache."""+ _serve(monkeypatch, _index("remote-a"))++ first = TemplateService.fetch_remote_templates(REPO_URL)+ first[0]["name"] = "mutated"++ second = TemplateService.fetch_remote_templates(REPO_URL)+ assert second[0]["name"] == "remote-a"