The agent that grows with you
Python aiai-agentai-agentsanthropicchatgptclaudeclaude-codecodexhermeshermes-agentllmnous-researchopenai
#!/usr/bin/env python3
"""
Skills Hub — hub state management and the public facade for the source adapters.
Library module (not an agent tool). Owns the hub paths, guarded HTTP, index
cache, lock file, taps and audit log. Install/uninstall/update live in
``skills_hub_install``, the index fetch/source router/search in
``skills_hub_search``, and the adapters in the other ``tools.skills_hub_*``
siblings; import each name from its defining module.
Used by hermes_cli/skills_hub.py for CLI commands and the /skills slash command.
"""
import json
import logging
import threading
import time
from contextvars import ContextVar
from contextlib import ExitStack, contextmanager
from datetime import datetime, timezone, UTC
from pathlib import Path
from typing import Any, Callable, Dict, Iterator, List, Optional
from urllib.parse import urljoin
import httpx
from hermes_constants import get_hermes_home
from tools.url_safety import is_safe_url
from tools.url_safety import create_ssrf_safe_client
from tools.website_policy import check_website_access
from tools.skills_hub_models import _normalize_lock_install_path, _validate_skill_name
from utils import atomic_write_text
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Paths
# ---------------------------------------------------------------------------
# Resolved per-call (not frozen at import) so the profile override is honored;
# import-time constants leaked across profiles in single-process multi-profile
# runtimes. The path names (SKILLS_DIR, ...) resolve live via __getattr__ below.
INDEX_CACHE_TTL = 3600 # 1 hour
def _path_resolver(name: str, parent: str, leaf: str):
"""Resolver for hub path ``<parent>/<leaf>``: a test-injected real module attribute
(patch.object/monkeypatch on SKILLS_DIR etc.) wins over live resolution."""
def resolve() -> Path:
forced = globals().get(name)
return Path(forced) if forced is not None else _DYNAMIC_PATH_RESOLVERS[parent]() / leaf
resolve.__name__ = f"_{name.lower()}"
return resolve
def _hermes_home() -> Path:
return get_hermes_home()
_skills_dir = _path_resolver("SKILLS_DIR", "HERMES_HOME", "skills")
_hub_dir = _path_resolver("HUB_DIR", "SKILLS_DIR", ".hub")
_lock_file = _path_resolver("LOCK_FILE", "HUB_DIR", "lock.json")
_quarantine_dir = _path_resolver("QUARANTINE_DIR", "HUB_DIR", "quarantine")
_audit_log = _path_resolver("AUDIT_LOG", "HUB_DIR", "audit.log")
_taps_file = _path_resolver("TAPS_FILE", "HUB_DIR", "taps.json")
_index_cache_dir = _path_resolver("INDEX_CACHE_DIR", "HUB_DIR", "index-cache")
_DYNAMIC_PATH_RESOLVERS = {"HERMES_HOME": _hermes_home, **{
r.__name__[1:].upper(): r
for r in (_skills_dir, _hub_dir, _lock_file, _quarantine_dir, _audit_log, _taps_file, _index_cache_dir)
}}
def __getattr__(name: str):
"""Resolve legacy path constants dynamically (PEP 562) so they reflect the
active profile override; a test's patch.object-set real attribute shadows it."""
resolver = _DYNAMIC_PATH_RESOLVERS.get(name)
if resolver is not None:
return resolver()
raise AttributeError(f"module {__name__!r} has no attribute {name!r}")
# ---------------------------------------------------------------------------
# Install-path safety + guarded HTTP
# ---------------------------------------------------------------------------
_REDIRECT_STATUS_CODES = {301, 302, 303, 307, 308}
_MAX_SKILL_FETCH_REDIRECTS = 5
# Default per-request timeout every one-shot hub GET used before pooling existed.
_DEFAULT_HTTP_TIMEOUT = 20
# An inspect resolves metadata and then the preview bundle through the same
# source adapter. Keeping this context local to that operation lets httpx reuse
# its verified connection without extending a client beyond the CLI request.
_skills_hub_http_client: ContextVar[Optional[Any]] = ContextVar(
"skills_hub_http_client", default=None
)
@contextmanager
def skills_hub_http_session() -> Iterator[None]:
"""Reuse one SSRF-safe HTTP client for a single skills-hub resolution."""
if _skills_hub_http_client.get() is not None:
yield
return
with create_ssrf_safe_client(timeout=_DEFAULT_HTTP_TIMEOUT, follow_redirects=False) as client:
token = _skills_hub_http_client.set(client)
try:
yield
finally:
_skills_hub_http_client.reset(token)
def _skills_hub_http_get(url: str, **kwargs: Any) -> httpx.Response:
"""GET through the active resolution pool, or preserve one-shot behavior."""
client = _skills_hub_http_client.get()
if client is not None:
return client.get(url, **kwargs)
return httpx.get(url, **kwargs)
def _ssrf_safe_http_get(url: str, *, timeout: int = _DEFAULT_HTTP_TIMEOUT,
headers: Optional[dict[str, str]] = None) -> httpx.Response:
"""Fetch one URL with connect-time SSRF validation and no automatic redirects."""
with skills_hub_http_session():
return _skills_hub_http_client.get().get(url, timeout=timeout, headers=headers)
def _guarded_http_get(url: str, *, timeout: int = _DEFAULT_HTTP_TIMEOUT,
headers: Optional[dict[str, str]] = None) -> Optional[httpx.Response]:
"""Fetch a URL with SSRF and redirect-target validation (each hop re-checked).
*headers* are plain request headers (no credentials) and are sent on every hop."""
from tools.url_safety import SSRFConnectionBlocked
current_url = url
for _ in range(_MAX_SKILL_FETCH_REDIRECTS + 1):
if not is_safe_url(current_url):
logger.warning("Blocked unsafe Skills Hub URL: %s", current_url)
return None
blocked = check_website_access(current_url)
if blocked:
logger.info(
"Blocked Skills Hub fetch for %s by rule %s",
blocked["host"],
blocked["rule"],
)
return None
try:
resp = _ssrf_safe_http_get(current_url, timeout=timeout, headers=headers)
except (SSRFConnectionBlocked, httpx.HTTPError) as exc:
logger.debug("Skills Hub fetch failed for %s: %s", current_url, exc)
return None
if resp.status_code in _REDIRECT_STATUS_CODES:
location = getattr(resp, "headers", {}).get("location")
if not location:
return None
current_url = urljoin(current_url, location)
continue
return resp
logger.warning("Skills Hub fetch exceeded redirect limit for %s", url)
return None
@contextmanager
def _guarded_http_stream(
url: str,
*,
params: Optional[dict[str, str]] = None,
timeout: int = _DEFAULT_HTTP_TIMEOUT,
) -> Iterator[Optional[httpx.Response]]:
"""Stream one response with bounded, policy-checked redirects."""
from tools.url_safety import SSRFConnectionBlocked, create_ssrf_safe_client
current_url = url
current_params = params
response: Optional[httpx.Response] = None
stack = ExitStack()
try:
for _ in range(_MAX_SKILL_FETCH_REDIRECTS + 1):
if not is_safe_url(current_url):
logger.warning("Blocked unsafe Skills Hub URL: %s", current_url)
response = None
break
blocked = check_website_access(current_url)
if blocked:
logger.info(
"Blocked Skills Hub fetch for %s by rule %s",
blocked["host"],
blocked["rule"],
)
response = None
break
stack.close()
stack = ExitStack()
try:
client = stack.enter_context(
create_ssrf_safe_client(timeout=timeout, follow_redirects=False)
)
response = stack.enter_context(
client.stream("GET", current_url, params=current_params)
)
except (SSRFConnectionBlocked, httpx.HTTPError) as exc:
logger.debug("Skills Hub stream failed for %s: %s", current_url, exc)
response = None
break
if response.status_code not in _REDIRECT_STATUS_CODES:
break
location = response.headers.get("location")
if not location:
response = None
break
current_url = urljoin(current_url, location)
current_params = None
else:
logger.warning("Skills Hub fetch exceeded redirect limit for %s", url)
response = None
yield response
finally:
stack.close()
# ---------------------------------------------------------------------------
# Shared index cache (used by every adapter)
# ---------------------------------------------------------------------------
def _read_json_if_fresh(path: Path, ttl: float) -> Optional[Any]:
"""Parsed JSON from ``path`` when it exists and is younger than ``ttl`` seconds."""
try:
if time.time() - path.stat().st_mtime > ttl:
return None
return json.loads(path.read_text(encoding="utf-8-sig"))
except (OSError, json.JSONDecodeError):
return None
def _read_index_cache(key: str) -> Optional[Any]:
return _read_json_if_fresh(_index_cache_dir() / f"{key}.json", INDEX_CACHE_TTL)
def _write_index_cache(key: str, data: Any) -> None:
index_cache_dir = _index_cache_dir()
index_cache_dir.mkdir(parents=True, exist_ok=True)
# Cache files hold unvetted community text (possible prompt injection);
# a .ignore keeps ripgrep and .ignore-aware tools out of the hub dir.
ignore_file = _hub_dir() / ".ignore"
if not ignore_file.exists():
try:
ignore_file.write_text("# Exclude hub internals from search tools\n*\n", encoding="utf-8")
except OSError:
pass
try:
(index_cache_dir / f"{key}.json").write_text(
json.dumps(data, ensure_ascii=False, default=str), encoding="utf-8"
)
except OSError as e:
logger.debug("Could not write cache: %s", e)
# ---------------------------------------------------------------------------
# Hub state files: lock.json, taps.json, audit.log
# ---------------------------------------------------------------------------
class _JsonStateFile:
"""A JSON file under the hub dir with a fixed empty shape (``EMPTY``, deep-copied
on every miss/corrupt read); ``DEFAULT_PATH`` is the hub path resolver."""
EMPTY: dict = {}
DEFAULT_PATH: Any = None
_LOCK_HOLDER: threading.local
def __init__(self, path: Optional[Path] = None):
self.path = path if path is not None else type(self).DEFAULT_PATH()
def _read(self) -> dict:
try:
return json.loads(self.path.read_text(encoding="utf-8-sig"))
except (json.JSONDecodeError, OSError):
return json.loads(json.dumps(self.EMPTY))
def _write(self, data: dict, **dumps_kwargs) -> None:
self.path.parent.mkdir(parents=True, exist_ok=True)
# Temp + rename: another process (CLI vs desktop install) reading mid-write would see an
# empty file, load EMPTY and save a lock that drops every other installed skill.
atomic_write_text(self.path, json.dumps(data, indent=2, **dumps_kwargs) + "\n", preserve_mode=True)
def _update(self, mutate: Callable[[dict], bool], **dumps_kwargs) -> bool:
"""Re-read, ``mutate`` and save under a cross-process lock; the file is rewritten only
when ``mutate`` returns True. The CLI and desktop rewrite the same file, and a save from a
snapshot loaded outside the lock drops whatever the other process recorded meanwhile.
An unusable lock (no flock on the filesystem, a wedged holder) degrades to an unlocked
update rather than failing the install that triggered it."""
from hermes_cli.auth import _file_lock
with ExitStack() as stack:
try:
stack.enter_context(_file_lock(
self.path.with_name(self.path.name + ".lock"), type(self)._LOCK_HOLDER, 10.0,
f"Timed out waiting for the skills hub state lock ({self.path})",
))
except (OSError, TimeoutError):
logger.warning("Updating %s without its lock", self.path, exc_info=True)
data = self._read()
changed = mutate(data)
if changed:
self._write(data, **dumps_kwargs)
return changed
class HubLockFile(_JsonStateFile):
"""skills/.hub/lock.json — provenance of installed hub skills."""
EMPTY = {"version": 1, "installed": {}}
DEFAULT_PATH = staticmethod(_lock_file)
_LOCK_HOLDER = threading.local()
def load(self) -> dict:
return self._read()
def save(self, data: dict) -> None:
self._write(data, ensure_ascii=False)
def record_install(
self,
name: str,
source: str,
identifier: str,
trust_level: str,
scan_verdict: str,
skill_hash: str,
install_path: str,
files: list[str],
metadata: Optional[dict[str, Any]] = None,
scan_provenance: Optional[dict[str, Any]] = None,
) -> None:
# Validate name and install-path SHAPE at write time: a poisoned lock
# entry is the precondition for the uninstall_skill rmtree-escape.
safe_name = _validate_skill_name(name)
safe_install_path = _normalize_lock_install_path(install_path, safe_name)
def record(data: dict) -> bool:
now = datetime.now(UTC).isoformat()
data.setdefault("installed", {})[safe_name] = {
"source": source,
"identifier": identifier,
"trust_level": trust_level,
"scan_verdict": scan_verdict,
"content_hash": skill_hash,
"install_path": safe_install_path,
"files": files,
"metadata": metadata or {},
"scan_provenance": scan_provenance or {},
"installed_at": now,
"updated_at": now,
}
return True
self._update(record, ensure_ascii=False)
def record_uninstall(self, name: str) -> None:
def forget(data: dict) -> bool:
data.setdefault("installed", {}).pop(name, None)
return True
self._update(forget, ensure_ascii=False)
def get_installed(self, name: str) -> Optional[dict]:
return self.load()["installed"].get(name)
def list_installed(self) -> list[dict]:
return [{"name": name, **entry} for name, entry in self.load()["installed"].items()]
def by_identifier(self) -> dict[str, dict]:
"""Installed entries keyed by identifier. A ClawHub row is also keyed by the
``clawhub/<slug>`` id a catalog card installs, since the lock records ``@owner/slug``.
That alias names its source, so a ClawHub install never answers for another source's
id. Exact identifiers win."""
from tools.skills_hub_clawhub import ClawHubSource
rows = [entry for entry in self.list_installed() if entry.get("identifier")]
installed = {entry["identifier"]: entry for entry in rows}
for entry in rows:
if entry.get("source") == "clawhub":
if alias := ClawHubSource.catalog_identifier(entry["identifier"]):
installed.setdefault(alias, entry)
return installed
def installed_entry(installed: dict[str, dict], identifier: str,
source: Optional[str] = None) -> Optional[dict]:
"""The ``HubLockFile.by_identifier`` entry a hub result names. Search and the index list a
ClawHub skill by its bare slug, so a ``source="clawhub"`` result also matches through its
``clawhub/<slug>`` card id. Any other source matches its exact id only."""
if identifier in installed:
return installed[identifier]
if source != "clawhub":
return None
from tools.skills_hub_clawhub import ClawHubSource
alias = ClawHubSource.catalog_identifier(identifier)
return installed.get(alias) if alias else None
class TapsManager(_JsonStateFile):
"""skills/.hub/taps.json — custom GitHub repo sources."""
EMPTY = {"taps": []}
DEFAULT_PATH = staticmethod(_taps_file)
_LOCK_HOLDER = threading.local()
def load(self) -> list[dict]:
return self._read().get("taps", [])
def save(self, taps: list[dict]) -> None:
self._write({"taps": taps})
def add(self, repo: str, path: str = "skills/") -> bool:
"""Add a tap. Returns False if already exists."""
def append(data: dict) -> bool:
taps = data.setdefault("taps", [])
if any(t["repo"] == repo for t in taps):
return False
taps.append({"repo": repo, "path": path})
return True
return self._update(append)
def remove(self, repo: str) -> bool:
"""Remove a tap by repo name. Returns False if not found."""
def drop(data: dict) -> bool:
taps = data.get("taps", [])
kept = [t for t in taps if t["repo"] != repo]
data["taps"] = kept
return len(kept) != len(taps)
return self._update(drop)
list_taps = load
def append_audit_log(action: str, skill_name: str, source: str,
trust_level: str, verdict: str, extra: str = "") -> None:
"""Append one space-separated line to the audit log (best-effort)."""
audit_log = _audit_log()
audit_log.parent.mkdir(parents=True, exist_ok=True)
timestamp = datetime.now(UTC).strftime("%Y-%m-%dT%H:%M:%SZ")
parts = [timestamp, action, skill_name, f"{source}:{trust_level}", verdict]
if extra:
parts.append(extra)
try:
with open(audit_log, "a", encoding="utf-8") as f:
f.write(" ".join(parts) + "\n")
except OSError as e:
logger.debug("Could not write audit log: %s", e)
# ---------------------------------------------------------------------------
# Hub operations (high-level)
# ---------------------------------------------------------------------------
def ensure_hub_dirs() -> None:
"""Create the .hub directory structure if it doesn't exist."""
_hub_dir().mkdir(parents=True, exist_ok=True)
_quarantine_dir().mkdir(exist_ok=True)
_index_cache_dir().mkdir(exist_ok=True)
for path, initial in (
(_lock_file(), '{"version": 1, "installed": {}}\n'),
(_audit_log(), ""),
(_taps_file(), '{"taps": []}\n'),
):
if not path.exists():
path.write_text(initial, encoding="utf-8")