Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.
Python agent-orchestrationai-agentai-coding-assistantclaude-codecoding-agentcomputer-usedesktop-applocal-firstmcppersonal-jarvisvoice-assistantvoice-coding
fix(ci): recognize credential-free public plugin manifests
rubenluetke10-beepa7c28e548677
diff --git a/scripts/ci/privacy_pre_push.py b/scripts/ci/privacy_pre_push.pyindex 982cc95068..828a223b16 100644--- a/scripts/ci/privacy_pre_push.py+++ b/scripts/ci/privacy_pre_push.py@@ -42,15 +42,29 @@ stdlib only (cloud-first €5-VPS doctrine: must run on a bare python:3.11-slim)from __future__ import annotationsimport importlib.util+import jsonimport reimport subprocessimport sysfrom pathlib import Path+from urllib.parse import urlsplit# A SHA placeholder of all zeros means "the remote has nothing yet" (a brand-new# branch) — see the git pre-push hook stdin contract._ALL_ZERO_RE = re.compile(r"^0+$")+# Reviewed public endpoints, deliberately independent of mutable catalog data.+# A new HTTP manifest requires reviewing its exact endpoint here: opaque URL+# paths can carry capability credentials even when ordinary secret regexes miss them.+_PUBLIC_CATALOG_MCP_URLS = {+ "agentmail": "https://mcp.agentmail.to/mcp",+ "apollo": "https://mcp.apollo.io/mcp",+ "aws": "https://aws-mcp.eu-central-1.api.aws/mcp",+ "github": "https://api.githubcopilot.com/mcp/",+ "granola": "https://mcp.granola.ai/mcp",+ "salesforce": "https://api.salesforce.com/platform/mcp/v1/platform/sobject-all",+}+# --------------------------------------------------------------------------- ## Pure functions (unit-tested; no subprocess, no filesystem, no network)@@ -93,9 +107,7 @@ def offenders_from_log(log_text: str, private_emails: set[str]) -> list[dict]:if author_email.lower() in private_emails:offenders.append({"sha": sha, "email": author_email, "role": "author"})if committer_email.lower() in private_emails:- offenders.append(- {"sha": sha, "email": committer_email, "role": "committer"}- )+ offenders.append({"sha": sha, "email": committer_email, "role": "committer"})return offenders@@ -104,6 +116,76 @@ def forbidden_file(name: str, forbidden: set[str]) -> bool:return name in forbidden+def is_public_catalog_mcp(rel: str, text: str) -> bool:+ """Recognize only credential-free catalog manifests, using a strict schema subset.++ This exempts the filename only; callers must still scan the original text+ for secret patterns. Personal MCP configuration is never exempted.+ """+ match = re.fullmatch(r"jarvis/marketplace/plugins/([a-z0-9]+(?:-[a-z0-9]+)*)/mcp\.json", rel)+ if match is None:+ return False+ plugin_id = match.group(1)++ def unique_object(pairs: list[tuple[str, object]]) -> dict:+ result = {}+ for key, value in pairs:+ if key in result:+ raise ValueError("Duplicate manifest key")+ result[key] = value+ return result++ try:+ manifest = json.loads(text, object_pairs_hook=unique_object)+ if not isinstance(manifest, dict) or set(manifest) != {"$schema", "mcpServers"}:+ return False+ if manifest["$schema"] != "https://agent-plugins.org/schemas/1.0.0/mcp.schema.json":+ return False+ servers = manifest["mcpServers"]+ if not isinstance(servers, dict) or len(servers) != 1:+ return False+ name, server = next(iter(servers.items()))+ if name.replace("_", "-") != plugin_id or not isinstance(server, dict):+ return False+ if server.get("type") == "streamable-http":+ if set(server) != {"type", "url"} or not isinstance(server["url"], str):+ return False+ url = server["url"]+ if url != _PUBLIC_CATALOG_MCP_URLS.get(plugin_id):+ return False+ if any(char.isspace() or ord(char) < 32 for char in url):+ return False+ parts = urlsplit(url)+ return bool(+ parts.scheme == "https"+ and parts.hostname+ and parts.username is None+ and parts.password is None+ and "?" not in url+ and "#" not in url+ and "\\" not in url+ and parts.port in (None, 443)+ )+ if set(server) != {"type", "command", "args"}:+ return False+ if server["type"] != "stdio" or server["command"] != "python":+ return False+ expected = ["-m", "jarvis.plugins.tool.connected_server", plugin_id.replace("-", "_")]+ return server["args"] == expected or (+ plugin_id == "amd-gpu" and server["args"] == ["-m", "jarvis.marketplace.amd_mcp"]+ )+ except (ValueError, TypeError, RecursionError):+ # Malformed or ambiguous JSON/URLs remain forbidden, without echoing content.+ return False+++def forbidden_pushed_file(rel: str, text: str, forbidden: set[str]) -> bool:+ """Apply the basename denylist with the narrow public catalog exception."""+ return forbidden_file(rel.rsplit("/", 1)[-1], forbidden) and not (+ is_public_catalog_mcp(rel, text)+ )++def scan_text_for_secrets(rel: str,text: str,@@ -163,8 +245,7 @@ def load_private_emails() -> set[str]:"""try:out = subprocess.run(- ["git", "-C", str(_repo_root()),- "config", "--get-all", "privacy.private-email"],+ ["git", "-C", str(_repo_root()), "config", "--get-all", "privacy.private-email"],check=False,capture_output=True,).stdout.decode("utf-8", "surrogateescape")@@ -210,21 +291,15 @@ def load_secret_scanner(repo_root: Path) -> tuple[dict | None, set, set]:and layer 3 (CI) is the backstop."""try:- gate_dir = (- repo_root / "scripts" / "ci" / "privacy_gate"- )+ gate_dir = repo_root / "scripts" / "ci" / "privacy_gate"script = gate_dir / "scripts" / "strip_and_scan.py"- spec = importlib.util.spec_from_file_location(- "ship_strip_and_scan", script- )+ spec = importlib.util.spec_from_file_location("ship_strip_and_scan", script)if spec is None or spec.loader is None:return None, set(), set()module = importlib.util.module_from_spec(spec)spec.loader.exec_module(module)- compiled = {- name: re.compile(p) for name, p in module.SECRET_PATTERNS.items()- }+ compiled = {name: re.compile(p) for name, p in module.SECRET_PATTERNS.items()}forbidden = set(module.FORBIDDEN_BASENAMES)allowlist = module._load_allowlist(gate_dir)return compiled, forbidden, allowlist@@ -235,10 +310,7 @@ def load_secret_scanner(repo_root: Path) -> tuple[dict | None, set, set]:file=sys.stderr,)try:- compiled = {- name: re.compile(p)- for name, p in _FALLBACK_SECRET_PATTERNS.items()- }+ compiled = {name: re.compile(p) for name, p in _FALLBACK_SECRET_PATTERNS.items()}except re.error as bad: # pragma: no cover - only a bug in this fileprint(f"privacy-pre-push: WARNING built-in secret patterns are broken "@@ -249,9 +321,7 @@ def load_secret_scanner(repo_root: Path) -> tuple[dict | None, set, set]:return compiled, set(_FALLBACK_FORBIDDEN_BASENAMES), set()-def resolve_base(- remote_sha: str, local_sha: str, fallback_base: str = "origin/main"-) -> str:+def resolve_base(remote_sha: str, local_sha: str, fallback_base: str = "origin/main") -> str:"""Pick the diff base: the remote sha, or ``fallback_base`` for a new ref.`local_sha` is accepted for symmetry / future use; an all-zero remote sha@@ -431,17 +501,13 @@ def main(argv: list[str], stdin) -> int:files = []for rel, text in files:- basename = rel.rsplit("/", 1)[-1]- if forbidden_file(basename, forbidden):+ if forbidden_pushed_file(rel, text, forbidden):blocked = Trueprint(- f"\nPUSH BLOCKED: forbidden secret file in push: "- f"{rel}",+ f"\nPUSH BLOCKED: forbidden secret file in push: {rel}",file=sys.stderr,)- secrets = scan_text_for_secrets(- rel, text, compiled, allowlist- )+ secrets = scan_text_for_secrets(rel, text, compiled, allowlist)if secrets:blocked = Truefor s in secrets:diff --git a/scripts/ci/privacy_scan_ci.py b/scripts/ci/privacy_scan_ci.pyindex 167839dbcd..64d67d666b 100644--- a/scripts/ci/privacy_scan_ci.py+++ b/scripts/ci/privacy_scan_ci.py@@ -19,6 +19,7 @@ cannot itself leak a credential into a public build log.Usage:python scripts/ci/privacy_scan_ci.py --base <ref> [--head <ref>]"""+from __future__ import annotationsimport argparse@@ -63,22 +64,19 @@ def main(argv: list[str]) -> int:files = gate.pushed_text_files(args.base, args.head)except Exception as exc:print(- f"privacy-ci: ERROR could not diff {args.base}..{args.head} "- f"({exc!r}).",+ f"privacy-ci: ERROR could not diff {args.base}..{args.head} ({exc!r}).",file=sys.stderr,)return 1for rel, text in files:- basename = rel.rsplit("/", 1)[-1]- if gate.forbidden_file(basename, forbidden):+ if gate.forbidden_pushed_file(rel, text, forbidden):blocked = Trueprint(f"PRIVACY-CI BLOCK: forbidden secret file: {rel}", file=sys.stderr)for finding in gate.scan_text_for_secrets(rel, text, compiled, allowlist):blocked = True# Never echo the secret value into the (potentially public) log.print(- f"PRIVACY-CI BLOCK: secret ({finding['pattern']}) in "- f"{finding['path']}",+ f"PRIVACY-CI BLOCK: secret ({finding['pattern']}) in {finding['path']}",file=sys.stderr,)@@ -86,21 +84,17 @@ def main(argv: list[str]) -> int:private_emails = _private_emails_from_env()if private_emails:try:- log = gate._git(- "log", "--format=%H%x09%ae%x09%ce", f"{args.base}..{args.head}"- )+ log = gate._git("log", "--format=%H%x09%ae%x09%ce", f"{args.base}..{args.head}")except Exception as exc:print(- f"privacy-ci: ERROR could not read git log "- f"({exc!r}).",+ f"privacy-ci: ERROR could not read git log ({exc!r}).",file=sys.stderr,)return 1for off in gate.offenders_from_log(log, private_emails):blocked = Trueprint(- f"PRIVACY-CI BLOCK: private maintainer email on {off['sha'][:12]} "- f"({off['role']})",+ f"PRIVACY-CI BLOCK: private maintainer email on {off['sha'][:12]} ({off['role']})",file=sys.stderr,)else:@@ -116,10 +110,7 @@ def main(argv: list[str]) -> int:file=sys.stderr,)return 1- print(- "privacy-ci: OK — no secrets / forbidden files / private-email commits "- "in the diff."- )+ print("privacy-ci: OK — no secrets / forbidden files / private-email commits in the diff.")return 0diff --git a/tests/unit/ci/test_public_mcp_privacy.py b/tests/unit/ci/test_public_mcp_privacy.pynew file mode 100644index 0000000000..46f82f8cc1--- /dev/null+++ b/tests/unit/ci/test_public_mcp_privacy.py@@ -0,0 +1,164 @@+"""Public catalog filenames never exempt credentials from either privacy gate."""++from __future__ import annotations++import io+import json+import re+import sys+from pathlib import Path++import pytest++ROOT = Path(__file__).resolve().parents[3]+sys.path.insert(0, str(ROOT / "scripts" / "ci"))+import privacy_pre_push as gate # noqa: E402+import privacy_scan_ci as ci # noqa: E402++PATH = "jarvis/marketplace/plugins/github/mcp.json"+HTTP = {"type": "streamable-http", "url": "https://api.githubcopilot.com/mcp/"}+++def manifest(server=None, name="github"):+ return json.dumps(+ {+ "$schema": "https://agent-plugins.org/schemas/1.0.0/mcp.schema.json",+ "mcpServers": {name: HTTP if server is None else server},+ }+ )+++def test_existing_catalog_manifests_are_public():+ paths = list((ROOT / "jarvis/marketplace/plugins").glob("*/mcp.json"))+ assert paths+ for path in paths:+ assert gate.is_public_catalog_mcp(path.relative_to(ROOT).as_posix(), path.read_text()), path+++ "path",+ [+ "mcp.json",+ "private/mcp.json",+ "jarvis/marketplace/plugins/example/../mcp.json",+ "./jarvis/marketplace/plugins/example/mcp.json",+ PATH.replace("/", "\\"),+ "/" + PATH,+ "jarvis/marketplace/plugins/example/nested/mcp.json",+ ],+)+def test_personal_and_ambiguous_paths_stay_forbidden(path):+ assert not gate.is_public_catalog_mcp(path, manifest())++[email protected]("extra", ["env", "headers", "token", "authorization", "extra"])+def test_extra_server_fields_are_forbidden(extra):+ assert gate.forbidden_pushed_file(PATH, manifest({**HTTP, extra: "private"}), {"mcp.json"})+++ "url",+ [+ "http://example.invalid/mcp",+ "https://user:[email protected]/mcp",+ "https://example.invalid/mcp?token=private",+ "https://example.invalid/mcp#private",+ "https://example.invalid/mcp?",+ "https://example.invalid/mcp#",+ "https://example.invalid/\nprivate",+ "https://example.invalid:bad/mcp",+ "https:///mcp",+ "https://example.invalid\\private",+ ],+)+def test_unsafe_urls_are_forbidden(url):+ assert not gate.is_public_catalog_mcp(PATH, manifest({**HTTP, "url": url}))+++ "url",+ [+ "https://unknown.invalid/mcp",+ "https://api.githubcopilot.com/mcp/opaque-capability-token",+ "https://api.githubcopilot.com/another-endpoint",+ "https://mcp.agentmail.to/mcp",+ ],+)+def test_unreviewed_endpoint_or_capability_path_is_forbidden(url):+ assert gate.forbidden_pushed_file(PATH, manifest({**HTTP, "url": url}), {"mcp.json"})+++def test_reviewed_endpoint_cannot_be_reused_by_unknown_plugin():+ assert not gate.is_public_catalog_mcp(+ PATH.replace("github", "unknown"), manifest(name="unknown")+ )+++ "text",+ [+ "null",+ "[]",+ "{",+ manifest(name="other"),+ manifest().replace('"mcpServers":', '"token": "private", "mcpServers":'),+ manifest().replace('"mcpServers":', '"mcpServers": {}, "mcpServers":'),+ manifest().replace("1.0.0/mcp.schema.json", "other.json"),+ ],+)+def test_invalid_structure_is_forbidden(text):+ assert not gate.is_public_catalog_mcp(PATH, text)+++ "args",+ [+ ["-c", "print('private')"],+ ["-m", "other"],+ ["-m", "jarvis.plugins.tool.connected_server", "other"],+ ["-m", "jarvis.marketplace.amd_mcp"],+ ["-m", "jarvis.plugins.tool.connected_server", "example", "private"],+ ],+)+def test_arbitrary_stdio_commands_are_forbidden(args):+ assert not gate.is_public_catalog_mcp(+ PATH, manifest({"type": "stdio", "command": "python", "args": args})+ )+++def test_normalized_stdio_identity_is_allowed():+ server = {+ "type": "stdio",+ "command": "python",+ "args": ["-m", "jarvis.plugins.tool.connected_server", "some_plugin"],+ }+ assert gate.is_public_catalog_mcp(+ PATH.replace("github", "some-plugin"), manifest(server, "some_plugin")+ )++[email protected]("runner", ["local", "ci"])[email protected]("secret", [False, True])+def test_both_gates_scan_public_manifest_content(monkeypatch, runner, secret):+ text = manifest()+ assert gate.is_public_catalog_mcp(PATH, text)+ # A synthetic detector matches safe public text to prove that filename+ # exemption never skips content scanning in either caller.+ patterns = {"test_detector": re.compile("githubcopilot")} if secret else {}+ monkeypatch.setattr(+ gate,+ "load_secret_scanner",+ lambda root: (patterns, {"mcp.json"}, set()),+ )+ monkeypatch.setattr(gate, "pushed_text_files", lambda *args: [(PATH, text)])+ monkeypatch.setattr(gate, "load_private_emails", lambda: set())+ monkeypatch.setattr(gate, "_git", lambda *args: "")+ monkeypatch.delenv("PRIVACY_PRIVATE_EMAILS", raising=False)+ if runner == "ci":+ result = ci.main(["gate", "--base", "HEAD~1"])+ else:+ result = gate.main(+ ["gate", "origin", "url"],+ io.StringIO(f"refs/heads/main {'1' * 40} refs/heads/main {'2' * 40}\n"),+ )+ assert result == int(secret)