Skip to content

Dodaios is an open-source agent OS for your desktop: it runs your coding agents, browser and apps by voice or chat, on your own machine. Formerly Personal Jarvis.

fix(ci): recognize credential-free public plugin manifests

rubenluetke10-beepa7c28e548677
diff --git a/scripts/ci/privacy_pre_push.py b/scripts/ci/privacy_pre_push.py
index 982cc95068..828a223b16 100644
--- a/scripts/ci/privacy_pre_push.py
+++ b/scripts/ci/privacy_pre_push.py
@@ -42,15 +42,29 @@ stdlib only (cloud-first €5-VPS doctrine: must run on a bare python:3.11-slim)
from __future__ import annotations
import importlib.util
+import json
import re
import subprocess
import sys
from pathlib import Path
+from urllib.parse import urlsplit
# A SHA placeholder of all zeros means "the remote has nothing yet" (a brand-new
# branch) — see the git pre-push hook stdin contract.
_ALL_ZERO_RE = re.compile(r"^0+$")
+# Reviewed public endpoints, deliberately independent of mutable catalog data.
+# A new HTTP manifest requires reviewing its exact endpoint here: opaque URL
+# paths can carry capability credentials even when ordinary secret regexes miss them.
+_PUBLIC_CATALOG_MCP_URLS = {
+ "agentmail": "https://mcp.agentmail.to/mcp",
+ "apollo": "https://mcp.apollo.io/mcp",
+ "aws": "https://aws-mcp.eu-central-1.api.aws/mcp",
+ "github": "https://api.githubcopilot.com/mcp/",
+ "granola": "https://mcp.granola.ai/mcp",
+ "salesforce": "https://api.salesforce.com/platform/mcp/v1/platform/sobject-all",
+}
+
# --------------------------------------------------------------------------- #
# Pure functions (unit-tested; no subprocess, no filesystem, no network)
@@ -93,9 +107,7 @@ def offenders_from_log(log_text: str, private_emails: set[str]) -> list[dict]:
if author_email.lower() in private_emails:
offenders.append({"sha": sha, "email": author_email, "role": "author"})
if committer_email.lower() in private_emails:
- offenders.append(
- {"sha": sha, "email": committer_email, "role": "committer"}
- )
+ offenders.append({"sha": sha, "email": committer_email, "role": "committer"})
return offenders
@@ -104,6 +116,76 @@ def forbidden_file(name: str, forbidden: set[str]) -> bool:
return name in forbidden
+def is_public_catalog_mcp(rel: str, text: str) -> bool:
+ """Recognize only credential-free catalog manifests, using a strict schema subset.
+
+ This exempts the filename only; callers must still scan the original text
+ for secret patterns. Personal MCP configuration is never exempted.
+ """
+ match = re.fullmatch(r"jarvis/marketplace/plugins/([a-z0-9]+(?:-[a-z0-9]+)*)/mcp\.json", rel)
+ if match is None:
+ return False
+ plugin_id = match.group(1)
+
+ def unique_object(pairs: list[tuple[str, object]]) -> dict:
+ result = {}
+ for key, value in pairs:
+ if key in result:
+ raise ValueError("Duplicate manifest key")
+ result[key] = value
+ return result
+
+ try:
+ manifest = json.loads(text, object_pairs_hook=unique_object)
+ if not isinstance(manifest, dict) or set(manifest) != {"$schema", "mcpServers"}:
+ return False
+ if manifest["$schema"] != "https://agent-plugins.org/schemas/1.0.0/mcp.schema.json":
+ return False
+ servers = manifest["mcpServers"]
+ if not isinstance(servers, dict) or len(servers) != 1:
+ return False
+ name, server = next(iter(servers.items()))
+ if name.replace("_", "-") != plugin_id or not isinstance(server, dict):
+ return False
+ if server.get("type") == "streamable-http":
+ if set(server) != {"type", "url"} or not isinstance(server["url"], str):
+ return False
+ url = server["url"]
+ if url != _PUBLIC_CATALOG_MCP_URLS.get(plugin_id):
+ return False
+ if any(char.isspace() or ord(char) < 32 for char in url):
+ return False
+ parts = urlsplit(url)
+ return bool(
+ parts.scheme == "https"
+ and parts.hostname
+ and parts.username is None
+ and parts.password is None
+ and "?" not in url
+ and "#" not in url
+ and "\\" not in url
+ and parts.port in (None, 443)
+ )
+ if set(server) != {"type", "command", "args"}:
+ return False
+ if server["type"] != "stdio" or server["command"] != "python":
+ return False
+ expected = ["-m", "jarvis.plugins.tool.connected_server", plugin_id.replace("-", "_")]
+ return server["args"] == expected or (
+ plugin_id == "amd-gpu" and server["args"] == ["-m", "jarvis.marketplace.amd_mcp"]
+ )
+ except (ValueError, TypeError, RecursionError):
+ # Malformed or ambiguous JSON/URLs remain forbidden, without echoing content.
+ return False
+
+
+def forbidden_pushed_file(rel: str, text: str, forbidden: set[str]) -> bool:
+ """Apply the basename denylist with the narrow public catalog exception."""
+ return forbidden_file(rel.rsplit("/", 1)[-1], forbidden) and not (
+ is_public_catalog_mcp(rel, text)
+ )
+
+
def scan_text_for_secrets(
rel: str,
text: str,
@@ -163,8 +245,7 @@ def load_private_emails() -> set[str]:
"""
try:
out = subprocess.run(
- ["git", "-C", str(_repo_root()),
- "config", "--get-all", "privacy.private-email"],
+ ["git", "-C", str(_repo_root()), "config", "--get-all", "privacy.private-email"],
check=False,
capture_output=True,
).stdout.decode("utf-8", "surrogateescape")
@@ -210,21 +291,15 @@ def load_secret_scanner(repo_root: Path) -> tuple[dict | None, set, set]:
and layer 3 (CI) is the backstop.
"""
try:
- gate_dir = (
- repo_root / "scripts" / "ci" / "privacy_gate"
- )
+ gate_dir = repo_root / "scripts" / "ci" / "privacy_gate"
script = gate_dir / "scripts" / "strip_and_scan.py"
- spec = importlib.util.spec_from_file_location(
- "ship_strip_and_scan", script
- )
+ spec = importlib.util.spec_from_file_location("ship_strip_and_scan", script)
if spec is None or spec.loader is None:
return None, set(), set()
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
- compiled = {
- name: re.compile(p) for name, p in module.SECRET_PATTERNS.items()
- }
+ compiled = {name: re.compile(p) for name, p in module.SECRET_PATTERNS.items()}
forbidden = set(module.FORBIDDEN_BASENAMES)
allowlist = module._load_allowlist(gate_dir)
return compiled, forbidden, allowlist
@@ -235,10 +310,7 @@ def load_secret_scanner(repo_root: Path) -> tuple[dict | None, set, set]:
file=sys.stderr,
)
try:
- compiled = {
- name: re.compile(p)
- for name, p in _FALLBACK_SECRET_PATTERNS.items()
- }
+ compiled = {name: re.compile(p) for name, p in _FALLBACK_SECRET_PATTERNS.items()}
except re.error as bad: # pragma: no cover - only a bug in this file
print(
f"privacy-pre-push: WARNING built-in secret patterns are broken "
@@ -249,9 +321,7 @@ def load_secret_scanner(repo_root: Path) -> tuple[dict | None, set, set]:
return compiled, set(_FALLBACK_FORBIDDEN_BASENAMES), set()
-def resolve_base(
- remote_sha: str, local_sha: str, fallback_base: str = "origin/main"
-) -> str:
+def resolve_base(remote_sha: str, local_sha: str, fallback_base: str = "origin/main") -> str:
"""Pick the diff base: the remote sha, or ``fallback_base`` for a new ref.
`local_sha` is accepted for symmetry / future use; an all-zero remote sha
@@ -431,17 +501,13 @@ def main(argv: list[str], stdin) -> int:
files = []
for rel, text in files:
- basename = rel.rsplit("/", 1)[-1]
- if forbidden_file(basename, forbidden):
+ if forbidden_pushed_file(rel, text, forbidden):
blocked = True
print(
- f"\nPUSH BLOCKED: forbidden secret file in push: "
- f"{rel}",
+ f"\nPUSH BLOCKED: forbidden secret file in push: {rel}",
file=sys.stderr,
)
- secrets = scan_text_for_secrets(
- rel, text, compiled, allowlist
- )
+ secrets = scan_text_for_secrets(rel, text, compiled, allowlist)
if secrets:
blocked = True
for s in secrets:
diff --git a/scripts/ci/privacy_scan_ci.py b/scripts/ci/privacy_scan_ci.py
index 167839dbcd..64d67d666b 100644
--- a/scripts/ci/privacy_scan_ci.py
+++ b/scripts/ci/privacy_scan_ci.py
@@ -19,6 +19,7 @@ cannot itself leak a credential into a public build log.
Usage:
python scripts/ci/privacy_scan_ci.py --base <ref> [--head <ref>]
"""
+
from __future__ import annotations
import argparse
@@ -63,22 +64,19 @@ def main(argv: list[str]) -> int:
files = gate.pushed_text_files(args.base, args.head)
except Exception as exc:
print(
- f"privacy-ci: ERROR could not diff {args.base}..{args.head} "
- f"({exc!r}).",
+ f"privacy-ci: ERROR could not diff {args.base}..{args.head} ({exc!r}).",
file=sys.stderr,
)
return 1
for rel, text in files:
- basename = rel.rsplit("/", 1)[-1]
- if gate.forbidden_file(basename, forbidden):
+ if gate.forbidden_pushed_file(rel, text, forbidden):
blocked = True
print(f"PRIVACY-CI BLOCK: forbidden secret file: {rel}", file=sys.stderr)
for finding in gate.scan_text_for_secrets(rel, text, compiled, allowlist):
blocked = True
# Never echo the secret value into the (potentially public) log.
print(
- f"PRIVACY-CI BLOCK: secret ({finding['pattern']}) in "
- f"{finding['path']}",
+ f"PRIVACY-CI BLOCK: secret ({finding['pattern']}) in {finding['path']}",
file=sys.stderr,
)
@@ -86,21 +84,17 @@ def main(argv: list[str]) -> int:
private_emails = _private_emails_from_env()
if private_emails:
try:
- log = gate._git(
- "log", "--format=%H%x09%ae%x09%ce", f"{args.base}..{args.head}"
- )
+ log = gate._git("log", "--format=%H%x09%ae%x09%ce", f"{args.base}..{args.head}")
except Exception as exc:
print(
- f"privacy-ci: ERROR could not read git log "
- f"({exc!r}).",
+ f"privacy-ci: ERROR could not read git log ({exc!r}).",
file=sys.stderr,
)
return 1
for off in gate.offenders_from_log(log, private_emails):
blocked = True
print(
- f"PRIVACY-CI BLOCK: private maintainer email on {off['sha'][:12]} "
- f"({off['role']})",
+ f"PRIVACY-CI BLOCK: private maintainer email on {off['sha'][:12]} ({off['role']})",
file=sys.stderr,
)
else:
@@ -116,10 +110,7 @@ def main(argv: list[str]) -> int:
file=sys.stderr,
)
return 1
- print(
- "privacy-ci: OK — no secrets / forbidden files / private-email commits "
- "in the diff."
- )
+ print("privacy-ci: OK — no secrets / forbidden files / private-email commits in the diff.")
return 0
diff --git a/tests/unit/ci/test_public_mcp_privacy.py b/tests/unit/ci/test_public_mcp_privacy.py
new file mode 100644
index 0000000000..46f82f8cc1
--- /dev/null
+++ b/tests/unit/ci/test_public_mcp_privacy.py
@@ -0,0 +1,164 @@
+"""Public catalog filenames never exempt credentials from either privacy gate."""
+
+from __future__ import annotations
+
+import io
+import json
+import re
+import sys
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[3]
+sys.path.insert(0, str(ROOT / "scripts" / "ci"))
+import privacy_pre_push as gate # noqa: E402
+import privacy_scan_ci as ci # noqa: E402
+
+PATH = "jarvis/marketplace/plugins/github/mcp.json"
+HTTP = {"type": "streamable-http", "url": "https://api.githubcopilot.com/mcp/"}
+
+
+def manifest(server=None, name="github"):
+ return json.dumps(
+ {
+ "$schema": "https://agent-plugins.org/schemas/1.0.0/mcp.schema.json",
+ "mcpServers": {name: HTTP if server is None else server},
+ }
+ )
+
+
+def test_existing_catalog_manifests_are_public():
+ paths = list((ROOT / "jarvis/marketplace/plugins").glob("*/mcp.json"))
+ assert paths
+ for path in paths:
+ assert gate.is_public_catalog_mcp(path.relative_to(ROOT).as_posix(), path.read_text()), path
+
+
+ "path",
+ [
+ "mcp.json",
+ "private/mcp.json",
+ "jarvis/marketplace/plugins/example/../mcp.json",
+ "./jarvis/marketplace/plugins/example/mcp.json",
+ PATH.replace("/", "\\"),
+ "/" + PATH,
+ "jarvis/marketplace/plugins/example/nested/mcp.json",
+ ],
+)
+def test_personal_and_ambiguous_paths_stay_forbidden(path):
+ assert not gate.is_public_catalog_mcp(path, manifest())
+
+
[email protected]("extra", ["env", "headers", "token", "authorization", "extra"])
+def test_extra_server_fields_are_forbidden(extra):
+ assert gate.forbidden_pushed_file(PATH, manifest({**HTTP, extra: "private"}), {"mcp.json"})
+
+
+ "url",
+ [
+ "http://example.invalid/mcp",
+ "https://user:[email protected]/mcp",
+ "https://example.invalid/mcp?token=private",
+ "https://example.invalid/mcp#private",
+ "https://example.invalid/mcp?",
+ "https://example.invalid/mcp#",
+ "https://example.invalid/\nprivate",
+ "https://example.invalid:bad/mcp",
+ "https:///mcp",
+ "https://example.invalid\\private",
+ ],
+)
+def test_unsafe_urls_are_forbidden(url):
+ assert not gate.is_public_catalog_mcp(PATH, manifest({**HTTP, "url": url}))
+
+
+ "url",
+ [
+ "https://unknown.invalid/mcp",
+ "https://api.githubcopilot.com/mcp/opaque-capability-token",
+ "https://api.githubcopilot.com/another-endpoint",
+ "https://mcp.agentmail.to/mcp",
+ ],
+)
+def test_unreviewed_endpoint_or_capability_path_is_forbidden(url):
+ assert gate.forbidden_pushed_file(PATH, manifest({**HTTP, "url": url}), {"mcp.json"})
+
+
+def test_reviewed_endpoint_cannot_be_reused_by_unknown_plugin():
+ assert not gate.is_public_catalog_mcp(
+ PATH.replace("github", "unknown"), manifest(name="unknown")
+ )
+
+
+ "text",
+ [
+ "null",
+ "[]",
+ "{",
+ manifest(name="other"),
+ manifest().replace('"mcpServers":', '"token": "private", "mcpServers":'),
+ manifest().replace('"mcpServers":', '"mcpServers": {}, "mcpServers":'),
+ manifest().replace("1.0.0/mcp.schema.json", "other.json"),
+ ],
+)
+def test_invalid_structure_is_forbidden(text):
+ assert not gate.is_public_catalog_mcp(PATH, text)
+
+
+ "args",
+ [
+ ["-c", "print('private')"],
+ ["-m", "other"],
+ ["-m", "jarvis.plugins.tool.connected_server", "other"],
+ ["-m", "jarvis.marketplace.amd_mcp"],
+ ["-m", "jarvis.plugins.tool.connected_server", "example", "private"],
+ ],
+)
+def test_arbitrary_stdio_commands_are_forbidden(args):
+ assert not gate.is_public_catalog_mcp(
+ PATH, manifest({"type": "stdio", "command": "python", "args": args})
+ )
+
+
+def test_normalized_stdio_identity_is_allowed():
+ server = {
+ "type": "stdio",
+ "command": "python",
+ "args": ["-m", "jarvis.plugins.tool.connected_server", "some_plugin"],
+ }
+ assert gate.is_public_catalog_mcp(
+ PATH.replace("github", "some-plugin"), manifest(server, "some_plugin")
+ )
+
+
[email protected]("runner", ["local", "ci"])
[email protected]("secret", [False, True])
+def test_both_gates_scan_public_manifest_content(monkeypatch, runner, secret):
+ text = manifest()
+ assert gate.is_public_catalog_mcp(PATH, text)
+ # A synthetic detector matches safe public text to prove that filename
+ # exemption never skips content scanning in either caller.
+ patterns = {"test_detector": re.compile("githubcopilot")} if secret else {}
+ monkeypatch.setattr(
+ gate,
+ "load_secret_scanner",
+ lambda root: (patterns, {"mcp.json"}, set()),
+ )
+ monkeypatch.setattr(gate, "pushed_text_files", lambda *args: [(PATH, text)])
+ monkeypatch.setattr(gate, "load_private_emails", lambda: set())
+ monkeypatch.setattr(gate, "_git", lambda *args: "")
+ monkeypatch.delenv("PRIVACY_PRIVATE_EMAILS", raising=False)
+ if runner == "ci":
+ result = ci.main(["gate", "--base", "HEAD~1"])
+ else:
+ result = gate.main(
+ ["gate", "origin", "url"],
+ io.StringIO(f"refs/heads/main {'1' * 40} refs/heads/main {'2' * 40}\n"),
+ )
+ assert result == int(secret)