AutoGPT is the vision of accessible AI for everyone, to use and to build on. Our mission is to provide the tools, so that you can focus on what matters.
feat(backend): declare each provider's content hosts so the swap proxy scrubs them (#14869)
Krzysztof Czerwinski1944691b3b93
diff --git a/autogpt_platform/backend/backend/copilot/integration_creds.py b/autogpt_platform/backend/backend/copilot/integration_creds.pyindex 0e95ff827e..32ba66900d 100644--- a/autogpt_platform/backend/backend/copilot/integration_creds.py+++ b/autogpt_platform/backend/backend/copilot/integration_creds.py@@ -396,8 +396,9 @@ def swap_placeholder(provider: str, credential_id: str) -> str:# ``git`` does not read GH_TOKEN; ``gh`` does. This helper, set through git's# environment-variable config rather than a file in the box, answers git's-# credential request for github.com with GH_TOKEN, so a push over HTTPS sends-# it as HTTP Basic, which the proxy swaps. It only ever sees the placeholder.+# credential request for github.com and gist.github.com with GH_TOKEN, so a+# push over HTTPS sends it as HTTP Basic, which the proxy swaps. It only ever+# sees the placeholder._GITHUB_CREDENTIAL_HELPER = ('!f() { test "$1" = get || return 0; ''echo username=x-access-token; echo "password=$GH_TOKEN"; }; f'@@ -406,12 +407,13 @@ _GITHUB_CREDENTIAL_HELPER = (def git_credential_helper_env() -> dict[str, str]:"""Git config, as environment variables, that answers git's credential- request for github.com with ``$GH_TOKEN``."""- return {- "GIT_CONFIG_COUNT": "1",- "GIT_CONFIG_KEY_0": "credential.https://github.com.helper",- "GIT_CONFIG_VALUE_0": _GITHUB_CREDENTIAL_HELPER,- }+ request for github.com and gist.github.com with ``$GH_TOKEN``."""+ hosts = ("github.com", "gist.github.com")+ env = {"GIT_CONFIG_COUNT": str(len(hosts))}+ for i, host in enumerate(hosts):+ env[f"GIT_CONFIG_KEY_{i}"] = f"credential.https://{host}.helper"+ env[f"GIT_CONFIG_VALUE_{i}"] = _GITHUB_CREDENTIAL_HELPER+ return envasync def placeholder_grants(diff --git a/autogpt_platform/backend/backend/copilot/integration_creds_test.py b/autogpt_platform/backend/backend/copilot/integration_creds_test.pyindex d7badd95ef..4df8ede9be 100644--- a/autogpt_platform/backend/backend/copilot/integration_creds_test.py+++ b/autogpt_platform/backend/backend/copilot/integration_creds_test.py@@ -789,6 +789,25 @@ class TestPlaceholders:assert "username=x-access-token\n" in result.stdoutassert "password=hsurr:github:cred-1\n" in result.stdout+ @pytest.mark.skipif(shutil.which("git") is None, reason="needs git")+ def test_git_answers_for_a_gist_too(self, tmp_path):+ env = {+ "PATH": "/usr/local/bin:/usr/bin:/bin",+ "HOME": str(tmp_path),+ "GIT_TERMINAL_PROMPT": "0",+ "GH_TOKEN": "hsurr:github:cred-1",+ **git_credential_helper_env(),+ }+ result = subprocess.run(+ ["git", "credential", "fill"],+ input="protocol=https\nhost=gist.github.com\npath=abc.git\n\n",+ capture_output=True,+ text=True,+ env=env,+ check=True,+ )+ assert "password=hsurr:github:cred-1\n" in result.stdout+class TestGitIdentityGoesThroughRequests:"""A host-side call with the real token: through Requests' address checksdiff --git a/autogpt_platform/backend/backend/copilot/providers.py b/autogpt_platform/backend/backend/copilot/providers.pyindex 8f08fd341d..d03c828308 100644--- a/autogpt_platform/backend/backend/copilot/providers.py+++ b/autogpt_platform/backend/backend/copilot/providers.py@@ -22,12 +22,27 @@ class ProviderEntry(TypedDict):subdomain). Empty means the token is never swapped in. List thehosts that *take* the token, not every host the provider servesfrom: a redirect target with a signed URL needs none.+ content_hosts: Hosts that serve back what was stored with the+ credential (raw files, attachments, release assets) but never take+ the token. The proxy opens their traffic too, and a value in a+ text response from one is turned back into its placeholder, so a+ token a user once committed or pasted cannot be read back through+ them; nothing is ever swapped into a request to one. Same syntax+ as *swap_hosts*. No content host may name a swap host exactly+ (it would take the token); a suffix may cover one, which stays a+ swap host.++ Both lists are the provider's whole footprint as far as the proxy is+ concerned: a host in neither is passed through unread. They are declared+ here, beside the provider's variables and scopes, because a provider is+ only ever handed to a box through this table."""name: strenv_vars: list[str]default_scopes: list[str]swap_hosts: list[str]+ content_hosts: list[str]def _is_github_oauth_configured() -> bool:@@ -51,7 +66,23 @@ SUPPORTED_PROVIDERS: dict[str, ProviderEntry] = {"name": "GitHub","env_vars": ["GH_TOKEN", "GITHUB_TOKEN"],"default_scopes": ["repo"],- "swap_hosts": ["github.com", "api.github.com", "uploads.github.com"],+ # The API, git over HTTPS, release asset uploads, and the two+ # GitHub-owned content hosts that do take the token: a private repo's+ # raw file (``Authorization: token ...`` to raw.githubusercontent.com)+ # and git over HTTPS to a gist (Basic auth to gist.github.com).+ "swap_hosts": [+ "github.com",+ "api.github.com",+ "uploads.github.com",+ "raw.githubusercontent.com",+ "gist.github.com",+ ],+ # objects., gist., media. (LFS) and the other user-content domains:+ # served from signed URLs, they never need the token, so they are only+ # scrubbed. codeload.github.com is left out: it serves only archives,+ # which are binary and never scrubbed, so opening its traffic would+ # cost TLS termination and protect nothing.+ "content_hosts": [".githubusercontent.com"],},}diff --git a/autogpt_platform/backend/backend/copilot/swap_credentials.py b/autogpt_platform/backend/backend/copilot/swap_credentials.pyindex cba308217d..36b6b8b459 100644--- a/autogpt_platform/backend/backend/copilot/swap_credentials.py+++ b/autogpt_platform/backend/backend/copilot/swap_credentials.py@@ -12,8 +12,11 @@ Lookup, OAuth refresh and cache invalidation are ``get_provider_token``'s, thesame path that fills ``GH_TOKEN`` today: there is no second store.A credential's name is its provider slug, and its hosts come from-``SUPPORTED_PROVIDERS``: that table is the binding registry until per-user-bindings exist (SECRT-2616, SECRT-2618).+``SUPPORTED_PROVIDERS``: ``swap_hosts``, where the value may be sent, and+``content_hosts``, where what was stored with it is served back. The proxy+opens both and scrubs responses from both; a credential it gets for either+names only the ``swap_hosts`` as the hosts it may be sent to, so the proxy's+own host check refuses to swap it into a request to a content host."""from typing import Optional@@ -25,7 +28,7 @@ from backend.copilot.integration_creds import (get_provider_token,granted_to_box,)-from backend.copilot.providers import SUPPORTED_PROVIDERS+from backend.copilot.providers import SUPPORTED_PROVIDERS, ProviderEntryfrom backend.util.e2b_network import credential_recordfrom backend.util.exceptions import NotFoundError@@ -54,13 +57,20 @@ def _host_is_bound(host: str, allowed_hosts: list[str]) -> bool:)+def _bound_hosts(entry: ProviderEntry) -> list[str]:+ """Every host the proxy opens for a provider: where its value may go and+ where it may come back from. A provider whose token is never swapped in+ still has its content hosts scrubbed."""+ return [*entry["swap_hosts"], *entry["content_hosts"]]++async def get_swap_bindings() -> dict[str, list[str]]:"""Credential name to bound hosts, with no values: what lets the proxyleave every other host's traffic alone without asking."""return {- slug: list(entry["swap_hosts"])+ slug: hostsfor slug, entry in SUPPORTED_PROVIDERS.items()- if entry["swap_hosts"]+ if (hosts := _bound_hosts(entry))}@@ -116,7 +126,7 @@ async def resolve_swap_credential(take down the box's others."""entry = SUPPORTED_PROVIDERS.get(name)- if entry is None or not _host_is_bound(host, entry["swap_hosts"]):+ if entry is None or not _host_is_bound(host, _bound_hosts(entry)):return Nonerecord = await _box_record(box, user_id)sandbox_id = record.get("sandbox_id")diff --git a/autogpt_platform/backend/backend/copilot/swap_credentials_test.py b/autogpt_platform/backend/backend/copilot/swap_credentials_test.pyindex 4220b54c1d..75f35ac45e 100644--- a/autogpt_platform/backend/backend/copilot/swap_credentials_test.py+++ b/autogpt_platform/backend/backend/copilot/swap_credentials_test.py@@ -20,7 +20,14 @@ from backend.copilot.swap_credentials import (from backend.util import e2b_network_M = "backend.copilot.swap_credentials"-_GITHUB_HOSTS = ["github.com", "api.github.com", "uploads.github.com"]+_GITHUB_HOSTS = [+ "github.com",+ "api.github.com",+ "uploads.github.com",+ "raw.githubusercontent.com",+ "gist.github.com",+]+_GITHUB_CONTENT = [".githubusercontent.com"]_BOX = "box-0123456789abcdef"@@ -68,7 +75,7 @@ def _token(tokens, granted=("cred-a",)):@pytest.mark.asyncioasync def test_bindings_carry_hosts_and_no_values():- assert await get_swap_bindings() == {"github": _GITHUB_HOSTS}+ assert await get_swap_bindings() == {"github": _GITHUB_HOSTS + _GITHUB_CONTENT}@pytest.mark.asyncio@@ -93,7 +100,8 @@ async def test_a_bound_host_gets_the_credential_granted_to_the_box(host):"evil.test","api.github.com.evil.test","notgithub.com",- "raw.githubusercontent.com", # serves GitHub content, takes no token+ "githubusercontent.com.evil.test",+ "codeload.github.com", # archives only: binary, nothing to scrub"",],)@@ -232,12 +240,84 @@ async def test_only_a_live_box_of_the_users_that_swaps_gets_a_value(record):lookup.assert_not_awaited()+ "host",+ [+ "objects.githubusercontent.com",+ "gist.githubusercontent.com",+ "media.githubusercontent.com",+ ],+)+async def test_a_content_host_gets_the_values_to_scrub_but_may_not_be_sent_them(+ host,+):+ """What a user committed or pasted comes back from these; the proxy needs+ the values to scrub it, and must never swap one into a request there."""+ with _token("ghp_real"):+ credential = await resolve_swap_credential("user-1", "github", host, _BOX)+ assert credential is not None+ assert credential.values["cred-a"] == "ghp_real"+ assert credential.allowed_hosts == _GITHUB_HOSTS+ assert not _host_is_bound(host, credential.allowed_hosts)+++_HOSTNAME = re.compile(r"\.?[a-z0-9-]+(\.[a-z0-9-]+)+")++def test_every_provider_says_where_its_token_may_go():"""A new provider must decide this; an empty list means never swapped."""for slug, entry in SUPPORTED_PROVIDERS.items():assert isinstance(entry["swap_hosts"], list), slug- for host in entry["swap_hosts"]:- assert host == host.lower() and "/" not in host and "*" not in host, host+ assert isinstance(entry["content_hosts"], list), slug+ for host in [*entry["swap_hosts"], *entry["content_hosts"]]:+ # Names or a leading-dot suffix, as ``host_in_list`` reads them:+ # no scheme, port, path, wildcard or bare address.+ assert _HOSTNAME.fullmatch(host), (slug, host)+ assert not host.replace(".", "").isdigit(), (slug, host)+ # A content host takes no token: in both lists it would take one.+ overlap = [+ h+ for h in entry["content_hosts"]+ if _host_is_bound(h.lstrip("."), entry["swap_hosts"])+ ]+ assert overlap == [], slug+++def test_no_host_is_bound_to_two_providers():+ """A host names one provider's credential; two would make the proxy fetch+ and scrub both, and leave which one a request meant to the box. Compared+ as bindings, not strings: ``.example.com`` binds ``api.example.com``."""+ for slug, entry in SUPPORTED_PROVIDERS.items():+ for host in [*entry["swap_hosts"], *entry["content_hosts"]]:+ for other, other_entry in SUPPORTED_PROVIDERS.items():+ if other == slug:+ continue+ other_hosts = [+ *other_entry["swap_hosts"],+ *other_entry["content_hosts"],+ ]+ assert not _host_is_bound(host.lstrip("."), other_hosts), (+ host,+ slug,+ other,+ )+++async def test_a_provider_that_swaps_nothing_still_has_its_content_hosts_scrubbed():+ """An empty ``swap_hosts`` means the token is never swapped in, not that+ its content hosts may pass unread."""+ entry = {+ "name": "Files",+ "env_vars": [],+ "default_scopes": [],+ "swap_hosts": [],+ "content_hosts": [".files.example.com"],+ }+ with patch.dict(SUPPORTED_PROVIDERS, {"files": entry}):+ bindings = await get_swap_bindings()+ assert bindings["files"] == [".files.example.com"]def test_minted_usernames_are_what_the_proxy_accepts():@@ -289,3 +369,17 @@ def test_the_proxys_own_copy_agrees_on_every_host():spec.loader.exec_module(swap)for host, entries, _ in HOST_BINDING_TABLE:assert swap.host_in_list(host, entries) is _host_is_bound(host, entries), host+++ "host",+ # A private repo's raw file with ``Authorization: token``; git over HTTPS+ # to a gist with Basic.+ ["raw.githubusercontent.com", "gist.github.com"],+)+async def test_the_github_content_hosts_that_take_the_token_may_be_sent_it(host):+ with _token("ghp_real"):+ credential = await resolve_swap_credential("user-1", "github", host, _BOX)+ assert credential is not None+ assert _host_is_bound(host, credential.allowed_hosts)diff --git a/autogpt_platform/swap_proxy/README.md b/autogpt_platform/swap_proxy/README.mdindex f31fe3cb21..3ee119d9d6 100644--- a/autogpt_platform/swap_proxy/README.md+++ b/autogpt_platform/swap_proxy/README.md@@ -22,7 +22,16 @@ users at once.address space; an allowed address is pinned so the check and the connectagree. (`swap_proxy/egress.py`)3. **Open it or not?** Only hosts some credential is bound to are intercepted.- Everything else is passed through as opaque bytes, never decrypted.+ Everything else is passed through as opaque bytes, never decrypted. A+ provider binds two kinds of host (`SUPPORTED_PROVIDERS` in+ `backend/copilot/providers.py`): `swap_hosts`, which take its token, and+ `content_hosts`, which serve back what was stored with it and never need+ the token (for GitHub, the rest of `*.githubusercontent.com`: objects,+ gists' raw files, LFS media; `raw.githubusercontent.com` and+ `gist.github.com` do take the token, for private raw files and git over+ HTTPS to a gist, so they are swap hosts). Both are intercepted and+ scrubbed; only a `swap_hosts` one is ever sent a value, because the+ credential the backend returns names only those as the hosts it may go to.4. **Swap.** A placeholder in the `Authorization` header becomes the owner'svalue: `Bearer` and `token` (curl, `gh`) and HTTP Basic, which is what gitsends, also for a token in the remote URL (git and curl turn URL userinfo@@ -272,8 +281,12 @@ run it by hand: `gh workflow run platform-swap-proxy-ci.yml --ref <branch>`.## Not here yet-- Per-user bindings and more providers: the binding table is- `SUPPORTED_PROVIDERS[...]["swap_hosts"]` in the backend, GitHub only.+- More providers: the binding table is `SUPPORTED_PROVIDERS` in the backend+ (`swap_hosts`, `content_hosts`), GitHub only. A provider is added there with+ its variables and its hosts; one that takes its key outside the+ `Authorization` header also needs `swap_anywhere`, which nothing sets.+- Path scoping: a credential can carry `allowed_paths`, but the table sets+ none, so binding is by host.- Time-limited grants and the approval step that spark-vm has.- E2B tunnels TCP only: DNS and QUIC leave a box without passing through here.@@ -297,7 +310,10 @@ run it by hand: `gh workflow run platform-swap-proxy-ci.yml --ref <branch>`.stored at a provider and served back so encoded; one stored there by othermeans (by the user, say) is not caught.- A text response over 5 MiB from a bound host is refused, not delivered, for a- box that gets swaps.+ box that gets swaps. That includes content hosts: a text file over 5 MiB+ fetched from `raw.githubusercontent.com` (served as `text/plain`) does not+ reach such a box; cloning the repository still works, since git's packs are+ binary. Lifting this needs a scrub that works on a stream, not a held body.- A compressed text response from a bound host that decodes to more than20 MiB, or uses an encoding other than `gzip`, `deflate`, `br` or `zstd` (orseveral at once), is refused for a box that gets swaps; a request body like@@ -317,9 +333,15 @@ run it by hand: `gh workflow run platform-swap-proxy-ci.yml --ref <branch>`.tell bound hosts from others: a swapping box's TLS connections are thenopened whatever their host, and the same refusals apply to all of them untilthe backend first answers.-- Only bound hosts are scrubbed. A value stored at a provider and served back- from a host that is not bound (a raw-content domain, say) passes through- unread. The name is the box's to+- Only bound hosts are scrubbed: a provider's `swap_hosts` and the+ `content_hosts` it serves stored data back from. A value served from any+ other host passes through unread: a mirror, a CDN the provider does not+ declare, a third party the value was copied to, or an archive+ (`codeload.github.com` is not bound, as it serves only binary archives,+ which are never scrubbed wherever they come from). GitHub Pages+ (`*.github.io`) is not bound either, though it serves back what a repository+ publishes: a token committed to a Pages branch can be read back from it. The+ name is the box's tochoose: TLS to a provider's address under an SNI that is not bound, or plainhttp with such a `Host`, is the same case. For a box that gets swaps, TLSwith no SNI at all and plain http to a bare address are refuseddiff --git a/autogpt_platform/swap_proxy/swap_proxy/addon_test.py b/autogpt_platform/swap_proxy/swap_proxy/addon_test.pyindex 0541ec8045..6d3ef82ef5 100644--- a/autogpt_platform/swap_proxy/swap_proxy/addon_test.py+++ b/autogpt_platform/swap_proxy/swap_proxy/addon_test.py@@ -891,3 +891,35 @@ async def test_a_websocket_upgrade_is_not_refused():flow.live = Trueawait addon_for(flow).response(flow)assert flow.error is None+++# ------------------------------------------------------------ content hosts++CONTENT_HOST = "objects.githubusercontent.com"+++class ContentHostSource(Source):+ """The backend's table with a content host: opened and scrubbed like a+ bound host, while the credential it returns may go only to *HOST*."""++ async def bound_names(self, host):+ return {"github"} if host in (HOST, CONTENT_HOST) else set()+++async def test_a_content_host_is_scrubbed_and_never_sent_the_value(caplog):+ flow = tflow.tflow(resp=True)+ assert flow.response is not None+ addon = addon_for(flow, source=ContentHostSource())+ flow.request.host = flow.server_conn.sni = CONTENT_HOST+ flow.request.headers["host"] = CONTENT_HOST+ flow.request.headers["authorization"] = "token hsurr:github"+ flow.response.headers["content-type"] = "text/plain"+ flow.response.text = f"stored earlier: {TOKEN}"+ with caplog.at_level(logging.INFO, logger="swap_proxy.audit"):+ await addon.requestheaders(flow)+ await addon.request(flow)+ await addon.responseheaders(flow)+ await addon.response(flow)+ assert flow.request.headers["authorization"] == "token hsurr:github"+ assert flow.response.text == "stored earlier: hsurr:github"+ assert audit(caplog) == [("refused", "hsurr:github"), ("scrubbed", None)]