LLM-Driven Extraction of Unstructured Data — Built for API Deployments & ETL Pipeline Workflows
Python ai-agentsdata-engineeringdocument-aigenerative-aiidpjson-extractionllmmcp-serverocrpdf-extractionprompt-engineeringstructured-output
feat(mcp): host deployment-scoped and organization-scoped MCP servers (#2207)
Hari John Kuriakoseaac5edef9f5f
diff --git a/backend/api_v2/execution_urls.py b/backend/api_v2/execution_urls.pyindex 738340013..8fb3fd46f 100644--- a/backend/api_v2/execution_urls.py+++ b/backend/api_v2/execution_urls.py@@ -1,4 +1,4 @@-from django.urls import re_path+from django.urls import include, re_pathfrom rest_framework.urlpatterns import format_suffix_patternsfrom api_v2.api_deployment_views import DeploymentExecution@@ -6,7 +6,15 @@ from api_v2.api_deployment_views import DeploymentExecutionexecute = DeploymentExecution.as_view()-urlpatterns = format_suffix_patterns(+# The deployment MCP server hangs off the execution URL+# (``…/<api_name>/mcp``), so it is matched first. The execution pattern below+# anchors with ``$`` and cannot swallow the ``/mcp`` suffix on its own, but+# ordering it first keeps that independent of the other pattern's exact shape.+# Kept outside ``format_suffix_patterns`` because that helper rewrites leaf+# patterns and is not meant to wrap an ``include()``.+urlpatterns = [re_path(r"^", include("mcp_server.urls"))]++urlpatterns += format_suffix_patterns([re_path(r"^api/(?P<org_name>[\w-]+)/(?P<api_name>[\w-]+)/?$",diff --git a/backend/backend/settings/base.py b/backend/backend/settings/base.pyindex 2f2809024..d14f87b30 100644--- a/backend/backend/settings/base.py+++ b/backend/backend/settings/base.py@@ -119,6 +119,23 @@ API_DEPLOYMENT_PATH_PREFIX = os.environ.get("API_DEPLOYMENT_PATH_PREFIX", "deployment").strip("/")+# Organization-scoped MCP server, mounted at /api/v1/unstract/<org>/mcp/.+# OFF by default: only the deployment-scoped server is required for the initial+# release, so this ships disabled rather than removed — enabling it is a+# settings change, not a revert. Its 23 tools reach the whole organization, so+# it should stay off until the tier model behind it is settled (a `read` key+# cannot use it at all today, because every MCP call is an HTTP POST).+MCP_PLATFORM_SERVER_ENABLED = (+ os.environ.get("MCP_PLATFORM_SERVER_ENABLED", "false").strip().lower() == "true"+)++# Budget for billable MCP tool calls (LLM inference, indexing, pipeline runs),+# counted per organization over a rolling window. Bounds how often an agent can+# trigger paid work; it counts calls, not tokens — see mcp_server/spend_guard.py.+# Applies to whichever servers are enabled above.+MCP_BILLABLE_CALL_LIMIT = int(os.environ.get("MCP_BILLABLE_CALL_LIMIT", 50))+MCP_BILLABLE_WINDOW_SECONDS = int(os.environ.get("MCP_BILLABLE_WINDOW_SECONDS", 3600))+# Maximum file size for presigned URLs in API deployments (in MB)API_DEPL_PRESIGNED_URL_MAX_FILE_SIZE_MB = int(os.environ.get("API_DEPL_PRESIGNED_URL_MAX_FILE_SIZE_MB", 20)@@ -379,6 +396,7 @@ SHARED_APPS = ("pipeline_v2","platform_settings_v2","api_v2",+ "mcp_server","usage_v2","notification_v2","prompt_studio.prompt_profile_manager_v2",@@ -648,7 +666,9 @@ WHITELISTED_PATHS_LIST = ["/static",]WHITELISTED_PATHS = [f"/{PATH_PREFIX}{PATH}" for PATH in WHITELISTED_PATHS_LIST]-# White lists workflow-api-deployment path+# White lists workflow-api-deployment path. This also covers the deployment MCP+# server, which hangs off the same URL and authenticates with the deployment's+# own API key rather than a session.WHITELISTED_PATHS.append(f"/{API_DEPLOYMENT_PATH_PREFIX}")# Whitelisting health check APIdiff --git a/backend/backend/settings/test.py b/backend/backend/settings/test.pyindex f1ad7d584..04fa60768 100644--- a/backend/backend/settings/test.py+++ b/backend/backend/settings/test.py@@ -6,3 +6,11 @@ DEBUG = True# users per test spend most of their time there. Test fixtures need speed, not# resistance to offline cracking.PASSWORD_HASHERS = ["django.contrib.auth.hashers.MD5PasswordHasher"]++# The organization-scoped MCP server ships disabled (see+# MCP_PLATFORM_SERVER_ENABLED in base.py), but its tests must still exercise+# it: several drive requests through the full URL stack precisely so the auth+# middleware runs, and an unmounted route would make them 404 — turning a suite+# that checks a credential is *rejected* into one that passes because nothing+# is there. Shipping-off and untested are different things.+MCP_PLATFORM_SERVER_ENABLED = Truediff --git a/backend/backend/urls_v2.py b/backend/backend/urls_v2.pyindex 7c0ead02a..4b9636a35 100644--- a/backend/backend/urls_v2.py+++ b/backend/backend/urls_v2.py@@ -15,6 +15,7 @@ Including another URLconf2. Add a URL to urlpatterns: path('blog/', include('blog.urls'))"""+from django.conf import settingsfrom django.conf.urls import * # noqa: F401, F403from django.urls import include, path@@ -67,3 +68,17 @@ urlpatterns = [path("platform-api/", include("platform_api.urls")),path("global-api-deployment/", include("global_api_deployment_key.urls")),]++# Organization-scoped MCP server — OFF by default.+#+# Only the deployment-scoped server (mounted in api_v2/execution_urls.py) is+# required for the initial release, so this one ships disabled rather than+# removed: the code is reviewed and tested, and turning it on is a settings+# change rather than a revert.+#+# When enabled it must stay on this tenant path and never under the deployment+# prefix — that prefix is in WHITELISTED_PATHS, so CustomAuthMiddleware skips+# it, and this server has no in-view authentication of its own. Mounting it+# there would expose 23 organization-scoped tools with no auth at all.+if settings.MCP_PLATFORM_SERVER_ENABLED:+ urlpatterns += [path("mcp/", include("mcp_server.platform_urls"))]diff --git a/backend/mcp_server/README.md b/backend/mcp_server/README.mdnew file mode 100644index 000000000..1ba02525f--- /dev/null+++ b/backend/mcp_server/README.md@@ -0,0 +1,480 @@+# Hosted MCP servers++Exposes Unstract to coding agents over the+[Model Context Protocol](https://modelcontextprotocol.io), so an agent can+discover and run document extraction as tool calls instead of hand-rolling HTTP+requests.++Both servers are hosted inside the existing backend — Django views served by the+same gunicorn process, not separate services to deploy or scale. They share the+JSON-RPC transport in `transport.py` and differ only in how a caller is+authenticated and which tools they expose.++| | Deployment server | Platform server |+| --- | --- | --- |+| Scope | one API deployment | one organization |+| Credential | that deployment's API key | a platform API key |+| Authenticated by | the view itself | `CustomAuthMiddleware` |+| URL | `/deployment/api/<org>/<api_name>/mcp` | `/api/v1/unstract/<org>/mcp/` |+| Tools | extract, poll status | extract, plus discovery, observability, state changes and Prompt Studio |++The split is not cosmetic. A deployment key grants exactly one workflow and+resolves to no user, so it cannot authorize anything organization-wide; a+platform key resolves to a service-account user and is checked by the shared+auth middleware. Neither key works on the other server.++**Both servers run extraction, and that is deliberate.** They are two blast+radii for the same operation. A deployment key reaches one deployment, so a+leaked one costs one workflow — but obtaining it is a separate step, and the+platform server cannot hand it over (key retrieval is excluded, see below). A+platform key reaches every deployment in the organization and needs no second+credential. Requiring the narrow key bought no safety when the platform server+already spends money through `executePipeline` and Prompt Studio; it only made+extraction awkward to reach. Pick whichever tradeoff you want.++---++# Deployment server++## Endpoint++An MCP session is scoped to exactly one API deployment, and mirrors the URL+shape of that deployment's REST endpoint:++```+POST /deployment/api/<org_name>/<api_name>/ # REST+POST /deployment/api/<org_name>/<api_name>/mcp # MCP+```++Authentication is the deployment's **existing API key** — the same key used for+REST execution, managed from the same place in the UI. There is no separate MCP+credential to mint or revoke.++```+Authorization: Bearer <api_key>+```++That header is the only accepted credential. There is deliberately no route+carrying the key in the URL: the spec requires `Authorization` on every+authenticated request and forbids access tokens in the URI, and a key in the+path reaches nginx and gunicorn access logs, APM traces and any intermediary —+places a header never goes. `Authorization: Basic` would be spec-legal and is+supported by MCP clients, but buys nothing over Bearer: same header, and base64+is encoding rather than encryption.++Both servers speak MCP revision **2025-06-18** (Streamable HTTP), and also+accept a client that asks for `2024-11-05`: the subset implemented here —+`initialize`, `tools/list`, `tools/call`, `ping` — is identical across the two,+so refusing the older revision would disconnect working clients for no+behavioural reason. Per the spec's negotiation rule `initialize` *echoes* the+client's revision when it is one of these and offers `2025-06-18` otherwise,+rather than announcing its own regardless.++`GET` on either URL answers `405 Method Not Allowed` with `Allow: POST`. Under+Streamable HTTP a client issues `GET` to open a server-to-client SSE stream,+and a server that offers none must decline — every tool call here is+request/response, so there is nothing to stream. The 405 still carries a small+server-identity body for uptime probes and humans with `curl`; it is+unauthenticated and reveals nothing about the deployment behind it.++## Connecting++With Claude Code:++```bash+claude mcp add --transport http unstract \+ https://<host>/deployment/api/<org_name>/<api_name>/mcp \+ --header "Authorization: Bearer <api_key>"+```++## Tools++| Tool | Purpose |+| --- | --- |+| `readMeFirst` | Orientation guide, built from the live deployment. Call first. |+| `getApiInfo` | Name, description, workflow and active state of the deployment. |+| `extractDocument` | Run extraction over document URLs. **Consumes quota.** |+| `getExecutionStatus` | Poll for the result of a pending extraction. |++Documents are passed as **S3 pre-signed URLs** and fetched server-side; the+execution serializer rejects anything else, so a plain public link will not+work. Extraction is asynchronous:+`extractDocument` returns an `execution_id` when it does not finish within the+timeout, and the agent polls `getExecutionStatus` with it.++## Adding a tool++Write a handler taking `MCPContext` as its first argument, then register it in+`registry.py` with a JSON schema:++```python+registry.register(+ MCPTool(+ name="myTool",+ description="What it does, written for an LLM to read.",+ input_schema={"type": "object", "properties": {...}, "required": [...]},+ handler=my_tool,+ writes=False,+ )+)+```++Tool descriptions are prompts, not documentation — they are the only guidance+the calling agent gets. Say what the tool does, when to use it, and what it+costs.++Raise `MCPToolError` for failures the agent can act on (bad arguments, inactive+deployment, rate limit); the message reaches the agent verbatim, so write it as+an instruction. Any other exception is logged and reported generically so+internal detail does not leak to the client.++## Design notes++- **Auth reuses the deployment key path.** `resolve_context` calls the same+ `DeploymentHelper` validation the REST endpoint uses, so the two surfaces+ cannot drift apart on who is allowed in.+- **Execution reuses `ExecutionRequestSerializer`.** URL validation and the+ file-count cap live there; reimplementing them for MCP would let the MCP+ surface quietly diverge from the REST one.+- **All auth failures answer identically** (401, no detail), so the endpoint+ cannot be used to enumerate deployment names.+- **Tool errors are JSON-RPC results, not protocol errors.** Clients treat+ protocol errors as unrecoverable transport faults; an agent-fixable problem+ comes back as `isError: true` content it can read and retry.++---++# Platform server++Organization-scoped: an agent can discover what exists in an Unstract+organization — deployments, workflows, pipelines, Prompt Studio projects — and+change the running state of those resources.++## Endpoint++```+POST /api/v1/unstract/<org_name>/mcp/+Authorization: Bearer <platform_api_key>+```++```bash+claude mcp add --transport http unstract-platform \+ https://<host>/api/v1/unstract/<org_name>/mcp/ \+ --header "Authorization: Bearer <platform_api_key>"+```++Platform API keys are managed at `/api/v1/unstract/<org>/platform-api/keys/`.++## Tools++**Discovery** — what exists:++| Tool | Tier | Purpose |+| --- | --- | --- |+| `readMeFirst` | read | Orientation, including what this credential can reach. |+| `whoami` | read | The key's organization, tier, and remaining billable-call budget. |+| `listApiDeployments` | read | Deployed extraction endpoints, with their `api_name`. |+| `listWorkflows` | read | The workflows behind them. |+| `listPipelines` | read | ETL and task pipelines, with schedule and last-run state. |+| `listPromptStudioProjects` | read | Where extraction prompts are authored. |+| `listPromptStudioDocuments` | read | Documents in a project. The only source of `document_id`. |+| `listPrompts` | read | A project's prompts. The only source of `prompt_id`. |+| `getWorkflowEndpoints` | read | How a workflow is wired — shape only, never config. |+| `listToolInstances` | read | The tool steps inside a workflow. |+| `listTags` | read | Execution tags. |++**Observability** — what happened:++| Tool | Tier | Purpose |+| --- | --- | --- |+| `listExecutions` | read | Recent runs and their status. Start here to debug. |+| `getExecutionDetail` | read | Per-file results and errors for one run. |+| `getUsageSummary` | read | Tokens and cost recorded so far. |+| `getExecutionStatus` | read | Poll an extraction started by `extractDocument`. |++**State changes** — cheap, reversible:++| Tool | Tier | Purpose |+| --- | --- | --- |+| `setApiDeploymentActive` | `read_write` | Take a deployment offline, or bring it back. |+| `setPipelineActive` | `read_write` | Pause or resume a pipeline's schedule. |++**Billable** — each call costs real money, and is budgeted:++| Tool | Tier | Purpose |+| --- | --- | --- |+| `executePipeline` | `read_write` | Trigger a real pipeline run. |+| `indexDocument` | `read_write` | Embed a document so prompts can run against it. |+| `fetchResponse` | `read_write` | Run one prompt against an indexed document. |+| `bulkFetchResponse` | `read_write` | Run several prompts in one pass. |+| `singlePassExtraction` | `read_write` | Run a project's whole prompt set. |+| `extractDocument` | `read_write` | Run a named deployment's extraction workflow. |++To extract a document through a *deployed* API, an agent calls+`listApiDeployments` for an `api_name`, then `extractDocument` with that name —+no second credential and no second session. Polling is free: `getExecutionStatus`+is a read tool, so an agent waiting on a result never spends budget to check on+it. The Prompt Studio tools are for working with prompts *before* they are+deployed; `extractDocument` runs what is already deployed.++Two things differ from the same tool on the deployment server, both because the+caller is org-scoped rather than deployment-scoped:++- **`api_name` is a required argument.** The deployment server learns its target+ from the credential; here the agent names it.+- **`llm_profile_id` is not offered.** That argument is validated against the+ API key's owner, and a platform key resolves to a service account with no+ meaningful owner to check against. Supplying some deployment's key to satisfy+ the check would assert a principal the caller is not, so the argument is+ dropped instead. Pinned by `test_registry_reachability.py`, because the+ coupling is otherwise invisible: the field is what would make the serializer+ read an `api_key` this context does not have.++`getExecutionStatus` re-checks the execution against the organization before+returning it. `DeploymentHelper.get_execution_status` does a bare lookup by id+with no tenant filter, and this deployment runs one shared schema rather than a+schema per tenant, so the check is made here rather than assumed.++## The spend guard++The billable tools drive LLM inference, embedding and vector-store writes. An+agent looping over a list, or retrying on a misread error, can spend a great+deal without anyone watching — so they are budgeted per organization over a+fixed window (`MCP_BILLABLE_CALL_LIMIT`, default 50 per+`MCP_BILLABLE_WINDOW_SECONDS`, default one hour). Fixed rather than rolling:+the counter's TTL expires it, so the window opens at the first billable call and+closes an hour later regardless of the pattern of calls in between.++**A call that could never have run does not cost a slot.** The budget is+consumed on invocation and never refunded, which is right for a call that may+have spent tokens upstream — but wrong for one refused on an argument the+server could check for free. Every billable tool therefore declares a+`preflight` that resolves its target (the deployment, project or pipeline it+names) *before* the budget is claimed. This matters most for `extractDocument`,+whose `api_name` is caller-supplied prose rather than an id copied from a+listing: without it, an agent guessing at names could exhaust the window+without ever reaching an LLM. `test_spend_guard.py` fails if a billable tool is+registered without one.++**It counts calls, not tokens or currency.** Unstract's open-source backend+records usage after the fact but has no pre-flight allowance to check against —+subscription and quota enforcement live in the enterprise overlay, which this+app cannot depend on. A call counter is the strongest guard implementable here,+and it is a blunt one: it bounds how *often* an agent triggers paid work, not+how expensive each call is.++Three behaviours are deliberate:++- **Budget is consumed on invocation and never refunded**, including when the+ tool then fails. A Prompt Studio call that fails partway may already have+ spent tokens upstream, and refunding on failure would let an agent burn+ unbounded spend by failing in a loop. This is the opposite of the rate-limit+ slot in `tools/execution.py`, which models concurrency rather than cost.+- **Exhaustion is temporary, not a permission error.** It comes back as an+ `isError` result naming when to retry, so the agent waits rather than+ concluding the tool is forbidden.+- **It fails open.** If the cache is unavailable the call is allowed and the+ event logged. This guard bounds runaway loops; it is not a licence check, and+ taking the whole MCP surface offline because Redis blipped would be worse.++`whoami` reports the remaining budget so an agent can pace itself instead of+discovering the limit by hitting it.++### Where the counter lives++One key per organization, `mcp:billable:<org_id>`, expired by TTL — there is no+window-start timestamp, so the window begins at the first billable call rather+than at a wall-clock boundary. `cache.add` initialises the counter without+disturbing an in-flight window, and `cache.incr` is atomic, so concurrent+requests cannot race past the limit.++Budget state lives in the project's shared Django cache, like every other cache+user here — which also keeps `override_settings(CACHES=...)` working in tests.++Note the consequence: sharing the general cache DB means a `FLUSHDB` or a+cache-wide eviction resets every organization's window. That is a fail-open+outcome, consistent with the rest of the guard's design — this is a loop+bound, not an audited ledger.++## How write tools are authorized++Platform API key tiers are defined in terms of HTTP methods+(`ApiKeyPermission.allows`), but every JSON-RPC message arrives as an HTTP+`POST` whatever the tool inside it does — so the auth middleware's tier check+cannot tell `listWorkflows` from `executePipeline`.++Each tool therefore declares the method its REST equivalent would use+(`required_method`), and `check_tool_allowed` re-applies the key's tier against+*that*. This reuses the semantics the codebase already enforces instead of+inventing a parallel scheme, and it means a tool marked `DELETE` is+`full_access`-only for the same reason a REST `DELETE` is.++A `writes=True` tool left at the default `required_method="GET"` would slip past+that guard, so a test asserts none exists.++## What is deliberately not exposed++The platform has a much larger API surface than this server wraps. What is left+out is left out on purpose, and falls into three groups.++### Anything whose response carries a credential++Several subsystems return decrypted secrets as part of their ordinary+responses — connector configuration includes the credentials used to reach the+source system, adapter configuration includes the provider API key, notification+configuration includes the webhook's authorization token, and the key-management+endpoints return key material by design.++**No tool wraps any of them.** An agent's context is not a safe place for a+credential: it is logged, it may be replayed to a model provider, and an agent+processing an untrusted document can be induced to repeat what it has seen.++Concretely, this server has no tool for:++- connector or adapter configuration (`listConnectors`, `getAdapter`, …)+- platform, deployment, or platform-API key management — including **creating+ or rotating** a key, which returns the new secret in its response+- notification / webhook configuration+- the Postman collection export, which embeds a live API key++Two mechanisms keep this true as tools are added:++- `test_no_credential_leak.py` seeds an organization with recognizable fake+ secrets, invokes **every** read tool in the registry, and fails if any of them+ appears in the output. New tools are covered the moment they are registered.+- A test asserts that no tool name suggests credential handling, and none+ suggests connector or adapter access.++Where a tool must touch something adjacent to a credential, it names the fields+it returns rather than serializing a model. `getWorkflowEndpoints` is the case+to look at: a workflow endpoint points at a connector instance, so the tool+returns the *shape* of the connection — endpoint type, connection type,+connector name — and never the configuration. Free-form error text from failed+executions is additionally passed through `redact_secrets`, because a failing+connector reports the connection string it tried.++### Destructive operations++Deleting a workflow, deployment, Prompt Studio project, tag or connector+destroys work that no inverse call restores; so does removing organization+members. Every write tool here is reversible by construction — the opposite call+puts things back — and `required_method="DELETE"` exists so that if a+destructive tool is ever added it is `full_access`-only from the start.++Also excluded: password resets, role assignment and revocation, and member+removal. These change who can access the organization, which is not a decision+to delegate to an agent.++### Endpoints that reach out to a caller-influenced host++The connector and adapter *test* endpoints exist to verify a configuration by+making a live outbound call to the host it names. Exposing them would hand any+platform API key a request-forgery primitive: the agent chooses the+destination, the server makes the call from inside the deployment's network,+and the response comes back. That the same endpoints also take credentials as+*input* is a second reason, but the outbound call is the disqualifying one.++Prompt Studio project **import** is excluded on the same grounds — it ingests a+bundle the caller supplies. **Export** is excluded because a project bundle may+carry adapter references, which is the credential rule above applied to a file+rather than a JSON field.++### Workflow endpoint and tool-instance *configuration*++Reading the shape of a workflow's endpoints is exposed+(`getWorkflowEndpoints`, `listToolInstances`); writing it is not. Endpoint+configuration is where connector instances are bound to a workflow, so a write+there redirects where a workflow reads its input from and writes its output to+— without changing anything that a later read would show as unusual. That is a+data-exfiltration path dressed as a settings change, so it stays out.++### Everything else, for now++Not excluded on principle, simply not built: file upload/download, and Prompt+Studio project and prompt *authoring*. The coherent scope of this server is+that an agent operates on projects and documents a human has already set up —+it can discover them, run them, and read the results, but it does not create+the raw material. These are candidates for later, with the same rules applied.++Document and prompt *listing* used to sit in this section. It has since been+built (`listPromptStudioDocuments`, `listPrompts`) because it was not a nicety:+those two tools are the only source of the `document_id` and `prompt_id` that+every billable Prompt Studio tool requires, so without them those tools were+listed but uncallable. `tests/test_registry_reachability.py` now fails if any+tool requires an id that no tool is declared to produce.++## Why the URL matters++This server is mounted under the tenant prefix, next to `platform-api/`, and+**not** under the deployment prefix. `WHITELISTED_PATHS` is matched with+`startswith`, so everything under `/deployment/...` — including the deployment+MCP server — is exempt from `CustomAuthMiddleware`, while+`/api/v1/unstract/<org>/mcp/` is not. That is exactly how this server inherits+platform-key authentication.++Moving these URLs under the whitelisted prefix would silently remove all+authentication. `test_platform_auth.py::test_the_endpoint_is_not_whitelisted`+guards against that.++Because auth lives in middleware, its tests go through `django.test.Client` and+the real URL. A test using `APIRequestFactory` or calling the view directly+bypasses the middleware and would pass against a completely open endpoint.++## Two constraints worth knowing++**A `read`-tier key cannot use this server at all.** The middleware's tier check+gates on HTTP method, and every JSON-RPC message is a `POST`, which `read`+disallows —+so a read-only key is refused before reaching the view, even for the read tools.+Use a `read_write` key. Changing that would mean special-casing MCP paths in the+middleware, which is a decision for maintainers rather than something this app+should do unilaterally.++**These tools reach the whole organization.** A platform key resolves to a+service account, and `is_service_account=True` makes `for_user()` managers+return `self.all()` — so it ignores per-user sharing regardless of the `USER`+role the key was created with. For reads that is a disclosure caveat; for+writes it means the key can modify *any* org resource. `whoami`, `readMeFirst`+and every write tool's description say so.++## Adding a tool here++Same registry mechanics as the deployment server, with `PlatformMCPContext`+(`user`, `platform_key`, `org_name`, `request`) as the first argument. Query+through the model's `for_user(context.user)` manager so the platform's own+visibility rules apply, and cap listings — `LIST_LIMIT` with a `truncated` note,+never silent truncation.++**Never** build a response with `serializer.data`, `model_to_dict`, or a `**`+splat. Name every field. Several models decrypt credentials on attribute+access, so wholesale serialization is how a secret escapes — see the exclusion+list above.++For a write tool, also:++- set `writes=True` and a `required_method` (`"POST"` for a mutation, `"DELETE"`+ for anything destructive — that makes it `full_access`-only);+- resolve the target through `for_user` so a caller cannot reach another+ organization's resource, and raise `MCPToolError` naming the list tool when it+ is not found;+- include `_ORG_WIDE_WARNING` in the description — the description is the prompt+ the agent reads before acting, and the service-account caveat matters more+ here than anywhere else;+- return `changed: False` rather than silently succeeding on a no-op, so an+ agent can tell "I did this" from "this was already so".++For a tool that costs money, also set `billable=True`. That flag is the only+thing wiring it into the spend guard, so forgetting it leaves the tool+unbudgeted — `test_spend_guard.py` names the tools that must carry it.++If a tool returns free-form text that originates outside this app (an error+message, a log line), pass it through `redact_secrets` first.++---++## Not implemented++OAuth 2.1 with dynamic client registration, which MCP defines for browser-based+one-click connectors. Bearer auth covers Claude Code and API clients on both+servers. Adding OAuth is additive — it would mount discovery endpoints+alongside these routers without changing the transport.diff --git a/backend/mcp_server/__init__.py b/backend/mcp_server/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/mcp_server/apps.py b/backend/mcp_server/apps.pynew file mode 100644index 000000000..3447b5355--- /dev/null+++ b/backend/mcp_server/apps.py@@ -0,0 +1,5 @@+from django.apps import AppConfig+++class MCPConfig(AppConfig):+ name = "mcp_server"diff --git a/backend/mcp_server/constants.py b/backend/mcp_server/constants.pynew file mode 100644index 000000000..b11bbeb97--- /dev/null+++ b/backend/mcp_server/constants.py@@ -0,0 +1,65 @@+"""Constants for the hosted MCP server."""+++class MCPServer:+ """Identity advertised to MCP clients during `initialize`."""++ NAME: str = "unstract"+ VERSION: str = "0.1.0"++ # Protocol revision this transport implements, and the one offered to a+ # client that asks for something unsupported. This is 2025-06-18 because+ # that is the revision whose rules the transport actually follows —+ # Streamable HTTP, and answering GET with 405 rather than an SSE stream.+ PROTOCOL_VERSION: str = "2025-06-18"++ # Revisions this server will speak if a client asks for one by name.+ #+ # 2024-11-05 is kept because the request/response subset used here — the+ # `initialize` handshake, `tools/list`, `tools/call` and `ping` — is+ # unchanged between the two, and a client pinned to it works against this+ # server today. What differs is the parts this server does not implement+ # anyway (server-initiated streaming, session ids), so accepting the older+ # revision costs nothing and disconnects no one.+ #+ # Ordered newest first: `initialize` echoes the client's revision when it+ # appears here, and otherwise offers the first entry.+ SUPPORTED_PROTOCOL_VERSIONS: tuple[str, ...] = ("2025-06-18", "2024-11-05")+++class JSONRPC:+ """JSON-RPC 2.0 wire constants."""++ VERSION: str = "2.0"++ # Standard error codes (JSON-RPC 2.0 spec).+ PARSE_ERROR: int = -32700+ INVALID_REQUEST: int = -32600+ METHOD_NOT_FOUND: int = -32601+ INVALID_PARAMS: int = -32602+ INTERNAL_ERROR: int = -32603++ # Implementation-defined codes (-32000 to -32099 reserved for servers).+ UNAUTHORIZED: int = -32001+ TOOL_EXECUTION_ERROR: int = -32002++ # Canonical `message` for each code. The JSON-RPC 2.0 spec fixes these+ # strings, and a client may match on them, so they are defined once beside+ # the code they belong to rather than repeated at each raise site.+ MSG_PARSE_ERROR: str = "Parse error"+ MSG_INVALID_REQUEST: str = "Invalid Request"+ MSG_METHOD_NOT_FOUND: str = "Method not found"+ MSG_INVALID_PARAMS: str = "Invalid params"+ MSG_INTERNAL_ERROR: str = "Internal error"+++class MCPMethod:+ """MCP methods handled by this transport."""++ INITIALIZE: str = "initialize"+ PING: str = "ping"+ TOOLS_LIST: str = "tools/list"+ TOOLS_CALL: str = "tools/call"++ # Notifications carry no `id` and expect no response body.+ NOTIFICATION_PREFIX: str = "notifications/"diff --git a/backend/mcp_server/context.py b/backend/mcp_server/context.pynew file mode 100644index 000000000..2ed88fa7b--- /dev/null+++ b/backend/mcp_server/context.py@@ -0,0 +1,67 @@+"""Resolved request context handed to MCP tool handlers.++Auth, organization resolution and deployment lookup all happen once in the+transport view. Tools receive the result and never repeat that work.+"""++from dataclasses import dataclass+from typing import Any++from api_v2.models import APIDeployment+++@dataclass(frozen=True)+class PlatformMCPContext:+ """Everything a tool needs about an organization-scoped caller.++ Built from what ``CustomAuthMiddleware`` already resolved — this server+ does not re-authenticate. Deliberately shares no base class with+ ``MCPContext``: the two servers expose disjoint tool sets, and a common+ base would invite a tool to be written against whichever fields happen to+ overlap.++ Attributes:+ user: The service-account user the platform key resolves to. Passed to+ org-scoped managers so tools inherit the platform's own sharing+ rules rather than reimplementing them.+ platform_key: The validated key, carried for its permission tier.+ org_name: Organization identifier from the URL, already validated+ against the key's own organization by the middleware.+ request: The DRF request. Carried because several platform helpers+ (notably ``PipelineManager.execute_pipeline``) take a request+ rather than a user and mutate ``request.data`` on the way through.+ Passing the real request is safer than fabricating one, since it+ keeps the org and auth state those helpers read.+ """++ user: Any+ platform_key: Any+ org_name: str+ request: Any = None+++@dataclass(frozen=True)+class MCPContext:+ """Everything a tool needs about the caller's deployment.++ Attributes:+ api: The API deployment this MCP server session is scoped to.+ api_key: The validated API key used to authenticate the request, or+ ``None`` when the caller reached this deployment with an+ organization-scoped platform key instead.++ It is read in exactly one place: validating that an+ ``llm_profile_id`` belongs to the key's owner+ (``ExecutionRequestSerializer.validate_llm_profile_id``). The+ platform server therefore does not offer that argument, so the+ validator never runs and the ``None`` is never dereferenced — see+ ``tools/platform_execution``. Supplying some other deployment's key+ to fill the field would make that ownership check pass against a+ principal the caller is not.+ org_name: Organization identifier taken from the URL, matching the+ value already placed in the state store by the auth layer.+ """++ api: APIDeployment+ api_key: str | None+ org_name: strdiff --git a/backend/mcp_server/exceptions.py b/backend/mcp_server/exceptions.pynew file mode 100644index 000000000..3200cba49--- /dev/null+++ b/backend/mcp_server/exceptions.py@@ -0,0 +1,15 @@+"""Exceptions raised by MCP tool handlers."""+++class MCPToolError(Exception):+ """A tool failed in a way the calling agent can understand and act on.++ Raised for conditions the agent could plausibly recover from — bad+ arguments, an inactive deployment, a rate limit. The transport turns these+ into a JSON-RPC error carrying the message verbatim, so the message is+ prompt-facing: state what went wrong and what the agent should do next.++ Unexpected exceptions are deliberately *not* funnelled through this type;+ they are logged and reported generically so internal detail does not leak+ to the client.+ """diff --git a/backend/mcp_server/platform_urls.py b/backend/mcp_server/platform_urls.pynew file mode 100644index 000000000..71b11189a--- /dev/null+++ b/backend/mcp_server/platform_urls.py@@ -0,0 +1,26 @@+"""URLs for the organization-scoped MCP server.++Mounted under the tenant prefix in ``backend/urls_v2.py``, alongside+``platform-api/``:++ POST /api/v1/unstract/<org_name>/mcp/++That placement is load-bearing, and the reason is the *prefix*, not the+``/mcp`` segment. ``WHITELISTED_PATHS`` is matched with ``startswith`` and+contains ``/{API_DEPLOYMENT_PATH_PREFIX}`` — i.e. ``/deployment`` — so the+deployment MCP server is exempt from ``CustomAuthMiddleware`` because it hangs+off ``/deployment/api/...``, and authenticates its own key instead. Nothing in+that list mentions ``/mcp``.++This path starts with ``/api/v1/unstract/`` and so matches no whitelist entry,+which is precisely how it inherits platform-key authentication for free.+Re-mounting it under ``/deployment/`` would silently remove all authentication.+"""++from django.urls import path++from mcp_server.platform_views import PlatformMCPServerView++urlpatterns = [+ path("", PlatformMCPServerView.as_view(), name="platform_mcp_server"),+]diff --git a/backend/mcp_server/platform_views.py b/backend/mcp_server/platform_views.pynew file mode 100644index 000000000..dea2c1b7e--- /dev/null+++ b/backend/mcp_server/platform_views.py@@ -0,0 +1,184 @@+"""Organization-scoped MCP server.++Mounted on a tenant URL — deliberately *not* under the whitelisted ``/mcp/``+prefix — so ``CustomAuthMiddleware`` runs ahead of it and performs the+authentication. By the time this view executes, the middleware has already:++* resolved the ``Authorization: Bearer <uuid>`` platform key,+* rejected unknown, inactive, and wrong-organization keys,+* rejected keys whose permission tier disallows the HTTP method,+* set ``request.user`` to the key's service account and pinned+ ``ORGANIZATION_ID`` in the state store.++This view therefore reads that result rather than re-authenticating. Two+consequences worth stating plainly, because both are easy to get wrong:++1. ``authentication_classes`` must NOT be emptied here. The deployment server+ empties it because it owns its own auth; doing the same here is harmless+ only by accident, and setting DRF authenticators that return a user would+ actively override what the middleware resolved.+2. Because the auth lives in middleware, a test that calls this view through+ ``APIRequestFactory`` bypasses it entirely and would pass against a+ completely open endpoint. The auth-boundary tests go through the full URL+ stack for exactly this reason.+"""++import logging+from typing import Any++from platform_api.models import ApiKeyPermission+from rest_framework.request import Request++from mcp_server import spend_guard+from mcp_server.context import PlatformMCPContext+from mcp_server.registry import PLATFORM_TOOLS+from mcp_server.transport import BaseMCPView++logger = logging.getLogger(__name__)+++class PlatformMCPServerView(BaseMCPView):+ """MCP JSON-RPC endpoint scoped to an organization.++ Exposes discovery, observability, state-change and billable tools over the+ organization's resources, authenticated by a platform API key. Tool access+ is gated twice: by the key's permission tier (``check_tool_allowed``) and,+ for tools that cost money, by a per-organization budget+ (``check_spend_allowed``).+ """++ registry = PLATFORM_TOOLS++ # Intentionally not overriding authentication_classes: the project sets no+ # DEFAULT_AUTHENTICATION_CLASSES that would resolve a user, so the+ # middleware's request.user survives into the view.+ permission_classes: list = []++ def initialize_request(self, request: Request, *args: Any, **kwargs: Any) -> Request:+ """Skip CSRF, but only for a bearer-authenticated request.++ The middleware already marks those CSRF-exempt; this makes the view's+ behaviour explicit rather than dependent on that. Gating on the key+ matters because ``CustomAuthMiddleware`` falls through to session auth+ on this path — exempting unconditionally would let a logged-in browser+ session reach the organization-wide write and billable surface with+ CSRF off.+ """+ if getattr(request, "platform_api_key", None):+ request.csrf_processing_done = True+ return super().initialize_request(request, *args, **kwargs)++ def auth_failure_message(self) -> str:+ return "A valid platform API key is required"++ def server_instructions(self, context: PlatformMCPContext) -> str:+ return (+ "Unstract runs LLM-driven extraction over unstructured documents "+ "and returns structured JSON. This server is scoped to the "+ f"organization '{context.org_name}'. It can discover deployments, "+ "workflows, pipelines and Prompt Studio projects, and can change "+ "their running state — activating deployments, pausing schedules, "+ "triggering pipeline runs. Those writes affect the whole "+ "organization and consume quota. Call readMeFirst before any "+ "other tool."+ )++ def resolve_context(+ self, request: Request, **kwargs: Any+ ) -> PlatformMCPContext | None:+ """Read the credential the auth middleware already resolved.++ Returns None when the middleware did not attach a platform key. That is+ a defence-in-depth check rather than the primary gate: an unauthenticated+ request should never reach this view, because the middleware rejects it+ first with its own 401. If it does reach here — a whitelist edit, a+ routing change — refusing is the safe answer.+ """+ platform_key = getattr(request, "platform_api_key", None)+ user = getattr(request, "user", None)++ # The organization is not a URL kwarg here: the tenant prefix consumes+ # it, and OrganizationMiddleware parses it onto the request. Read it+ # from there rather than from the route.+ org_name = getattr(request, "organization_id", None)++ if platform_key is None or user is None or not user.is_authenticated:+ logger.warning(+ "Platform MCP reached without a resolved platform key "+ f"(org '{org_name}'). Check that this path is not whitelisted."+ )+ return None++ return PlatformMCPContext(+ user=user,+ platform_key=platform_key,+ # Fall back to the key's own organization so the context is never+ # built with a null org, whatever the routing did.+ org_name=org_name or str(platform_key.organization.organization_id),+ request=request,+ )++ def check_tool_allowed(self, tool: Any, context: PlatformMCPContext) -> str | None:+ """Apply the platform key's permission tier to this specific tool.++ The middleware already checked the tier, but it can only check it+ against the HTTP method of the request — and every JSON-RPC message+ arrives as an HTTP POST, whatever the tool inside it does.+ That makes its verdict uselessly coarse here: it cannot tell+ ``listWorkflows`` from ``executePipeline``.++ So each tool declares the method its REST equivalent would use, and the+ tier is re-evaluated against that. This deliberately reuses+ ``ApiKeyPermission.allows`` rather than inventing a second permission+ scheme, so a tool marked DELETE is full_access-only for exactly the+ same reason a REST DELETE is.++ **This refuses nothing today, and that is expected.** A ``read`` key+ cannot reach the view at all — the middleware rejects POST on tier+ before dispatch — and both remaining tiers (``read_write``,+ ``full_access``) allow POST, while every registered tool declares GET+ or POST and none declares DELETE. So the check is prospective:+ defense-in-depth that starts refusing the day a DELETE-tier tool is+ added, or the day the middleware grows an MCP carve-out that lets a+ ``read`` key through.++ Its future value rests entirely on ``required_method`` being declared+ truthfully, which is why ``MCPToolRegistry.register`` refuses a+ ``writes=True`` tool left at the ``"GET"`` default rather than trusting+ a reviewer to catch it.+ """+ try:+ permission = ApiKeyPermission(context.platform_key.permission)+ except ValueError:+ # The middleware rejects unrecognised tiers, so this is+ # unreachable in practice; refusing is the safe answer if it ever+ # becomes reachable.+ logger.error(+ f"Unrecognised platform key tier "+ f"'{context.platform_key.permission}'; refusing tool access."+ )+ return "This API key has an unrecognised permission tier."++ if permission.allows(tool.required_method):+ return None++ return (+ f"Tool '{tool.name}' requires a platform API key permitting "+ f"{tool.required_method}; this key is '{permission.value}'."+ )++ def check_spend_allowed(self, tool: Any, context: PlatformMCPContext) -> str | None:+ """Claim one unit of the organization's billable-call budget.++ Only billable tools consume budget; everything else is free. The claim+ is made here rather than inside each handler so a new billable tool+ cannot be added without being budgeted — the flag on the registry entry+ is the only thing it has to get right.+ """+ if not tool.billable:+ return None++ state = spend_guard.consume(context.org_name)+ if state.allowed:+ return None+ return state.message()diff --git a/backend/mcp_server/registry.py b/backend/mcp_server/registry.pynew file mode 100644index 000000000..e3e16bf71--- /dev/null+++ b/backend/mcp_server/registry.py@@ -0,0 +1,738 @@+"""Tool registry for the hosted MCP server.++Tools are plain callables registered with the JSON schema MCP clients need in+order to call them. Each tool receives the resolved deployment context as its+first argument, so tool implementations never re-do auth or org resolution.+"""++import logging+from collections.abc import Callable+from dataclasses import dataclass, field+from typing import Any++logger = logging.getLogger(__name__)+++@dataclass(frozen=True)+class MCPTool:+ """A single tool exposed over the MCP transport.++ Attributes:+ name: Tool name as seen by the MCP client.+ description: Prompt-facing description. Written for an LLM audience —+ it is the only guidance the calling agent gets.+ input_schema: JSON schema for the tool's arguments.+ handler: Callable invoked as ``handler(context, **arguments)``.+ title: Human-readable name a host shows in its UI, where the camelCase+ tool name reads poorly. Defaults to the tool name when unset.+ writes: True when the tool has side effects (consumes quota, starts an+ execution). Read-only tools are safe to retry; write tools are not.+ required_method: The HTTP method this tool's REST equivalent would use.+ Platform API key tiers are defined in terms of HTTP methods+ (``ApiKeyPermission.allows``), and every JSON-RPC message arrives+ as an HTTP POST regardless of the tool it carries — so+ declaring the *equivalent* method is what lets the existing tier+ semantics apply per tool instead of per request. "GET" for reads,+ "POST" for mutations, "DELETE" for destructive operations (which+ only ``full_access`` permits). Unused by the deployment server,+ whose key has no tiers.+ billable: True when invoking the tool costs money — LLM inference,+ embedding, indexing, or a pipeline run. Budgeted per organization+ by ``mcp_server.spend_guard``. Deliberately distinct from+ ``writes``: a tool can mutate cheaply (pausing a schedule) or cost+ money without changing configuration (running an extraction).+ idempotent: True when repeating the call with the same arguments lands+ in the same state — activating an already-active deployment, or+ pausing an already-paused schedule. False when each call creates+ new work, such as starting an execution. Only meaningful for tools+ that write; a read tool is idempotent by definition.++ Declared rather than derived from ``billable``: the deployment+ server's ``extractDocument`` is not billable there (its cost is+ bounded by the rate limiter rather than the spend guard) but still+ starts a fresh execution on every call, so deriving would have+ advertised it as safe to repeat.+ consumes_result: True when *reading* is itself destructive — the result+ store is one-shot, so a successful read acknowledges the execution+ and the payload cannot be fetched again. ``getExecutionStatus`` on+ a COMPLETED execution is the case: it neither writes configuration+ nor spends money, so ``writes``/``billable`` are both false, but+ calling it twice loses data. Without this flag the derivation below+ would advertise ``readOnlyHint: true`` and tell an agent it is safe+ to poll freely — the opposite of the truth.+ preflight: Optional ``preflight(context, **arguments)`` run *before* the+ billable budget is claimed. It exists because the budget is+ consumed on invocation and never refunded, so a call refused for a+ reason the caller could have been told about up front — a+ deployment name that does not resolve — would otherwise burn a slot+ having spent nothing. Raise ``MCPToolError`` to refuse.++ Only for checks that are cheap and certain: resolving a named+ resource, not anything that could itself fail transiently. A tool+ whose arguments are all opaque ids the caller got from a listing+ does not need one.+ """++ name: str+ description: str+ input_schema: dict[str, Any]+ handler: Callable[..., Any]+ title: str = ""+ writes: bool = False+ required_method: str = "GET"+ billable: bool = False+ idempotent: bool = False+ consumes_result: bool = False+ preflight: Callable[..., Any] | None = None++ def annotations(self) -> dict[str, Any]:+ """Behaviour hints for `tools/list`, built from the flags above.++ ``readOnlyHint`` and ``destructiveHint`` are *derived* from+ ``writes``/``billable``/``required_method`` rather than declared per+ tool, so a tool cannot advertise something its own flags contradict —+ the same reason ``required_method`` is enforced at registration instead+ of being trusted. ``idempotentHint`` is the exception and is declared;+ see the ``idempotent`` field for why no flag here implies it.++ Spec defaults matter here and are not what you would guess+ (`ToolAnnotations`, rev 2025-06-18):++ * ``readOnlyHint`` defaults to **false**, so a read tool that stays+ silent looks like a mutator. Every read tool must say so explicitly.+ * ``destructiveHint`` defaults to **true**, so a write tool that stays+ silent looks destructive. Every write tool here is reversible —+ activating a deployment, pausing a schedule, starting an execution —+ and none deletes anything, which is a deliberate property of this+ surface, so it is stated rather than left to the default.+ * ``destructiveHint`` and ``idempotentHint`` are meaningful only when+ ``readOnlyHint`` is false, so they are omitted for read tools rather+ than set to a value a client is told to ignore.++ These are hints, not enforcement: the spec tells clients to distrust+ annotations from untrusted servers, and the real gate stays+ ``check_tool_allowed`` plus the spend guard.+ """+ # ``consumes_result`` participates because a one-shot read is not a+ # read as far as a client is concerned: the spec's readOnlyHint means+ # "no modification of its environment", and acknowledging an execution+ # so its result can never be fetched again plainly modifies it.+ read_only = not self.writes and not self.billable and not self.consumes_result+ hints: dict[str, Any] = {+ "title": self.title or self.name,+ "readOnlyHint": read_only,+ # Every tool here reaches Unstract's own APIs and, for the+ # extraction paths, the documents and model providers behind them.+ # That is an open world by the spec's definition.+ "openWorldHint": True,+ }+ if not read_only:+ # DELETE is the tier reserved for destructive operations; nothing+ # on this surface declares it today, and the registry refuses a+ # write tool that leaves required_method at the "GET" default.+ hints["destructiveHint"] = self.required_method == "DELETE"+ hints["idempotentHint"] = self.idempotent+ return hints++ def to_mcp_schema(self) -> dict[str, Any]:+ """Serialize to the shape returned by `tools/list`."""+ return {+ "name": self.name,+ "title": self.title or self.name,+ "description": self.description,+ "inputSchema": self.input_schema,+ "annotations": self.annotations(),+ }+++@dataclass+class MCPToolRegistry:+ """Ordered name -> tool mapping.++ Ordering is preserved so `tools/list` presents tools in the order they were+ registered; agents weight earlier tools more heavily, and `readMeFirst`+ needs to come first.+ """++ _tools: dict[str, MCPTool] = field(default_factory=dict)++ def register(self, tool: MCPTool) -> None:+ if tool.name in self._tools:+ raise ValueError(f"Duplicate MCP tool registration: '{tool.name}'")+ if tool.writes and tool.required_method == "GET":+ # `writes` and `required_method` are independent fields, so+ # MCPTool(..., writes=True) silently keeps the "GET" default — and+ # ApiKeyPermission.allows("GET") is true for *every* tier, so+ # check_tool_allowed would wave the tool through for a read-only+ # key. The unsafe pairing being the default is the wrong way round+ # for the invariant the whole tier model rests on.+ #+ # Enforced at registration rather than only in a test: a tool+ # declared wrongly cannot reach a running server, whereas a CI+ # assertion can be deselected or simply not run on the branch that+ # adds the tool.+ raise ValueError(+ f"MCP tool '{tool.name}' declares writes=True with "+ "required_method='GET'. Every key tier allows GET, so the tier "+ "guard would let a read-only key invoke it. Declare the method "+ "the tool's REST equivalent would use — 'POST' for a mutation, "+ "'DELETE' for a destructive operation."+ )+ self._tools[tool.name] = tool++ def get(self, name: str) -> MCPTool | None:+ return self._tools.get(name)++ def names(self) -> list[str]:+ return list(self._tools)++ def list_schemas(self) -> list[dict[str, Any]]:+ return [tool.to_mcp_schema() for tool in self._tools.values()]+++def build_deployment_registry() -> MCPToolRegistry:+ """Build the tools exposed by the deployment-scoped MCP server.++ Imported lazily inside the function so that registering a tool cannot+ trigger Django model imports at module-import time.+ """+ from mcp_server.tools.execution import (+ extract_document,+ extract_document_schema,+ get_execution_status,+ get_execution_status_schema,+ )+ from mcp_server.tools.info import get_api_info, get_api_info_schema, read_me_first++ registry = MCPToolRegistry()++ registry.register(+ MCPTool(+ name="readMeFirst",+ description=(+ "START HERE. Returns a guide to this MCP server: what the "+ "connected Unstract API deployment does, the available tools, "+ "and the recommended call sequence. Takes no arguments."+ ),+ input_schema={"type": "object", "properties": {}, "required": []},+ handler=read_me_first,+ )+ )+ registry.register(+ MCPTool(+ name="getApiInfo",+ description=(+ "Get details of the Unstract API deployment this MCP server is "+ "connected to: its display name, description, the workflow it "+ "runs, and whether it is active. Call this to learn what kind of "+ "document the deployment expects before extracting. "+ "Takes no arguments."+ ),+ input_schema=get_api_info_schema(),+ handler=get_api_info,+ )+ )+ registry.register(+ MCPTool(+ name="extractDocument",+ description=(+ "Run the connected Unstract API deployment over one or more "+ "documents and return the structured extraction result.\n\n"+ "Documents are supplied as HTTPS URLs on an AWS S3 host, which "+ "Unstract fetches server-side; links to any other host are "+ "rejected, so upload to S3 first if the document is not "+ "already there. Use a pre-signed URL for a private object — "+ "the usual case. Extraction is asynchronous: when it does not "+ "finish within `timeout` seconds this returns "+ "`execution_status: PENDING` along with an `execution_id`. Poll "+ "`getExecutionStatus` with that id to collect the result.\n\n"+ "This consumes the organization's extraction quota — do not call "+ "it speculatively or retry a call that already returned an "+ "execution_id."+ ),+ input_schema=extract_document_schema(),+ handler=extract_document,+ writes=True,+ # The deployment server has no tiers, so nothing reads this here —+ # but declaring it truthfully lets the write/method invariant in+ # `register` be unconditional rather than opt-in.+ required_method="POST",+ )+ )+ registry.register(+ MCPTool(+ name="getExecutionStatus",+ description=(+ "Fetch the status and, once available, the result of an "+ "extraction previously started by `extractDocument`. Pass the "+ "`execution_id` that call returned.\n\n"+ "Returns an `execution_status` of PENDING, EXECUTING, COMPLETED "+ "or ERROR. Poll while it is PENDING or EXECUTING, leaving a few "+ "seconds between calls."+ ),+ input_schema=get_execution_status_schema(),+ handler=get_execution_status,+ # Reading a COMPLETED execution acknowledges it and the result+ # store is one-shot, so the payload cannot be fetched again.+ # Not `writes` (no configuration changes) and not `billable`+ # (no spend), but emphatically not read-only either.+ consumes_result=True,+ )+ )++ return registry+++def build_platform_registry() -> MCPToolRegistry:+ """Build the tools exposed by the organization-scoped MCP server.++ Each tool declares the HTTP method its REST equivalent would use, which is+ how the platform key's existing permission tier applies per tool rather+ than per request — see ``MCPTool.required_method``.++ Two categories are deliberately absent:++ * **Credential operations.** Creating or rotating an API key returns the+ secret in its response, so an MCP tool for it would be an exfiltration+ path for an agent processing untrusted document content. The codebase+ already reasons this way — see ``CanRotatePlatformApiKey``'s docstring on+ why rotation is ``full_access``-only.+ * **Deletions.** Removing a workflow, deployment or Prompt Studio project+ destroys work that no inverse call restores. The write tools here are+ reversible by construction.+ """+ from mcp_server.tools.observability import (+ get_execution_detail,+ get_execution_detail_schema,+ get_usage_summary,+ get_usage_summary_schema,+ get_workflow_endpoints,+ get_workflow_endpoints_schema,+ list_executions,+ list_executions_schema,+ list_tags,+ list_tool_instances,+ list_tool_instances_schema,+ )+ from mcp_server.tools.platform import (+ _ORG_WIDE_WARNING,+ execute_pipeline,+ execute_pipeline_schema,+ list_api_deployments,+ list_pipelines,+ list_prompt_studio_projects,+ list_workflows,+ no_args_schema,+ platform_read_me_first,+ preflight_pipeline,+ set_api_deployment_active,+ set_api_deployment_active_schema,+ set_pipeline_active,+ set_pipeline_active_schema,+ whoami,+ )+ from mcp_server.tools.platform_execution import (+ get_platform_execution_status,+ platform_execution_status_schema,+ platform_extract_document,+ platform_extract_document_schema,+ preflight_extract_document,+ )+ from mcp_server.tools.prompt_studio import (+ bulk_fetch_response,+ bulk_fetch_response_schema,+ fetch_response,+ fetch_response_schema,+ index_document,+ index_document_schema,+ list_prompt_studio_documents,+ list_prompt_studio_documents_schema,+ list_prompts,+ list_prompts_schema,+ preflight_project,+ single_pass_extraction,+ single_pass_extraction_schema,+ )++ registry = MCPToolRegistry()++ registry.register(+ MCPTool(+ name="readMeFirst",+ description=(+ "START HERE. Returns a guide to this MCP server: what it can "+ "see in the connected Unstract organization, the available "+ "tools, and the recommended call sequence. Takes no arguments."+ ),+ input_schema=no_args_schema(),+ handler=platform_read_me_first,+ )+ )+ registry.register(+ MCPTool(+ name="whoami",+ description=(+ "Describe the credential this session is using: the "+ "organization it belongs to, its permission tier, and the "+ "scope of what it can see. Call this when a tool returns less "+ "or more than you expected. Takes no arguments."+ ),+ input_schema=no_args_schema(),+ handler=whoami,+ )+ )+ registry.register(+ MCPTool(+ name="listApiDeployments",+ description=(+ "List the organization's API deployments — the deployed "+ "extraction endpoints. Use this to discover what can be "+ "extracted and to find the api_name needed to connect a "+ "deployment-scoped MCP session. Takes no arguments."+ ),+ input_schema=no_args_schema(),+ handler=list_api_deployments,+ )+ )+ registry.register(+ MCPTool(+ name="listWorkflows",+ description=(+ "List the organization's workflows — the pipelines that API "+ "deployments and ETL pipelines run. Takes no arguments."+ ),+ input_schema=no_args_schema(),+ handler=list_workflows,+ )+ )+ registry.register(+ MCPTool(+ name="listPromptStudioProjects",+ description=(+ "List the organization's Prompt Studio projects, where "+ "extraction prompts are authored before being exported as "+ "tools and deployed. Takes no arguments."+ ),+ input_schema=no_args_schema(),+ handler=list_prompt_studio_projects,+ )+ )+ # Registered next to the project listing rather than beside the billable+ # tools they feed: an agent reads this list top-down, and the id-producing+ # step belongs immediately after the project it drills into.+ registry.register(+ MCPTool(+ name="listPromptStudioDocuments",+ description=(+ "List the documents uploaded to a Prompt Studio project.\n\n"+ "Call this to obtain the `document_id` that indexDocument, "+ "fetchResponse, bulkFetchResponse and singlePassExtraction "+ "require. Free to call — it reads metadata only and runs no "+ "inference."+ ),+ input_schema=list_prompt_studio_documents_schema(),+ handler=list_prompt_studio_documents,+ )+ )+ registry.register(+ MCPTool(+ name="listPrompts",+ description=(+ "List a Prompt Studio project's prompts: their keys, text, "+ "type and order.\n\n"+ "Call this to obtain the `prompt_id` that fetchResponse and "+ "bulkFetchResponse require, or to see what a project extracts "+ "before spending money running it. Free to call."+ ),+ input_schema=list_prompts_schema(),+ handler=list_prompts,+ )+ )+ registry.register(+ MCPTool(+ name="listPipelines",+ description=(+ "List the organization's ETL and task pipelines, with their "+ "schedule state and the status of their last run. Takes no "+ "arguments."+ ),+ input_schema=no_args_schema(),+ handler=list_pipelines,+ )+ )++ registry.register(+ MCPTool(+ name="setApiDeploymentActive",+ description=(+ "Activate or deactivate an API deployment. A deactivated "+ "deployment rejects extraction requests; activating it again "+ "restores service.\n\n"+ "Use this to take a misbehaving deployment offline, or to "+ "bring one back. The change is immediate and affects every "+ "caller of that deployment, not just this session.\n\n"+ f"{_ORG_WIDE_WARNING}"+ ),+ input_schema=set_api_deployment_active_schema(),+ handler=set_api_deployment_active,+ title="Activate or Deactivate API Deployment",+ writes=True,+ required_method="POST",+ # Setting active=True on an already-active deployment lands in the+ # same state; nothing new is created.+ idempotent=True,+ )+ )+ registry.register(+ MCPTool(+ name="setPipelineActive",+ description=(+ "Enable or pause a pipeline's schedule. A paused pipeline "+ "stops running on its schedule but is not deleted, and "+ "enabling it resumes the existing schedule.\n\n"+ f"{_ORG_WIDE_WARNING}"+ ),+ input_schema=set_pipeline_active_schema(),+ handler=set_pipeline_active,+ title="Enable or Pause Pipeline Schedule",+ writes=True,+ required_method="POST",+ # Pausing an already-paused schedule is a no-op, as is enabling an+ # enabled one.+ idempotent=True,+ )+ )+ registry.register(+ MCPTool(+ name="executePipeline",+ description=(+ "Trigger an immediate run of an ETL or task pipeline, "+ "independently of its schedule.\n\n"+ "This does real work: it processes whatever documents the "+ "pipeline's source currently holds and writes to its "+ "destination, consuming the organization's quota. It is not a "+ "dry run and cannot be undone — do not call it to 'test' a "+ "pipeline.\n\n"+ f"{_ORG_WIDE_WARNING}"+ ),+ input_schema=execute_pipeline_schema(),+ handler=execute_pipeline,+ writes=True,+ required_method="POST",+ billable=True,+ preflight=preflight_pipeline,+ )+ )++ # ---- observability: what happened, and what it cost ----++ registry.register(+ MCPTool(+ name="listExecutions",+ description=(+ "List recent workflow executions, newest first, with per-run "+ "status and file counts.\n\n"+ "This is the tool for 'did that run?' and 'why did it fail?'. "+ "Optionally filter by workflow_id or status."+ ),+ input_schema=list_executions_schema(),+ handler=list_executions,+ )+ )+ registry.register(+ MCPTool(+ name="getExecutionDetail",+ description=(+ "Get per-file detail for one execution — which files "+ "succeeded, which failed, and the error for each. Use after "+ "listExecutions has identified a run worth investigating."+ ),+ input_schema=get_execution_detail_schema(),+ handler=get_execution_detail,+ )+ )+ registry.register(+ MCPTool(+ name="getUsageSummary",+ description=(+ "Aggregate token and cost usage recorded for this "+ "organization. Historical accounting of what has already been "+ "spent — not a budget or a limit. Takes no arguments."+ ),+ input_schema=get_usage_summary_schema(),+ handler=get_usage_summary,+ )+ )+ registry.register(+ MCPTool(+ name="getWorkflowEndpoints",+ description=(+ "Describe how a workflow is wired: its source and destination "+ "endpoint types, and the name of the connector each uses.\n\n"+ "Connector credentials and endpoint configuration are "+ "deliberately not returned — only the shape of the connection."+ ),+ input_schema=get_workflow_endpoints_schema(),+ handler=get_workflow_endpoints,+ )+ )+ registry.register(+ MCPTool(+ name="listToolInstances",+ description=(+ "List the tool steps configured inside a workflow, in "+ "execution order. Use this to understand what a workflow "+ "actually does before running it."+ ),+ input_schema=list_tool_instances_schema(),+ handler=list_tool_instances,+ )+ )+ registry.register(+ MCPTool(+ name="listTags",+ description=(+ "List the organization's tags, which label workflow "+ "executions for grouping and filtering. Takes no arguments."+ ),+ input_schema=no_args_schema(),+ handler=list_tags,+ )+ )++ # ---- billable Prompt Studio operations ----+ #+ # Each drives real LLM inference or embedding work, so each is billable and+ # budgeted per organization by mcp_server.spend_guard.++ registry.register(+ MCPTool(+ name="indexDocument",+ description=(+ "Index a document in a Prompt Studio project so prompts can be "+ "run against it.\n\n"+ "**Costs money**: this embeds the document and writes to the "+ "vector store. Index once, then run as many prompts as you "+ "need against it — do not re-index between prompts.\n\n"+ f"{_ORG_WIDE_WARNING}"+ ),+ input_schema=index_document_schema(),+ handler=index_document,+ writes=True,+ required_method="POST",+ billable=True,+ preflight=preflight_project,+ )+ )+ registry.register(+ MCPTool(+ name="fetchResponse",+ description=(+ "Run one Prompt Studio prompt against an indexed document and "+ "return the extracted response.\n\n"+ "**Costs money**: this is a live LLM call. If you need several "+ "prompts against the same document, use bulkFetchResponse "+ "instead — it is cheaper and avoids an indexing race.\n\n"+ f"{_ORG_WIDE_WARNING}"+ ),+ input_schema=fetch_response_schema(),+ handler=fetch_response,+ writes=True,+ required_method="POST",+ billable=True,+ preflight=preflight_project,+ )+ )+ registry.register(+ MCPTool(+ name="bulkFetchResponse",+ description=(+ "Run several Prompt Studio prompts against one document in a "+ "single pass.\n\n"+ "**Costs money.** Prefer this over repeated fetchResponse "+ "calls: it indexes once and dispatches one task, which is both "+ "cheaper and avoids the 'document being indexed' race that "+ "concurrent single calls provoke.\n\n"+ f"{_ORG_WIDE_WARNING}"+ ),+ input_schema=bulk_fetch_response_schema(),+ handler=bulk_fetch_response,+ writes=True,+ required_method="POST",+ billable=True,+ preflight=preflight_project,+ )+ )+ registry.register(+ MCPTool(+ name="singlePassExtraction",+ description=(+ "Run a Prompt Studio project's entire prompt set against a "+ "document in one LLM pass.\n\n"+ "**Costs money**, and is the most expensive tool here. Use it "+ "when you want the project's full output; use fetchResponse "+ "when you want one field.\n\n"+ f"{_ORG_WIDE_WARNING}"+ ),+ input_schema=single_pass_extraction_schema(),+ handler=single_pass_extraction,+ writes=True,+ required_method="POST",+ billable=True,+ preflight=preflight_project,+ )+ )+ registry.register(+ MCPTool(+ name="extractDocument",+ description=(+ "Run a named API deployment's extraction workflow over one or "+ "more documents.\n\n"+ "**Costs money** and consumes the organization's extraction "+ "quota. Call listApiDeployments first for a valid `api_name`; "+ "the deployment fixes the prompts and output schema, so you "+ "supply documents, not instructions.\n\n"+ "The same operation is available on a deployment-scoped MCP "+ "server, where the credential reaches only one deployment. "+ "Use that one if you want a narrower blast radius.\n\n"+ f"{_ORG_WIDE_WARNING}"+ ),+ input_schema=platform_extract_document_schema(),+ handler=platform_extract_document,+ writes=True,+ required_method="POST",+ billable=True,+ # api_name is caller-supplied prose rather than an id from a+ # listing, so resolve it before the budget is claimed.+ preflight=preflight_extract_document,+ )+ )+ registry.register(+ MCPTool(+ name="getExecutionStatus",+ description=(+ "Poll for the result of an extraction started by "+ "extractDocument. Free to call.\n\n"+ "If extractDocument already returned execution_status "+ "COMPLETED, the result is in that response and no polling is "+ "needed. Otherwise poll this until COMPLETED or ERROR, "+ "pausing a few seconds between calls."+ ),+ input_schema=platform_execution_status_schema(),+ handler=get_platform_execution_status,+ # Reading a COMPLETED execution acknowledges it and the result+ # store is one-shot, so the payload cannot be fetched again.+ # Not `writes` (no configuration changes) and not `billable`+ # (no spend), but emphatically not read-only either.+ consumes_result=True,+ )+ )++ return registry+++# Registries are static, so building them once at import time is safe and keeps+# per-request work down. Kept separate rather than merged with a filter, so a+# tool cannot be exposed on the wrong server by forgetting a flag.+DEPLOYMENT_TOOLS = build_deployment_registry()+PLATFORM_TOOLS = build_platform_registry()diff --git a/backend/mcp_server/sanitize.py b/backend/mcp_server/sanitize.pynew file mode 100644index 000000000..de0908ce5--- /dev/null+++ b/backend/mcp_server/sanitize.py@@ -0,0 +1,344 @@+"""Shared input guards and output scrubbing for the MCP tool surface.++These sit above ``tools/`` rather than inside it because every tool module needs+them and none owns them. Parked in a tool module they were peers of their own+consumers, which forced function-local imports to dodge the resulting cycle —+the usual sign that a shared primitive is at the wrong layer.+"""++import logging+import re+import uuid+from typing import Any++from mcp_server.exceptions import MCPToolError++logger = logging.getLogger(__name__)++# How many rows a listing tool returns. Listings are read by an agent with a+# finite context, so an unbounded list is not more useful than a capped one —+# it just costs more and crowds out the rest of the conversation.+LIST_LIMIT = 100++_REDACTED = "[REDACTED]"++# Error text from a failed execution is one of the few places a secret can+# reach an agent without any tool asking for it: a connector that fails to+# connect may report the connection string it tried, and a provider client may+# echo the key it authenticated with. These patterns catch the common shapes.+#+# Each entry is (pattern, replacement). The replacement keeps the surrounding+# context — the key name, the URL prefix — so the message stays diagnosable+# while the value itself is gone.+_SECRET_PATTERNS = (+ # Bearer tokens first: "Authorization: Bearer <token>" would otherwise be+ # consumed by the key=value rule below, which would mask the word "Bearer"+ # and leave the token itself in place.+ (+ # ``(?i)`` applies to the whole pattern, so the token class is written+ # in one case only: spelling it ``[A-Za-z0-9._-]`` here would be the+ # same class twice over, not "both cases handled". The class is+ # deliberately ASCII-explicit rather than ``\w`` — ``\w`` is Unicode by+ # default, which would quietly widen what counts as a token character.+ re.compile(r"(?i)\b(bearer\s+)([a-z0-9._-]{8,})"),+ rf"\1{_REDACTED}",+ ),+ # key=value / key: value, where the key looks credential-ish. Stops at+ # whitespace, quote or delimiter so the rest of the message survives.+ (+ re.compile(+ # Not ``\b``: underscore is a word character, so ``\b`` finds no+ # boundary inside ``aws_secret_access_key`` and the compound names+ # that credentials actually travel under — ``aws_secret_access_key``,+ # ``db_password``, ``x-api-key`` — all failed to match, which is the+ # opposite of the intended reading of "whole word only".+ #+ # The lookbehind keeps that intent where it pays: a match still+ # cannot start mid-word after an alphanumeric, so ``hastoken`` is+ # not a hit. What it now permits is a ``_``/``-`` separated prefix.+ # Guarding against a bare-word false positive is the delimiter's+ # job anyway — ``token_count=5`` has never matched, because ``token``+ # is not followed by ``=``.+ r"(?i)(?<![A-Za-z0-9])"+ # Kept deliberately in step with ``_SECRET_KEY_NAMES`` below, whose+ # comment already claims they share a vocabulary. They did not:+ # ``secret_key``, ``private_key``, ``access_key_id`` and the plural+ # ``credentials`` were listed there as credentials and were not+ # matchable here, so the same name was redacted as a dict key and+ # left in place in free text.+ #+ # There is no bare ``key`` alternative on purpose — ``prompt_key``+ # and ``foreign_key`` are ordinary field names on these paths, and+ # this pattern also runs over *extracted document output* via+ # ``redact_structure``, where a false positive destroys real data+ # rather than merely being noisy.+ r"(password|passwd|pwd|"+ r"secret[_-]?access[_-]?key|access[_-]?key[_-]?id|access[_-]?key|"+ r"secret[_-]?key|private[_-]?key|api[_-]?key|"+ # `credential` and not `credentials?`: the plural would match+ # inside `has_credentials: true`, an ordinary boolean field on+ # these paths, because the lookbehind allows a `_` prefix and the+ # trailing `s` is what currently stops the delimiter from lining+ # up. The dict-key form is covered exactly by `_is_secret_key`,+ # which is where the plural belongs — it can require a whole-key+ # match, and free text cannot.+ r"secret|token|authorization|credential)"+ r"(\s*[=:]\s*)"+ r"([^\s,;'\"&)}\]]+)"+ ),+ rf"\1\2{_REDACTED}",+ ),+ # Credentials embedded in a URL: scheme://user:secret@host+ #+ # The userinfo is `*` not `+`: Redis URLs conventionally omit the username+ # entirely — `redis://:password@host:6379/0` — and requiring at least one+ # character before the colon meant the most common credential-bearing URL+ # in this codebase was the one shape that did not match.+ (+ re.compile(r"(?i)\b([a-z][a-z0-9+.-]*://[^\s:/@]*:)([^\s@]+)(@)"),+ rf"\1{_REDACTED}\3",+ ),+)++# Every pattern above requires one of these literals to match, so text without+# any of them can skip three regex passes that would each rebuild the whole+# string. ``redact_structure`` walks every string in an execution result, so on+# short error messages — the common case — this is most of the work avoided.+#+# **The equivalence holds for ASCII text only, and the filter is gated on that.**+# ``(?i)`` and ``str.lower()`` are different case-folding functions: Python's+# regex engine folds U+017F (ſ) to "s" and U+212A (K) to "k", while ``.lower()``+# leaves both unchanged. So ``"paſſword=hunter2"`` matches pattern 2 but contains+# no anchor once lowercased — skipping the regexes there would leak the secret.+# For ASCII input the two agree, which is what makes the fast path sound; any+# non-ASCII string takes the full regex path. ``test_redaction.SecretAnchorTest``+# pins both halves.+#+# Measured on a 500KB body: ~8x faster when pure ASCII, but **~1x as soon as a+# single non-ASCII character appears anywhere in the string**, since that flips+# the whole body onto the regex path. That is the honest cost of correctness+# here: the payload this most wants to help — ``include_extracted_text`` OCR+# output, which realistically carries accents and curly quotes — is largely the+# case that does not benefit. The gate stays regardless; a leaked credential is+# not worth 60ms.+#+# ``in`` on a str beats a combined regex alternation here — Python's regex+# engine does not do multi-literal search well.+_SECRET_ANCHORS = (+ "bearer",+ "password",+ "passwd",+ "pwd",+ "secret",+ "access_key",+ "apikey",+ "api_key",+ "api-key",+ # Every spelling the `private[_-]?key` alternative accepts. An anchor that+ # is not a substring of the text short-circuits the whole regex pass, so a+ # pattern whose keyword appears in no anchor is dead code — `private_key`+ # was exactly that, matchable in principle and never reached in practice.+ # `SecretAnchorTest` is what keeps this list honest.+ "privatekey",+ "private_key",+ "private-key",+ "token",+ "authorization",+ "credential",+ "://",+)+++# Dict keys whose value is a credential regardless of how it is formatted.+# Same vocabulary as the key=value pattern above, kept as its own tuple because+# this matches a whole key rather than a substring of free text: matching+# loosely here would redact an ordinary field like "token_count".+_SECRET_KEY_NAMES = frozenset(+ {+ "password",+ "passwd",+ "pwd",+ "secret",+ "secret_key",+ "secret_access_key",+ "access_key",+ "access_key_id",+ "apikey",+ # Hyphens are normalised to underscores before lookup, so "api-key"+ # matches via this entry rather than needing its own.+ "api_key",+ "token",+ "access_token",+ "refresh_token",+ "auth_token",+ "authorization",+ "credential",+ "credentials",+ "private_key",+ "client_secret",+ }+)+++def _normalise_key(key: str) -> str:+ """Fold a field name to snake_case for lookup.++ camelCase has to be split *before* lowercasing, or the words run together:+ ``secretAccessKey`` would become ``secretaccesskey`` and match nothing.+ That matters because AWS-style credentials arrive camelCased in exactly the+ upstream JSON these tools pass through — ``secretAccessKey``,+ ``accessKeyId``, ``clientSecret`` — so the compressed form was the common+ case rather than an exotic one.+ """+ spaced = re.sub(r"(?<=[a-z0-9])(?=[A-Z])", "_", key.strip())+ return spaced.lower().replace("-", "_").replace(" ", "_")+++def _is_secret_key(key: str) -> bool:+ """True when a dict key names a credential.++ Exact match on a normalised key, not a substring test: ``token_count`` and+ ``has_credentials`` are ordinary fields an agent needs, and redacting them+ would make output harder to act on without making it safer.+ """+ return _normalise_key(key) in _SECRET_KEY_NAMES+++def valid_uuid(value: str, field: str, hint: str) -> str:+ """Reject a malformed id before it reaches a UUID-typed ORM filter.++ Filtering a UUID column by an unparseable string raises Django's+ ``ValidationError`` from the field itself, *before* the ``is None`` check+ each call site writes — so the agent gets a generic failure instead of the+ actionable message the site already prepared, and on the preflight path it+ escaped the JSON-RPC envelope entirely.++ Returns the value unchanged so it can be used inline.+ """+ try:+ uuid.UUID(str(value))+ except (ValueError, AttributeError, TypeError) as error:+ raise MCPToolError(+ f"'{value}' is not a valid {field}. Expected a UUID. {hint}"+ ) from error+ return value+++def redact_secrets(text: str | None) -> str | None:+ """Mask credential-shaped substrings in free-form text.++ Applied to the free-form error fields the observability tools read off+ execution records (``error_message``, ``execution_error``), and — via+ ``redact_structure`` — to upstream payloads that this app does not itself+ assemble field by field.++ **``MCPToolError`` messages are not automatically redacted.** A raise site+ interpolating upstream text — ``raise MCPToolError(f"...{error}")`` — must+ call this itself; the transport returns that message to the agent verbatim.++ This is a safety net, not a guarantee — it cannot recognise a secret that+ looks like ordinary prose — so it complements the rule that structured+ fields are named explicitly rather than replacing it.+ """+ if not text:+ return text++ # Cheap literal pre-filter, ASCII-only — see _SECRET_ANCHORS for why the+ # isascii() gate is load-bearing rather than defensive.+ if text.isascii():+ lowered = text.lower()+ if not any(anchor in lowered for anchor in _SECRET_ANCHORS):+ return text++ redacted = text+ for pattern, replacement in _SECRET_PATTERNS:+ redacted = pattern.sub(replacement, redacted)+ return redacted+++def redact_structure(value: Any) -> Any:+ """Apply ``redact_secrets`` to every string inside a nested structure.++ Most tools serialize named fields, so what they return is known. The ones+ that **delegate** do not: the Prompt Studio tools, ``executePipeline`` and+ ``extractDocument`` hand back whatever the view or execution helper produced+ — ``response.data`` including non-2xx error bodies, and raw execution+ results — which can carry a failed connector's error text verbatim. Those+ paths get the same net the error fields get. (``setApiDeploymentActive`` and+ ``setPipelineActive`` write too, but build named fields, so they do not.)++ Two mechanisms, because upstream payloads carry credentials in two shapes:+ free text where ``redact_secrets``'s patterns anchor on a delimiter+ (``"connect failed: password=x"``), and structured fields where the *key*+ names the secret and the value is bare (``{"api_key": "sk-x"}``). The+ second is the ordinary shape of serializer output — the exact thing the+ delegating tools hand back — and text scrubbing alone does not see it.++ Containers are rebuilt rather than mutated, so a caller's queryset row or+ cached dict is never altered as a side effect of being reported. Sequences+ come back as plain lists — see the comment below for why the input subclass+ is deliberately not preserved.+ """+ if isinstance(value, str):+ return redact_secrets(value)+ if isinstance(value, dict):+ # A credential-named *key* whose value is a bare secret has no+ # delimiter for the text patterns to anchor on — `{"api_key": "sk-x"}`+ # is not the string `api_key=sk-x` — so the key name is what marks it.+ # This is the ordinary shape of serializer output, which is exactly+ # what the delegating tools return, so text scrubbing alone left the+ # most likely case uncovered.+ return {+ key: _REDACTED+ if isinstance(key, str) and _is_secret_key(key) and isinstance(item, str)+ else redact_structure(item)+ for key, item in value.items()+ }+ if isinstance(value, (list, tuple)):+ # Plain containers, deliberately not `type(value)(...)`. Rebuilding the+ # exact subclass breaks on anything whose __init__ is not "an iterable":+ # DRF's ReturnList requires a `serializer` kwarg, and a namedtuple takes+ # positional fields. Either would raise here — on the *return* path of a+ # billable tool, after the budget was spent and the upstream work paid+ # for. The result is json.dumps-ed by the transport, so the subclass buys+ # nothing.+ return [redact_structure(item) for item in value]+ return value+++def truncation_note(shown: int, total: int) -> dict[str, Any]:+ """Describe a capped listing, or nothing when it was complete.++ Silent truncation would read to an agent as "this is everything", which is+ exactly the sort of wrong premise it would then build on.+ """+ if total <= shown:+ return {}+ return {+ "truncated": True,+ "note": (+ f"Showing {shown} of {total}. Narrow the question or use the "+ "Unstract UI to see the rest."+ ),+ }+++def log_exception(log: logging.Logger, message: str, error: BaseException) -> None:+ """Log an upstream exception without piping its secrets into the log.++ ``logger.exception(f"...: {error}")`` writes the exception text *and* a+ traceback, both unredacted. That is a real exposure on these paths: the+ exceptions being logged come from connectors, provider clients and the+ execution stack, and a failed connection reports the string it tried. Logs+ are shipped to aggregation, so a secret there outlives the request and+ reaches an audience the credential was never scoped to.++ So the message is redacted, and the traceback is deliberately **not**+ attached: ``redact_secrets`` can only clean the string it is given, while a+ traceback renders frame locals — the very place a connection string or key+ sits. Losing the stack is the cost; the exception type and redacted message+ are kept, which is what identifies the failure. Where a stack is genuinely+ needed, log it separately from a site that controls what the exception+ holds.+ """+ log.error(f"{message}: {type(error).__name__}: {redact_secrets(str(error))}")diff --git a/backend/mcp_server/spend_guard.py b/backend/mcp_server/spend_guard.pynew file mode 100644index 000000000..6fcae4111--- /dev/null+++ b/backend/mcp_server/spend_guard.py@@ -0,0 +1,214 @@+"""Per-organization budget for billable MCP tool calls.++Some platform tools cost real money each time they run: they drive LLM+inference, embedding and vector-store writes. An agent looping over a list, or+retrying on a misread error, can spend a great deal without anyone watching. So+billable tools are budgeted per organization over a fixed window.++**Fixed, not rolling.** The counter's TTL is what expires it, so the window+opens at the first billable call and closes ``MCP_BILLABLE_WINDOW_SECONDS``+later regardless of what happens in between — it does not slide. An org that+spends its whole budget in the first minute waits out the rest of the window.++**This is a call budget, not a spend budget.** It counts invocations, not+tokens or currency. Unstract's open-source backend records usage after the fact+(``usage_v2`` aggregates what was consumed) but has no pre-flight token+allowance to check against — subscription and quota enforcement live in the+enterprise overlay, which this app cannot depend on. A call counter is the+strongest guard implementable here, and it is a blunt one: it bounds *how many+times* an agent can trigger billable work, not how expensive each call is.++Two properties are deliberate and easy to get backwards:++* **Consumption happens on invocation and is never refunded**, including when+ the tool then fails. A failed Prompt Studio call may already have spent+ tokens upstream, and refunding on failure would let an agent burn unbounded+ spend by triggering failures in a loop. This is the opposite of the+ rate-limit slot handling in ``tools/execution.py``, where the slot models+ concurrency rather than cost — see ``test_spend_guard`` for the pin.+* **Exhaustion is temporary, not a permission error.** The caller is told when+ the window resets so it can retry later, rather than concluding the tool is+ forbidden and giving up.+"""++import logging+from dataclasses import dataclass++import redis+from django.conf import settings+from django.core.cache import cache as default_cache+from django_redis.exceptions import ConnectionInterrupted++from mcp_server.sanitize import log_exception++logger = logging.getLogger(__name__)++# The failures the guard is willing to fail *open* on: the cache backend being+# unreachable or misbehaving. Deliberately not bare `Exception` — that also+# swallows TypeError, AttributeError and any future bug in this module, each of+# which would silently unbudget every billable call in the deployment with only+# a log line as signal. A real defect here should surface as a 500, not as an+# invisibly disabled spend guard.+#+# `redis.RedisError` is the load-bearing member: django-redis's `omit_exception`+# unwraps its `_main_exceptions` and re-raises the underlying cause, and those+# are redis-py's own types, which derive from `Exception` rather than+# `OSError` — so the stdlib entries below do NOT cover them. `redis` reaches+# this environment as a django-redis dependency rather than a declared one;+# `ConnectionInterrupted` is the wrapper raised when django-redis does not+# unwrap, and the stdlib entries cover backends that are not redis at all.+CACHE_ERRORS = (+ redis.RedisError,+ ConnectionInterrupted,+ ConnectionError,+ TimeoutError,+ OSError,+)++# Key namespace. The window start is not encoded in the key: the TTL is what+# expires the counter, giving a fixed window that begins at the first billable+# call rather than at a wall-clock boundary.+_KEY_PREFIX = "mcp:billable"+++def _key(org_id: str) -> str:+ return f"{_KEY_PREFIX}:{org_id}"+++def get_cache():+ """Return the cache client for MCP budget state.++ The project's shared cache, like every other cache user here — which also+ means ``override_settings(CACHES=...)`` works in tests.+ """+ return default_cache+++@dataclass(frozen=True)+class BudgetState:+ """The outcome of a budget check."""++ allowed: bool+ used: int+ limit: int+ window_seconds: int+ retry_after_seconds: int | None = None++ def message(self) -> str:+ """Explain exhaustion to the calling agent.++ Written so the agent understands this is a wait, not a refusal — and+ knows roughly how long to wait.+ """+ minutes = max(1, round((self.retry_after_seconds or self.window_seconds) / 60))+ return (+ f"This organization has used its budget for billable MCP "+ f"operations ({self.used}/{self.limit} in the last "+ f"{round(self.window_seconds / 60)} minutes). This is a temporary "+ f"limit, not a permission problem — retry in about {minutes} "+ f"minute(s), or ask an administrator to raise "+ f"MCP_BILLABLE_CALL_LIMIT."+ )+++def get_limit() -> int:+ return int(getattr(settings, "MCP_BILLABLE_CALL_LIMIT", 50))+++def get_window_seconds() -> int:+ return int(getattr(settings, "MCP_BILLABLE_WINDOW_SECONDS", 3600))+++def peek(org_id: str) -> BudgetState:+ """Report budget state without consuming any of it.++ Used by ``whoami`` so an agent can pace itself instead of discovering the+ limit by hitting it.+ """+ limit = get_limit()+ window = get_window_seconds()+ try:+ used = get_cache().get(_key(org_id)) or 0+ except CACHE_ERRORS as error:+ # Reporting the budget must never be the thing that breaks whoami —+ # `consume` fails open for the same reason, and a read is even less+ # load-bearing than a claim.+ # Redacted: a cache error can render the Redis URL, which carries a+ # password in any deployment that sets one.+ log_exception(logger, f"MCP spend guard unreadable for org '{org_id}'", error)+ used = 0+ return BudgetState(+ allowed=used < limit, used=used, limit=limit, window_seconds=window+ )+++def consume(org_id: str) -> BudgetState:+ """Claim one billable call for this organization.++ Returns a state with ``allowed=False`` when the budget is already spent, in+ which case nothing is consumed and the caller must not run the tool.++ Fails **open** if the cache is unavailable: this guard exists to stop an+ agent looping, not to be a hard licence check, and taking the whole MCP+ surface offline because Redis blipped would be a worse failure than briefly+ unbudgeted calls. The event is logged so it is visible.+ """+ limit = get_limit()+ window = get_window_seconds()+ key = _key(org_id)+ cache = get_cache()++ try:+ # `add` only sets when absent, so it both initialises the counter and+ # starts the window's TTL without clobbering an in-flight window.+ cache.add(key, 0, timeout=window)+ used = cache.incr(key)+ except ValueError:+ # The key expired between `add` and `incr`. Re-seed and count this call+ # as the first of a fresh window.+ try:+ cache.set(key, 1, timeout=window)+ except CACHE_ERRORS as error:+ # Guarded for the same reason the outer handler is: an unguarded+ # set here would fail *closed* — a 500 on a billable call — which+ # is the opposite of this function's documented behaviour, and+ # reachable whenever Redis drops during the add/incr race.+ log_exception(+ logger,+ f"MCP spend guard unavailable for org '{org_id}' while re-seeding, "+ "allowing call",+ error,+ )+ return BudgetState(allowed=True, used=0, limit=limit, window_seconds=window)+ used = 1+ except CACHE_ERRORS as error:+ log_exception(+ logger,+ f"MCP spend guard unavailable for org '{org_id}', allowing call",+ error,+ )+ return BudgetState(allowed=True, used=0, limit=limit, window_seconds=window)++ if used > limit:+ # Over budget. The counter is deliberately left incremented: decrementing+ # here would let a caller probe the limit repeatedly at no cost.+ ttl = None+ try:+ ttl = cache.ttl(key)+ except (*CACHE_ERRORS, AttributeError, NotImplementedError):+ # Cosmetic only: `ttl` sharpens the retry hint. AttributeError+ # covers a backend without `.ttl` (LocMemCache in tests), which is+ # not a fault worth failing a budget check over.+ pass+ logger.warning(+ f"MCP billable budget exhausted for org '{org_id}': {used}/{limit}"+ )+ return BudgetState(+ allowed=False,+ used=used,+ limit=limit,+ window_seconds=window,+ retry_after_seconds=ttl,+ )++ return BudgetState(allowed=True, used=used, limit=limit, window_seconds=window)diff --git a/backend/mcp_server/tests/__init__.py b/backend/mcp_server/tests/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/mcp_server/tests/test_input_guards.py b/backend/mcp_server/tests/test_input_guards.pynew file mode 100644index 000000000..928bd93a1--- /dev/null+++ b/backend/mcp_server/tests/test_input_guards.py@@ -0,0 +1,382 @@+"""Guards that turn a malformed request into something the agent can act on.++Three layers, each covered here because each was added or changed without+coverage and each fails invisibly:++* ``valid_uuid`` — rejects an unparseable id *before* it reaches a UUID-typed+ ORM filter, where Django's field would raise ``ValidationError`` ahead of the+ call site's own ``is None`` check.+* the transport's JSON-RPC envelope guards — malformed body, non-object+ ``params``, and a preflight that raises something unexpected. Each exists to+ stop a Django 500 reaching a client that is parsing for an envelope.+* the platform view's CSRF gate — exempting only bearer-authenticated requests.+"""++from __future__ import annotations++import json+from unittest.mock import MagicMock, patch++from django.test import SimpleTestCase+from rest_framework.test import APIRequestFactory++from mcp_server.constants import JSONRPC+from mcp_server.exceptions import MCPToolError+from mcp_server.registry import MCPTool, MCPToolRegistry+from mcp_server.sanitize import valid_uuid+from mcp_server.transport import BaseMCPView++GOOD_UUID = "88888888-8888-8888-8888-888888888888"+++class ValidUuidTest(SimpleTestCase):+ def test_a_valid_uuid_passes_through_unchanged(self) -> None:+ assert valid_uuid(GOOD_UUID, "execution id", "hint") == GOOD_UUID++ def test_malformed_ids_are_agent_errors(self) -> None:+ for bad in ("not-a-uuid", "", "1234", None, 42):+ with self.subTest(bad):+ with self.assertRaises(MCPToolError) as caught:+ valid_uuid(bad, "execution id", "Call listExecutions.")+ message = str(caught.exception)+ assert "not a valid execution id" in message+ # The hint is what makes this actionable rather than just a+ # refusal, so it is part of the contract.+ assert "Call listExecutions." in message++ def test_the_field_name_and_hint_reach_the_message(self) -> None:+ with self.assertRaises(MCPToolError) as caught:+ valid_uuid("x", "pipeline id", "Call listPipelines to see valid ids.")++ assert "pipeline id" in str(caught.exception)+ assert "listPipelines" in str(caught.exception)+++class ExecutionScopeGuardTest(SimpleTestCase):+ """The call site refuses before touching the ORM."""++ def test_a_malformed_execution_id_never_reaches_the_filter(self) -> None:+ from mcp_server.tools.execution import get_execution_status++ context = MagicMock()+ context.api.workflow_id = "wf"++ with (+ patch(+ "mcp_server.tools.execution.WorkflowExecution.objects.filter"+ ) as scope_filter,+ patch(+ "mcp_server.tools.execution.ExecutionQuerySerializer.is_valid",+ return_value=True,+ ),+ patch(+ "mcp_server.tools.execution.ExecutionQuerySerializer.validated_data",+ {+ "execution_id": "not-a-uuid",+ "include_metadata": False,+ "include_metrics": False,+ "include_extracted_text": False,+ },+ ),+ ):+ with self.assertRaises(MCPToolError):+ get_execution_status(context, execution_id="not-a-uuid")++ scope_filter.assert_not_called()+++class TransportEnvelopeGuardTest(SimpleTestCase):+ """Malformed input comes back as JSON-RPC, never as a Django error page."""++ def _view(self, registry=None):+ reg = registry or MCPToolRegistry()++ class Probe(BaseMCPView):+ authentication_classes: list = []+ permission_classes: list = []+ registry = reg++ def resolve_context(self, **kwargs):+ return MagicMock()++ def server_instructions(self, context):+ return ""++ return Probe.as_view()++ def _post(self, body: str):+ request = APIRequestFactory().post(+ "/mcp/", data=body, content_type="application/json"+ )+ return self._view()(request)++ def test_malformed_json_is_a_parse_error_envelope(self) -> None:+ """DRF would answer 400 with its own body; a JSON-RPC client is+ parsing for an envelope and would see a transport fault instead.+ """+ response = self._post("{not json")+ body = json.loads(response.content)++ assert body["error"]["code"] == JSONRPC.PARSE_ERROR+ # Unparseable input carries no id, so the spec requires null.+ assert body["id"] is None++ def test_non_object_params_is_an_invalid_params_envelope(self) -> None:+ """A JSON array is valid JSON-RPC and survives `or {}`, then reaches+ params.get() downstream as an AttributeError — a Django 500.+ """+ response = self._post(+ json.dumps(+ {"jsonrpc": "2.0", "id": 1, "method": "tools/list", "params": [1, 2]}+ )+ )+ body = json.loads(response.content)++ assert body["error"]["code"] == JSONRPC.INVALID_PARAMS+ assert body["id"] == 1++ def test_a_scalar_params_is_also_refused(self) -> None:+ response = self._post(+ json.dumps({"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": "x"})+ )+ body = json.loads(response.content)++ assert body["error"]["code"] == JSONRPC.INVALID_PARAMS++ def test_absent_params_is_still_accepted(self) -> None:+ """`params` is optional — the guard must not reject its absence."""+ response = self._post(+ json.dumps({"jsonrpc": "2.0", "id": 3, "method": "tools/list"})+ )+ body = json.loads(response.content)++ assert "error" not in body+ assert body["id"] == 3++ def test_a_preflight_raising_an_unexpected_error_stays_in_the_envelope(+ self,+ ) -> None:+ """A resolver filtering a UUID column by a malformed string raises+ Django ValidationError. Before the catch-all it escaped post() and the+ client got no envelope at all.+ """+ from django.core.exceptions import ValidationError++ def exploding_preflight(context, **kwargs):+ raise ValidationError("bad uuid")++ registry = MCPToolRegistry()+ registry.register(+ MCPTool(+ name="probeTool",+ description="",+ input_schema={"type": "object", "properties": {}},+ handler=lambda context, **kw: {"ok": True},+ preflight=exploding_preflight,+ )+ )++ request = APIRequestFactory().post(+ "/mcp/",+ data=json.dumps(+ {+ "jsonrpc": "2.0",+ "id": 4,+ "method": "tools/call",+ "params": {"name": "probeTool", "arguments": {}},+ }+ ),+ content_type="application/json",+ )+ response = self._view(registry)(request)+ body = json.loads(response.content)++ assert body["error"]["code"] == JSONRPC.TOOL_EXECUTION_ERROR+ assert body["id"] == 4+++class PlatformCsrfGateTest(SimpleTestCase):+ """CSRF is waived for the bearer key, not for everything.++ ``CustomAuthMiddleware`` falls through to session auth on this path, so an+ unconditional exemption would let a logged-in browser session reach the+ org-wide write and billable surface with CSRF off.+ """++ def _initialize(self, with_key: bool):+ from mcp_server.platform_views import PlatformMCPServerView++ request = APIRequestFactory().post("/api/v1/unstract/org/mcp/", data={})+ if with_key:+ request.platform_api_key = MagicMock()++ view = PlatformMCPServerView()+ view.initialize_request(request)+ return request++ def test_a_bearer_request_is_exempt(self) -> None:+ request = self._initialize(with_key=True)++ assert getattr(request, "csrf_processing_done", False) is True++ def test_a_session_request_is_not_exempt(self) -> None:+ request = self._initialize(with_key=False)++ assert getattr(request, "csrf_processing_done", False) is not True+++class PlatformPollDeploymentResolutionTest(SimpleTestCase):+ """The platform poll resolves the deployment the execution actually used.++ `get_platform_execution_status` finds the execution, resolves its deployment,+ then delegates to the deployment-scoped `get_execution_status`. That inner+ check filters on `pipeline_id`, so if the outer resolution picks a *different*+ deployment sharing the same workflow, the inner check rejects an execution the+ caller legitimately owns.++ `APIDeployment.workflow` has no uniqueness constraint (`related_name="apis"`),+ so "different deployment, same workflow" is a supported configuration rather+ than a hypothetical.+ """++ def test_the_deployment_is_resolved_by_pipeline_id(self) -> None:+ from mcp_server.tools import platform_execution++ execution = MagicMock(pipeline_id="dep-that-ran", workflow_id="shared-wf")+ context = MagicMock()+ context.org_name = "org-mcp"++ with (+ patch(+ "mcp_server.tools.observability._org_workflow_ids", return_value=["w"]+ ),+ patch(+ "workflow_manager.workflow_v2.models.execution."+ "WorkflowExecution.objects.filter"+ ) as exec_filter,+ patch.object(platform_execution, "APIDeployment") as deployment_model,+ patch(+ "mcp_server.tools.execution.get_execution_status",+ return_value={"ok": True},+ ),+ ):+ exec_filter.return_value.only.return_value.first.return_value = execution+ resolver = deployment_model.objects.for_user.return_value.filter+ resolver.return_value.first.return_value = MagicMock()++ platform_execution.get_platform_execution_status(+ context, execution_id=GOOD_UUID+ )++ # The lookup must name the recorded deployment, not the workflow: a+ # workflow filter returns an arbitrary sibling and the delegated check+ # then refuses the caller's own execution.+ resolver.assert_called_once_with(id="dep-that-ran")++ def test_a_non_deployment_execution_is_still_refused(self) -> None:+ """Pipeline and ETL runs have no APIDeployment, and must not resolve to+ an unrelated one just because they share a workflow.+ """+ from mcp_server.tools import platform_execution++ execution = MagicMock(pipeline_id=None, workflow_id="shared-wf")+ context = MagicMock()+ context.org_name = "org-mcp"++ with (+ patch(+ "mcp_server.tools.observability._org_workflow_ids", return_value=["w"]+ ),+ patch(+ "workflow_manager.workflow_v2.models.execution."+ "WorkflowExecution.objects.filter"+ ) as exec_filter,+ patch.object(platform_execution, "APIDeployment") as deployment_model,+ ):+ exec_filter.return_value.only.return_value.first.return_value = execution+ resolver = deployment_model.objects.for_user.return_value.filter+ resolver.return_value.first.return_value = None++ with self.assertRaises(MCPToolError) as caught:+ platform_execution.get_platform_execution_status(+ context, execution_id=GOOD_UUID+ )++ assert "did not come from an API deployment" in str(caught.exception)+++class ExecutionDetailDegradationTest(SimpleTestCase):+ """A partial `files` list must announce itself.++ `getExecutionDetail` answers "which files failed, and why". If the per-file+ load fails it keeps whatever it got — some detail beats none — but a short+ or empty list then reads to an agent as "no files failed", which is the+ opposite of the truth and the exact wrong premise for it to act on.+ """++ def _execution(self, total_files: int = 3):+ execution = MagicMock()+ execution.id = GOOD_UUID+ execution.workflow_id = "wf"+ execution.status = "ERROR"+ execution.total_files = total_files+ execution.successful_files = 0+ execution.failed_files = total_files+ execution.error_message = ""+ execution.execution_time = 1+ execution.created_at = "2026-01-01"+ return execution++ def _run(self, execution):+ from mcp_server.tools import observability++ context = MagicMock()+ context.org_name = "org-mcp"++ with (+ patch.object(observability, "_org_workflow_ids", return_value=["w"]),+ patch.object(observability.WorkflowExecution.objects, "filter") as f,+ ):+ f.return_value.first.return_value = execution+ return observability.get_execution_detail(context, execution_id=GOOD_UUID)++ def test_a_failed_file_load_is_announced(self) -> None:+ execution = self._execution()+ execution.file_executions.all.side_effect = RuntimeError("db gone")++ result = self._run(execution)++ assert result["files_complete"] is False+ assert "incomplete" in result["files_note"]+ # The count the agent should trust instead is still present.+ assert result["failed_files"] == 3++ def test_a_complete_load_says_nothing(self) -> None:+ """The marker must not appear on the happy path, or it means nothing."""+ execution = self._execution(total_files=1)+ row = MagicMock(id="f1", file_name="a.pdf", status="ERROR", execution_error="")+ execution.file_executions.all.return_value = [row]++ result = self._run(execution)++ assert "files_complete" not in result+ assert "files_note" not in result+ assert len(result["files"]) == 1++ def test_a_capped_list_is_also_announced(self) -> None:+ """Same wrong premise, different cause: hitting LOG_LIMIT truncates+ silently unless the response says so.+ """+ from mcp_server.tools.observability import LOG_LIMIT++ execution = self._execution(total_files=LOG_LIMIT + 5)+ rows = [+ MagicMock(id=f"f{i}", file_name=f"{i}.pdf", status="ERROR", execution_error="")+ for i in range(LOG_LIMIT)+ ]+ execution.file_executions.all.return_value = rows++ result = self._run(execution)++ assert result["files_complete"] is False+ assert str(LOG_LIMIT) in result["files_note"]diff --git a/backend/mcp_server/tests/test_mcp_auth.py b/backend/mcp_server/tests/test_mcp_auth.pynew file mode 100644index 000000000..8ea623f7d--- /dev/null+++ b/backend/mcp_server/tests/test_mcp_auth.py@@ -0,0 +1,249 @@+"""Critical path ``mcp-server-auth``: the hosted MCP endpoint rejects+unauthenticated callers before any tool runs.++``POST /mcp/<org>/<api_name>/`` is unauthenticated at the DRF layer — like the+deployment execution endpoint it mirrors, it is guarded entirely by the API key+check inside the view. A regression there exposes every registered tool,+including the one that spends the organization's extraction quota. These tests+assert rejection lands before the tool handler is reached. Needs a live DB+(integration tier).+"""++from __future__ import annotations++import json+import uuid+from unittest.mock import patch++import pytest+from account_v2.models import Organization+from api_v2.models import APIDeployment, APIKey+from django.test import TestCase+from rest_framework.test import APIRequestFactory+from utils.user_context import UserContext+from workflow_manager.workflow_v2.models.workflow import Workflow++from mcp_server.views import MCPServerView++ORG_ID = "org-mcp"+++def rpc_body(response):+ """Decode the JSON-RPC envelope from a rendered response."""+ return json.loads(response.content)+++class MCPServerAuthTest(TestCase):+ def setUp(self) -> None:+ self.org = Organization.objects.create(+ name=ORG_ID, display_name="Org MCP", organization_id=ORG_ID+ )+ UserContext.set_organization_identifier(ORG_ID)+ workflow = Workflow.objects.create(workflow_name="wf-mcp", is_active=True)++ self.api = APIDeployment.objects.create(api_name="live-api", workflow=workflow)+ self.key = APIKey.objects.create(api=self.api)+ self.inactive_key = APIKey.objects.create(api=self.api, is_active=False)++ self.inactive_api = APIDeployment.objects.create(+ api_name="dead-api", workflow=workflow, is_active=False+ )+ self.other_api = APIDeployment.objects.create(+ api_name="other-api", workflow=workflow+ )+ self.other_key = APIKey.objects.create(api=self.other_api)++ self.view = MCPServerView.as_view()+ self.factory = APIRequestFactory()++ def _post(+ self,+ api_name: str,+ auth: str | None,+ org: str = ORG_ID,+ payload: dict | None = None,+ ):+ headers = {"HTTP_AUTHORIZATION": auth} if auth is not None else {}+ body = (+ payload+ if payload is not None+ else {+ "jsonrpc": "2.0",+ "id": 1,+ "method": "tools/list",+ }+ )+ request = self.factory.post(+ f"/deployment/api/{org}/{api_name}/mcp", body, format="json", **headers+ )+ return self.view(request, org_name=org, api_name=api_name)++ @pytest.mark.critical_path("mcp-server-auth")+ @patch("mcp_server.registry.DEPLOYMENT_TOOLS.get")+ def test_bad_credentials_rejected_before_dispatch(self, registry_get) -> None:+ cases = [+ ("missing header", "live-api", None, ORG_ID),+ ("no bearer prefix", "live-api", f"Token {self.key.api_key}", ORG_ID),+ ("empty bearer", "live-api", "Bearer ", ORG_ID),+ ("unknown key", "live-api", f"Bearer {uuid.uuid4()}", ORG_ID),+ ("not a uuid", "live-api", "Bearer not-a-uuid", ORG_ID),+ ("inactive key", "live-api", f"Bearer {self.inactive_key.api_key}", ORG_ID),+ (+ "key of another api",+ "live-api",+ f"Bearer {self.other_key.api_key}",+ ORG_ID,+ ),+ ("unknown api", "ghost-api", f"Bearer {self.key.api_key}", ORG_ID),+ ("inactive api", "dead-api", f"Bearer {self.key.api_key}", ORG_ID),+ ("wrong org", "live-api", f"Bearer {self.key.api_key}", "no-such-org"),+ ]+ for label, api_name, auth, org in cases:+ with self.subTest(label):+ response = self._post(api_name, auth, org)+ assert response.status_code == 401, response.content+ assert rpc_body(response)["error"]["code"] == -32001++ registry_get.assert_not_called()++ @pytest.mark.critical_path("mcp-server-auth")+ def test_valid_key_reaches_tool_listing(self) -> None:+ """Guard the inverse: a check that rejected everything would pass all+ the rejection cases above.+ """+ response = self._post("live-api", f"Bearer {self.key.api_key}")++ assert response.status_code == 200, response.content+ tools = rpc_body(response)["result"]["tools"]+ assert [tool["name"] for tool in tools] == [+ "readMeFirst",+ "getApiInfo",+ "extractDocument",+ "getExecutionStatus",+ ]++ @pytest.mark.critical_path("mcp-server-auth")+ def test_no_route_accepts_the_api_key_in_the_url(self) -> None:+ """The credential travels in a header, never in the path.++ An earlier revision offered `/mcp/<api_key>/` for "clients that cannot+ set headers". That class does not exist for HTTP transports — the spec+ requires `Authorization` on every authenticated request and forbids+ tokens in the URI — while a key in the path reaches access logs, APM+ traces and intermediaries a header never touches.++ Asserted at the URL layer rather than the view: a route resolving here+ again is the regression, and it would not be caught by the view alone.+ """+ from django.urls import Resolver404, resolve++ base = f"/deployment/api/{ORG_ID}/live-api"++ with self.assertRaises(Resolver404):+ resolve(f"{base}/mcp/{self.key.api_key}/")+ with self.assertRaises(Resolver404):+ resolve(f"{base}/mcp/{uuid.uuid4()}")++ # The header route still works, so this is a narrowing rather than a+ # break.+ assert resolve(f"{base}/mcp").url_name == "mcp_server"++ @pytest.mark.critical_path("mcp-server-auth")+ def test_a_key_in_the_path_does_not_authenticate(self) -> None:+ """Belt and braces: even if a route were re-added, the view ignores it.++ `resolve_context` reads the bearer header only, so a caller supplying+ the key some other way is rejected like any unauthenticated request.+ """+ request = self.factory.post(+ f"/deployment/api/{ORG_ID}/live-api/mcp/{self.key.api_key}/",+ {"jsonrpc": "2.0", "id": 1, "method": "tools/list"},+ format="json",+ )+ response = self.view(request, org_name=ORG_ID, api_name="live-api")++ assert response.status_code == 401, response.content++ @pytest.mark.critical_path("mcp-server-auth")+ def test_the_bearer_scheme_is_case_insensitive(self) -> None:+ """RFC 9110 makes the auth scheme case-insensitive, and DRF's own+ authenticators lowercase it before comparing. A client sending+ `bearer <key>` is conformant, so rejecting it would be our bug.+ """+ for scheme in ("Bearer", "bearer", "BEARER", "BeArEr"):+ with self.subTest(scheme):+ response = self._post("live-api", f"{scheme} {self.key.api_key}")+ assert response.status_code == 200, response.content++ @pytest.mark.critical_path("mcp-server-auth")+ def test_a_non_bearer_scheme_is_still_refused(self) -> None:+ """Case-insensitivity must not turn into accepting any scheme."""+ for header in (+ f"Basic {self.key.api_key}",+ f"Token {self.key.api_key}",+ str(self.key.api_key),+ ):+ with self.subTest(header):+ assert self._post("live-api", header).status_code == 401++ @pytest.mark.critical_path("mcp-server-auth")+ def test_mcp_url_hangs_off_the_execution_url(self) -> None:+ """Pins the endpoint's address and its relationship to the REST one.++ The MCP endpoint is a suffix on the deployment's own execution URL, so+ the two are visibly the same resource. It must also stay inside the+ whitelisted deployment prefix: this server authenticates a deployment+ key itself, and routing it somewhere the session middleware applies+ would break every client.+ """+ from django.conf import settings+ from django.test import Client+ from django.urls import resolve++ base = f"/deployment/api/{ORG_ID}/live-api"++ assert resolve(f"{base}/mcp").url_name == "mcp_server"+ # The execution endpoint must still resolve to execution, not MCP.+ assert resolve(f"{base}/").url_name == "api_deployment_execution"++ assert any(+ f"{base}/mcp".startswith(path) for path in settings.WHITELISTED_PATHS+ ), "deployment MCP path must stay inside the whitelisted deployment prefix"++ # And it must really answer there, not merely resolve.+ response = Client().post(+ f"{base}/mcp",+ data=json.dumps({"jsonrpc": "2.0", "id": 1, "method": "tools/list"}),+ content_type="application/json",+ HTTP_AUTHORIZATION=f"Bearer {self.key.api_key}",+ )+ assert response.status_code == 200, response.content+ assert "readMeFirst" in response.content.decode()++ @pytest.mark.critical_path("mcp-server-auth")+ def test_get_probe_does_not_leak_deployment_details(self) -> None:+ """The unauthenticated GET probe advertises the server, not the+ deployment behind it.++ It answers 405: a Streamable HTTP client issues GET to open an SSE+ stream, and this server pushes nothing, so declining is the conformant+ response. The identity body rides along for uptime probes.++ ``Allow`` is asserted at the value DRF actually produces, not the+ ``"POST"`` a handler would like to set: ``finalize_response`` overwrites+ any handler value with ``self.allowed_methods``. In the real WSGI stack+ ``RemoveAllowHeaderMiddleware`` then pops the header entirely, but this+ test drives the view through ``APIRequestFactory``, which runs no+ middleware — so the header is visible here and worth pinning.+ """+ request = self.factory.get(f"/mcp/{ORG_ID}/live-api/")+ response = self.view(request, org_name=ORG_ID, api_name="live-api")+ body = json.loads(response.content)++ assert response.status_code == 405+ assert "POST" in response["Allow"]+ assert body["name"] == "unstract"+ # The whole rendered body, not just the parsed identity: the point of+ # this test is that nothing anywhere in the response names the tenant.+ assert "live-api" not in response.content.decode()+ assert ORG_ID not in response.content.decode()diff --git a/backend/mcp_server/tests/test_mcp_protocol.py b/backend/mcp_server/tests/test_mcp_protocol.pynew file mode 100644index 000000000..7d55f6efd--- /dev/null+++ b/backend/mcp_server/tests/test_mcp_protocol.py@@ -0,0 +1,289 @@+"""JSON-RPC protocol behaviour of the hosted MCP server.++Auth is stubbed here so these stay in the unit tier; the auth boundary itself+is covered by ``test_mcp_auth``. What is checked is the wire contract an MCP+client depends on: envelope shape, method routing, and the error mapping that+decides whether a client retries, re-prompts, or gives up.+"""++from __future__ import annotations++import json+from unittest.mock import patch++from django.test import SimpleTestCase+from rest_framework.test import APIRequestFactory++from mcp_server.constants import JSONRPC, MCPServer+from mcp_server.context import MCPContext+from mcp_server.exceptions import MCPToolError+from mcp_server.registry import DEPLOYMENT_TOOLS+from mcp_server.transport import negotiate_protocol_version+from mcp_server.views import MCPServerView++ORG_ID = "org-mcp"+API_NAME = "live-api"+++class FakeWorkflow:+ id = "11111111-1111-1111-1111-111111111111"+ workflow_name = "wf-mcp"+++class FakeAPI:+ """Stands in for an APIDeployment so these tests need no database."""++ id = "22222222-2222-2222-2222-222222222222"+ display_name = "Invoice Extractor"+ api_name = API_NAME+ description = "Extracts invoice fields"+ is_active = True+ workflow = FakeWorkflow()+ workflow_id = FakeWorkflow.id+++class MCPProtocolTest(SimpleTestCase):+ def setUp(self) -> None:+ self.view = MCPServerView.as_view()+ self.factory = APIRequestFactory()+ self.context = MCPContext(+ api=FakeAPI(), api_key="test-key", org_name=ORG_ID+ )++ def _failing_tool(self, error: Exception, name: str = "getApiInfo"):+ """Swap a registered tool for one that raises.++ The registry stores handler references captured at import, so patching+ the tool's module attribute would not affect an already-built registry —+ the substitution has to happen at the registry lookup.+ """+ from dataclasses import replace++ def boom(*args, **kwargs):+ raise error++ original = DEPLOYMENT_TOOLS.get(name)+ return patch.object(+ DEPLOYMENT_TOOLS, "get", return_value=replace(original, handler=boom)+ )++ def _call(self, body):+ """POST a JSON-RPC body with auth stubbed out."""+ request = self.factory.post(f"/mcp/{ORG_ID}/{API_NAME}/", body, format="json")+ with patch.object(+ MCPServerView, "resolve_context", return_value=self.context+ ):+ response = self.view(+ request, org_name=ORG_ID, api_name=API_NAME, api_key="test-key"+ )+ return response, json.loads(response.content)++ def test_initialize_returns_protocol_and_server_info(self) -> None:+ response, body = self._call(+ {"jsonrpc": "2.0", "id": 1, "method": "initialize", "params": {}}+ )++ assert response.status_code == 200+ assert body["id"] == 1+ result = body["result"]+ assert result["protocolVersion"] == MCPServer.PROTOCOL_VERSION+ assert result["serverInfo"]["name"] == MCPServer.NAME+ # The agent is told which deployment it is attached to, so it does not+ # have to guess what the tools will operate on.+ assert "Invoice Extractor" in result["instructions"]++ def test_initialize_echoes_the_clients_requested_revision(self) -> None:+ """Through the real view, so the handler is proven wired to the+ negotiation and not merely to the constant.+ """+ _, body = self._call(+ {+ "jsonrpc": "2.0",+ "id": 1,+ "method": "initialize",+ "params": {"protocolVersion": "2024-11-05"},+ }+ )++ assert body["result"]["protocolVersion"] == "2024-11-05"++ def test_initialize_offers_our_revision_when_the_client_asks_for_junk(self) -> None:+ _, body = self._call(+ {+ "jsonrpc": "2.0",+ "id": 1,+ "method": "initialize",+ "params": {"protocolVersion": "1.0.0"},+ }+ )++ assert body["result"]["protocolVersion"] == MCPServer.PROTOCOL_VERSION++ def test_notification_gets_no_result_body(self) -> None:+ """Notifications carry no id; answering one with a result would be a+ protocol violation that some clients treat as a fatal error.+ """+ response, body = self._call(+ {"jsonrpc": "2.0", "method": "notifications/initialized"}+ )++ assert response.status_code == 202+ assert body == {}++ def test_ping_returns_empty_result(self) -> None:+ _, body = self._call({"jsonrpc": "2.0", "id": 7, "method": "ping"})+ assert body["result"] == {}++ def test_wrong_jsonrpc_version_rejected(self) -> None:+ _, body = self._call({"jsonrpc": "1.0", "id": 1, "method": "ping"})+ assert body["error"]["code"] == JSONRPC.INVALID_REQUEST++ def test_missing_method_rejected(self) -> None:+ _, body = self._call({"jsonrpc": "2.0", "id": 1})+ assert body["error"]["code"] == JSONRPC.INVALID_REQUEST++ def test_unknown_method_rejected(self) -> None:+ _, body = self._call(+ {"jsonrpc": "2.0", "id": 1, "method": "resources/list"}+ )+ assert body["error"]["code"] == JSONRPC.METHOD_NOT_FOUND++ def test_unknown_tool_lists_available_tools(self) -> None:+ """The error names the real tools, so a mistaken agent can correct+ itself without another round trip.+ """+ _, body = self._call(+ {+ "jsonrpc": "2.0",+ "id": 1,+ "method": "tools/call",+ "params": {"name": "deleteEverything"},+ }+ )++ assert body["error"]["code"] == JSONRPC.METHOD_NOT_FOUND+ assert "readMeFirst" in body["error"]["data"]++ def test_tool_result_is_wrapped_in_mcp_content(self) -> None:+ _, body = self._call(+ {+ "jsonrpc": "2.0",+ "id": 1,+ "method": "tools/call",+ "params": {"name": "getApiInfo", "arguments": {}},+ }+ )++ result = body["result"]+ assert result["isError"] is False+ payload = json.loads(result["content"][0]["text"])+ assert payload["display_name"] == "Invoice Extractor"+ assert payload["workflow"]["name"] == "wf-mcp"++ def test_tool_error_returns_actionable_result_not_protocol_error(self) -> None:+ """An MCPToolError is the agent's problem to fix, so it comes back as+ an isError result the agent can read — not a protocol error, which+ many clients surface as an unrecoverable transport fault.+ """+ with self._failing_tool(MCPToolError("Deployment is not active")):+ _, body = self._call(+ {+ "jsonrpc": "2.0",+ "id": 1,+ "method": "tools/call",+ "params": {"name": "getApiInfo", "arguments": {}},+ }+ )++ assert "error" not in body+ assert body["result"]["isError"] is True+ assert "not active" in body["result"]["content"][0]["text"]++ def test_unexpected_tool_failure_does_not_leak_internals(self) -> None:+ with self._failing_tool(+ RuntimeError("psycopg2 connection string user=admin")+ ):+ _, body = self._call(+ {+ "jsonrpc": "2.0",+ "id": 1,+ "method": "tools/call",+ "params": {"name": "getApiInfo", "arguments": {}},+ }+ )++ assert body["error"]["code"] == JSONRPC.TOOL_EXECUTION_ERROR+ assert "psycopg2" not in json.dumps(body)++ def test_unexpected_argument_reported_as_invalid_params(self) -> None:+ _, body = self._call(+ {+ "jsonrpc": "2.0",+ "id": 1,+ "method": "tools/call",+ "params": {"name": "getApiInfo", "arguments": {"nope": 1}},+ }+ )++ assert body["error"]["code"] == JSONRPC.INVALID_PARAMS++ def test_batch_request_rejected(self) -> None:+ request = self.factory.post(+ f"/mcp/{ORG_ID}/{API_NAME}/",+ [{"jsonrpc": "2.0", "id": 1, "method": "ping"}],+ format="json",+ )+ with patch.object(+ MCPServerView, "resolve_context", return_value=self.context+ ):+ response = self.view(+ request, org_name=ORG_ID, api_name=API_NAME, api_key="test-key"+ )++ body = json.loads(response.content)+ assert body["error"]["code"] == JSONRPC.INVALID_REQUEST+++class ProtocolVersionNegotiationTest(SimpleTestCase):+ """`initialize` echoes the client's revision when it is one we speak.++ The spec's rule is an echo, not an announcement — a server that always+ replies with its own preferred revision silently tells every client on an+ older one to disconnect. Tested against the pure function rather than the+ view so the cases stay readable; the view path is covered above.+ """++ def test_a_supported_revision_is_echoed_back(self) -> None:+ for revision in MCPServer.SUPPORTED_PROTOCOL_VERSIONS:+ assert negotiate_protocol_version(revision) == revision++ def test_the_older_revision_is_still_accepted(self) -> None:+ """Named explicitly rather than left to the loop above.++ The request/response subset this server implements is identical across+ the two revisions, so dropping the older one would disconnect working+ clients for no behavioural reason.+ """+ assert negotiate_protocol_version("2024-11-05") == "2024-11-05"++ def test_an_unknown_revision_is_answered_with_our_preferred_one(self) -> None:+ """Not an error: the spec has the server offer what it does support and+ lets the client decide whether to continue.+ """+ assert negotiate_protocol_version("1.0.0") == MCPServer.PROTOCOL_VERSION+ assert negotiate_protocol_version("2099-01-01") == MCPServer.PROTOCOL_VERSION++ def test_a_missing_or_malformed_version_falls_back(self) -> None:+ """Clients in the wild omit the field or send the wrong type; neither+ should fail the handshake before any tool is reachable.+ """+ for bad in (None, 20250618, {"version": "2025-06-18"}, []):+ assert negotiate_protocol_version(bad) == MCPServer.PROTOCOL_VERSION++ def test_the_preferred_revision_is_the_one_the_transport_implements(self) -> None:+ """The bug this replaced: the server advertised 2024-11-05 while the+ transport followed 2025-06-18 rules (405 on GET rather than an SSE+ stream). The version string must track the behaviour.+ """+ assert MCPServer.PROTOCOL_VERSION == "2025-06-18"+ assert MCPServer.SUPPORTED_PROTOCOL_VERSIONS[0] == MCPServer.PROTOCOL_VERSIONdiff --git a/backend/mcp_server/tests/test_no_credential_leak.py b/backend/mcp_server/tests/test_no_credential_leak.pynew file mode 100644index 000000000..9dbe86b26--- /dev/null+++ b/backend/mcp_server/tests/test_no_credential_leak.py@@ -0,0 +1,824 @@+"""Registry-wide invariant: no tool output may contain a credential.++This is the test that keeps the README's exclusion list true as tools get+added. Several of this project's serializers return decrypted secrets in their+normal responses — ``ConnectorInstance.connector_metadata`` carries database+passwords and object-store keys, ``AdapterInstance.metadata`` carries provider+API keys — so a tool that reaches a model through ``serializer.data``,+``model_to_dict`` or a ``**`` splat would pipe those into an LLM's context+without anyone noticing at review time.++Rather than trusting each tool to be written carefully, this seeds an+organization with recognizable fake secrets, invokes **every** read tool in the+platform registry, and asserts none of those strings comes back. A new tool is+covered the moment it is registered.++Needs a live DB (integration tier).+"""++from __future__ import annotations++import json+from unittest.mock import Mock, patch++import pytest+from account_v2.models import Organization, User+from adapter_processor_v2.models import AdapterInstance+from api_v2.models import APIDeployment+from connector_v2.models import ConnectorInstance+from django.test import SimpleTestCase, TestCase, override_settings+from platform_api.models import PlatformApiKey+from prompt_studio.prompt_studio_core_v2.models import CustomTool+from prompt_studio.prompt_studio_document_manager_v2.models import DocumentManager+from prompt_studio.prompt_studio_v2.models import ToolStudioPrompt+from tenant_account_v2.models import OrganizationMember+from utils.user_context import UserContext+from workflow_manager.endpoint_v2.models import WorkflowEndpoint+from workflow_manager.workflow_v2.models.execution import WorkflowExecution+from workflow_manager.workflow_v2.models.workflow import Workflow++from mcp_server.context import PlatformMCPContext+from mcp_server.registry import PLATFORM_TOOLS++ORG_ID = "org-leak"++# `whoami` reports the spend budget, which reads the cache. Use local memory so+# this test exercises tool output rather than Redis availability.+LOCMEM = {+ "default": {+ "BACKEND": "django.core.cache.backends.locmem.LocMemCache",+ "LOCATION": "mcp-leak-tests",+ }+}++# Distinctive sentinels: if any of these appears in tool output, a secret+# escaped. Chosen not to collide with anything a tool would legitimately emit.+DB_PASSWORD = "LEAKCANARY-db-password-9f3a"+S3_SECRET = "LEAKCANARY-s3-secret-key-7c1b"+LLM_API_KEY = "LEAKCANARY-openai-api-key-4e8d"++# Only secrets are canaries. A platform key's *name* is deliberately surfaced+# by whoami — it is a label the operator chose, not credential material, and+# treating it as one would make this test object to correct behaviour.+CANARIES = (DB_PASSWORD, S3_SECRET, LLM_API_KEY)++# Ids for the delegate-stubbed sweep below. Real UUIDs because the tools now+# validate id shape before doing anything else.+PROJECT_UUID = "11111111-1111-1111-1111-111111111111"+DOCUMENT_UUID = "22222222-2222-2222-2222-222222222222"+PROMPT_UUID = "33333333-3333-3333-3333-333333333333"+PIPELINE_UUID = "44444444-4444-4444-4444-444444444444"+WORKFLOW_UUID = "55555555-5555-5555-5555-555555555555"+EXECUTION_UUID = "66666666-6666-6666-6666-666666666666"+DOC_URL = "https://b.s3.us-east-1.amazonaws.com/a.pdf?X-Amz-Signature=abc"++# Single marker for the field-seeded sweep. Distinct from the CANARIES above,+# which are credential-shaped values placed in specific fixtures; this one is+# written into *every* free-form field, so any wholesale serialization trips it.+CANARY = "LEAKCANARY-field-seeded-8b2e"+++@override_settings(CACHES=LOCMEM)+class NoCredentialLeakTest(TestCase):+ def setUp(self) -> None:+ self.org = Organization.objects.create(+ name=ORG_ID, display_name="Leak Org", organization_id=ORG_ID+ )+ UserContext.set_organization_identifier(ORG_ID)+ self.user = User.objects.create(+ username="svc-leak",+ email="[email protected]",+ user_id="uid-leak",+ is_service_account=True,+ )+ OrganizationMember.objects.create(+ user=self.user, organization=self.org, role="user"+ )+ self.key = PlatformApiKey.objects.create(+ name="leak-test-key",+ description="d",+ organization=self.org,+ api_user=self.user,+ permission="read_write",+ )++ # A connector holding credentials, wired into a workflow endpoint —+ # the transitive path that makes getWorkflowEndpoints risky.+ self.connector = ConnectorInstance.objects.create(+ connector_name="Leak Postgres",+ connector_id="postgresql|4d3a1b0e-1f2c-4a3d-9e8f-0a1b2c3d4e5f",+ connector_metadata={+ "user": "admin",+ "password": DB_PASSWORD,+ "secret_access_key": S3_SECRET,+ },+ )+ # `adapter_metadata` is the writable field; `metadata` is the property+ # that decrypts it on read — which is exactly why a tool must never+ # serialize this model wholesale.+ self.adapter = AdapterInstance.objects.create(+ adapter_name="Leak LLM",+ adapter_id="openai|1a2b3c4d-5e6f-7a8b-9c0d-1e2f3a4b5c6d",+ adapter_type="LLM",+ adapter_metadata={"api_key": LLM_API_KEY},+ )++ self.workflow = Workflow.objects.create(+ workflow_name="leak-wf", description="d", is_active=True+ )+ WorkflowEndpoint.objects.create(+ workflow=self.workflow,+ endpoint_type=WorkflowEndpoint.EndpointType.SOURCE,+ connection_type=WorkflowEndpoint.ConnectionType.DATABASE,+ connector_instance=self.connector,+ configuration={"table": "docs", "password_hint": DB_PASSWORD},+ )+ self.deployment = APIDeployment.objects.create(+ api_name="leak-api", display_name="Leak API", workflow=self.workflow+ )+ self.project = CustomTool.objects.create(+ tool_name="Leak Prompts", description="d", author="acme"+ )+ DocumentManager.objects.create(document_name="leak-doc.pdf", tool=self.project)+ # The webhook URL carries a canary because ToolStudioPrompt is the one+ # model in this sweep with a credential-bearing field that is not a+ # connector or adapter — a serializer-built response would carry it+ # straight out, and this is what catches that.+ ToolStudioPrompt.objects.create(+ prompt_key="leak_check",+ prompt="What is the total?",+ tool_id=self.project,+ sequence_number=1,+ prompt_type="Text",+ enforce_type="text",+ postprocessing_webhook_url=f"https://hooks.internal/x?token={LLM_API_KEY}",+ )+ # WorkflowExecution.save() mirrors into a Redis-backed execution cache+ # via its own client, which override_settings(CACHES=...) does not+ # cover. That mirroring is irrelevant here — the tool reads the DB —+ # so it is stubbed to keep this test off live infra.+ with patch(+ "workflow_manager.workflow_v2.models.execution."+ "WorkflowExecution._handle_execution_cache"+ ):+ self.execution = WorkflowExecution.objects.create(+ workflow_id=self.workflow.id,+ # The deployment that ran it. DeploymentHelper sets+ # `pipeline_id = api.id` on every real API-deployment run, and+ # getExecutionStatus scopes its poll to exactly that — so an+ # execution created without it is not a shape production+ # produces, and leaving it unset made this fixture reject the+ # tool rather than scan its output.+ pipeline_id=self.deployment.id,+ status="ERROR",+ # Error text is echoed back to the agent, so seed a canary+ # here too: a stack trace is where a connection string leaks.+ error_message=f"connect failed: password={DB_PASSWORD}",+ )++ self.context = PlatformMCPContext(+ user=self.user,+ platform_key=self.key,+ org_name=ORG_ID,+ request=Mock(data={}),+ )++ def _read_tools(self):+ """Every non-billable, non-writing tool — the ones safe to just call.++ Write and billable tools are excluded because invoking them here would+ start real executions and spend real budget, not because they are+ believed safe. They fall into two groups:++ * Those that **delegate** — the four Prompt Studio tools, ``executePipeline``+ and ``extractDocument`` — return upstream data this app did not+ assemble field by field, so their results pass through+ ``redact_structure``. Those call sites are pinned by+ ``test_redaction.WriteToolRedactionCallSitesTest``.+ * ``setApiDeploymentActive`` and ``setPipelineActive`` hand-assemble+ named fields exactly like the read tools; they are excluded only for+ their side effect, not because they return anything raw.++ That is still a weaker guarantee than this sweep gives the read tools:+ it pins the call sites individually rather than walking the registry, so+ a *new* delegating write tool added without redaction would be caught by+ nothing here. Extending the sweep to invoke the write tools against+ stubbed delegates is worth doing and is not attempted in this change.+ """+ for name in PLATFORM_TOOLS.names():+ tool = PLATFORM_TOOLS.get(name)+ if tool.writes or tool.billable:+ continue+ yield name, tool++ @pytest.mark.critical_path("mcp-platform-auth")+ def test_no_read_tool_returns_a_credential(self) -> None:+ """Invoke every read tool and scan its output for the canaries.++ Tools needing an argument get the seeded ids; anything that still+ cannot be called is reported rather than silently skipped, so this+ cannot quietly stop covering the registry.+ """+ arguments = {+ "getWorkflowEndpoints": {"workflow_id": str(self.workflow.id)},+ "listToolInstances": {"workflow_id": str(self.workflow.id)},+ "listExecutions": {},+ "getUsageSummary": {},+ "getExecutionDetail": {"execution_id": str(self.execution.id)},+ "listPromptStudioDocuments": {"project_id": str(self.project.tool_id)},+ "listPrompts": {"project_id": str(self.project.tool_id)},+ # Seeded against this org's workflow, so the tool gets past its+ # org-scope check and actually renders a response to scan.+ "getExecutionStatus": {"execution_id": str(self.execution.id)},+ }++ checked = []+ # The seeded execution has status ERROR, which ExecutionStatus counts+ # as terminal — so getExecutionStatus takes its is_completed branch and+ # reads results through a raw Redis client that+ # override_settings(CACHES=...) does not reach, exactly like+ # _handle_execution_cache above. Stubbed to keep this test off live+ # infra; the cached results are not what is being scanned here.+ result_cache = patch(+ "workflow_manager.endpoint_v2.result_cache_utils."+ "ResultCacheUtils.get_api_results",+ return_value=[],+ )+ for name, tool in self._read_tools():+ kwargs = arguments.get(name, {})+ required = tool.input_schema.get("required", [])+ if any(field not in kwargs for field in required):+ # A read tool needing arguments this test does not know how to+ # supply. Fail loudly: an uncovered tool is the whole risk.+ raise AssertionError(+ f"Read tool '{name}' requires {required} but this test has "+ f"no fixture for it — add one so it stays covered."+ )++ with result_cache:+ result = tool.handler(self.context, **kwargs)+ blob = json.dumps(result, default=str)+ checked.append(name)++ for canary in CANARIES:+ assert canary not in blob, (+ f"Tool '{name}' leaked a credential ({canary}). Build its "+ f"response from named fields — never serializer.data, "+ f"model_to_dict, or a ** splat."+ )++ # Guard the guard: if the filter above ever matches nothing, this test+ # would pass vacuously.+ assert len(checked) >= 5, f"Expected to check several tools, got {checked}"++ # Note on coverage depth: canaries live in the fixtures above, so a+ # tool reading a model nobody seeded is invoked and scanned but could+ # not have failed. `ReadToolCanaryCoverageTest` closes that by seeding+ # every free-form field of an unsaved instance per tool; this sweep+ # remains the stronger check for the tools it does cover, because it+ # runs real managers, `for_user` scoping and decrypting properties.++ @pytest.mark.critical_path("mcp-platform-auth")+ def test_workflow_endpoints_omits_connector_configuration(self) -> None:+ """The specific transitive path worth naming.++ A workflow endpoint points at a connector instance whose metadata is+ decrypted on access. Returning the endpoint's own `configuration`, or+ following the relation into the connector, would leak it.+ """+ from mcp_server.tools.observability import get_workflow_endpoints++ result = get_workflow_endpoints(self.context, workflow_id=str(self.workflow.id))+ blob = json.dumps(result, default=str)++ assert result["endpoints"], "fixture should produce one endpoint"+ endpoint = result["endpoints"][0]+ # The shape is exposed...+ assert endpoint["connection_type"] == "DATABASE"+ assert endpoint["connector_name"] == "Leak Postgres"+ assert endpoint["is_configured"] is True+ # ...but never the contents.+ assert "configuration" not in endpoint+ assert "connector_metadata" not in blob+ assert DB_PASSWORD not in blob++ @pytest.mark.critical_path("mcp-platform-auth")+ def test_no_tool_wraps_connectors_or_adapters(self) -> None:+ """These subsystems are excluded wholesale, not filtered.++ Their serializers return decrypted credentials by design, so the safe+ boundary is to expose no tool for them at all rather than to trust a+ field-level filter to stay correct.+ """+ forbidden = ("connector", "adapter")+ offenders = [+ name+ for name in PLATFORM_TOOLS.names()+ if any(word in name.lower() for word in forbidden)+ ]++ assert offenders == [], (+ f"Connector and adapter responses carry decrypted credentials; "+ f"no MCP tool should wrap them: {offenders}"+ )+++class WriteToolLeakSweepTest(SimpleTestCase):+ """The sweep above, extended to the tools it could not invoke.++ ``PlatformToolLeakSweepTest`` skips every ``writes``/``billable`` tool+ because calling them for real would start executions and spend budget — but+ those are precisely the tools that return upstream data this app did not+ assemble field by field, so leaving them uncovered left the guarantee+ hollow exactly where it mattered most.++ The fix is to stub the delegate rather than skip the tool: each tool is+ invoked for real, with the boundary it delegates to (a Prompt Studio view+ action, ``PipelineManager``, ``DeploymentHelper``) returning a canary. What+ is under test is whether the tool's own return path scrubs what it is+ handed — which is the same question the read sweep asks, just reachable+ without live infra.++ Both directions are covered, because they take different code paths: a 2xx+ body flows through the success branch, a non-2xx body through the+ error-reporting branch that ``_result`` deliberately returns *as data*.++ ``SimpleTestCase``: no database. The project/pipeline/deployment lookups are+ the part that would need one, and they are stubbed.+ """++ def _context(self):+ context = Mock()+ context.org_name = ORG_ID+ context.request = Mock(data={})+ context.platform_key = Mock(name="k")+ context.user = Mock(is_service_account=True)+ return context++ def _assert_clean(self, result, tool_name: str) -> None:+ blob = json.dumps(result, default=str)+ for canary in CANARIES:+ assert canary not in blob, (+ f"Tool '{tool_name}' leaked a credential ({canary}) from its "+ "delegate's response. Route the upstream payload through "+ "redact_structure before returning it."+ )++ # --- Prompt Studio: the four billable tools share _result ---------------++ def _run_prompt_studio(self, tool, status_code: int, data, **kwargs):+ """Invoke a Prompt Studio tool with the view action stubbed."""+ from mcp_server.tools import prompt_studio as ps++ project = Mock(tool_id=PROJECT_UUID, tool_name="Leak Prompts")+ response = Mock(status_code=status_code, data=data)++ class StubView:+ def __init__(self) -> None:+ pass++ def __getattr__(self, item):+ return lambda request, pk=None: response++ with (+ patch.object(ps, "_resolve_project", return_value=project),+ patch.object(ps, "PromptStudioCoreView", StubView),+ ):+ return tool(self._context(), project_id=PROJECT_UUID, **kwargs)++ def test_prompt_studio_tools_scrub_a_successful_response(self) -> None:+ from mcp_server.tools.prompt_studio import (+ bulk_fetch_response,+ fetch_response,+ index_document,+ single_pass_extraction,+ )++ # A delegated view returning an indexing summary that happens to embed+ # the adapter's key — the shape a real failure downstream produces.+ leaky = {"output": "ok", "profile": {"api_key": LLM_API_KEY}}+ cases = [+ (index_document, "indexDocument", {"document_id": DOCUMENT_UUID}),+ (+ fetch_response,+ "fetchResponse",+ {"document_id": DOCUMENT_UUID, "prompt_id": PROMPT_UUID},+ ),+ (+ bulk_fetch_response,+ "bulkFetchResponse",+ {"document_id": DOCUMENT_UUID, "prompt_ids": [PROMPT_UUID]},+ ),+ (+ single_pass_extraction,+ "singlePassExtraction",+ {"document_id": DOCUMENT_UUID},+ ),+ ]+ for tool, name, kwargs in cases:+ with self.subTest(name):+ result = self._run_prompt_studio(tool, 200, leaky, **kwargs)+ self._assert_clean(result, name)++ def test_prompt_studio_tools_scrub_an_error_response(self) -> None:+ """The path that matters more: `_result` returns non-2xx bodies as+ data, and a delegated view's error text is where a connection string+ actually surfaces.+ """+ from mcp_server.tools.prompt_studio import index_document++ leaky = {"detail": f"could not connect: password={DB_PASSWORD}"}++ result = self._run_prompt_studio(+ index_document, 500, leaky, document_id=DOCUMENT_UUID+ )++ self._assert_clean(result, "indexDocument")+ assert result["ok"] is False++ # --- executePipeline -----------------------------------------------------++ def test_execute_pipeline_scrubs_its_manager_response(self) -> None:+ from mcp_server.tools import platform as platform_tools+ from mcp_server.tools.platform import execute_pipeline++ pipeline = Mock(id=PIPELINE_UUID, pipeline_name="Leak Pipeline")+ for label, status, data in [+ ("success", 200, {"log": f"connected with password={DB_PASSWORD}"}),+ ("error", 500, {"detail": f"api_key={LLM_API_KEY} rejected"}),+ ]:+ with self.subTest(label):+ with (+ patch.object(+ platform_tools, "_resolve_pipeline", return_value=pipeline+ ),+ patch(+ "pipeline_v2.manager.PipelineManager.execute_pipeline",+ return_value=Mock(status_code=status, data=data),+ ),+ ):+ result = execute_pipeline(+ self._context(), pipeline_id=PIPELINE_UUID+ )+ self._assert_clean(result, "executePipeline")++ # --- extractDocument and its poll ---------------------------------------++ def test_extract_document_scrubs_the_execution_response(self) -> None:+ from mcp_server.tools.execution import extract_document++ context = Mock()+ context.org_name = ORG_ID+ context.api = Mock(is_active=True, api_name="leak-api")+ context.api_key = "k"++ leaky = {+ "execution_status": "ERROR",+ "error": f"connector refused: password={DB_PASSWORD}",+ }+ with (+ patch(+ "mcp_server.tools.execution.ExecutionRequestSerializer.is_valid",+ return_value=True,+ ),+ patch(+ "mcp_server.tools.execution.ExecutionRequestSerializer.validated_data",+ {"presigned_urls": [DOC_URL]},+ ),+ patch(+ "mcp_server.tools.execution.APIDeploymentRateLimiter.check_and_acquire",+ return_value=(True, {}),+ ),+ patch("mcp_server.tools.execution.DeploymentHelper.load_presigned_files"),+ patch(+ "mcp_server.tools.execution.DeploymentHelper.execute_workflow",+ return_value=leaky,+ ),+ ):+ result = extract_document(context, document_urls=[DOC_URL])++ self._assert_clean(result, "extractDocument")++ def test_get_execution_status_scrubs_the_polled_result(self) -> None:+ """The tool the credential-leak sweep originally caught a leak in."""+ from workflow_manager.workflow_v2.dto import ExecutionResponse++ from mcp_server.tools.execution import get_execution_status++ context = Mock()+ context.api = Mock(workflow_id=WORKFLOW_UUID)++ response = ExecutionResponse(+ workflow_id=WORKFLOW_UUID,+ execution_id=EXECUTION_UUID,+ execution_status="ERROR",+ result=[{"file": "a.pdf", "error": f"secret_access_key={S3_SECRET}"}],+ )+ with (+ patch("mcp_server.tools.execution.WorkflowExecution.objects.filter") as f,+ patch(+ "mcp_server.tools.execution.ExecutionQuerySerializer.is_valid",+ return_value=True,+ ),+ patch(+ "mcp_server.tools.execution.ExecutionQuerySerializer.validated_data",+ {+ "execution_id": EXECUTION_UUID,+ "include_metadata": False,+ "include_metrics": False,+ "include_extracted_text": False,+ },+ ),+ patch(+ "mcp_server.tools.execution.DeploymentHelper.get_execution_status",+ return_value=response,+ ),+ patch(+ "mcp_server.tools.execution.DeploymentHelper."+ "process_completed_execution"+ ),+ ):+ f.return_value.only.return_value.first.return_value = object()+ result = get_execution_status(context, execution_id=EXECUTION_UUID)++ self._assert_clean(result, "getExecutionStatus")++ # --- the guard on the guard ---------------------------------------------++ def test_every_write_and_billable_tool_is_covered_here(self) -> None:+ """Walks the registry so a *new* write tool cannot be added without+ either covering it or deliberately exempting it.++ This is the gap the per-call-site tests could not close: they pin the+ four sites that exist today, and would stay green while a fifth+ delegating tool shipped with no redaction at all.+ """+ covered = {+ "indexDocument",+ "fetchResponse",+ "bulkFetchResponse",+ "singlePassExtraction",+ "executePipeline",+ "extractDocument",+ }+ # These write but build their response from named fields, exactly like+ # the read tools — so they are covered by the read sweep's reasoning+ # rather than needing a delegate stub. Named explicitly so adding one+ # is a deliberate act.+ exempt = {"setApiDeploymentActive", "setPipelineActive"}++ writing = {+ name+ for name in PLATFORM_TOOLS.names()+ if PLATFORM_TOOLS.get(name).writes or PLATFORM_TOOLS.get(name).billable+ }+ uncovered = writing - covered - exempt++ assert uncovered == set(), (+ f"These tools write or spend but no leak test invokes them: "+ f"{sorted(uncovered)}. Add a delegate-stubbed case above, or add "+ f"the tool to `exempt` if it returns only named fields."+ )+++# Fields Django reports as concrete and free-form enough to hold a credential.+# Seeding is driven off this rather than a hand-kept list, so a model that+# grows a new text field is canaried without anyone remembering to.+_SEEDABLE_FIELD_TYPES = ("CharField", "TextField", "JSONField", "URLField", "EmailField")+++def seed_canaries(instance, canary: str, skip: tuple[str, ...] = ()) -> object:+ """Write ``canary`` into every free-form field of an unsaved model instance.++ The point is to make the scan non-vacuous *by construction*: rather than+ guessing which field a leak would come through, every field that could hold+ one gets a canary, so any tool that serializes wholesale is caught wherever+ it leaks from.++ ``skip`` is for fields a tool legitimately returns — a workflow's name is+ not a secret, and the tool is supposed to echo it — so seeding those would+ assert the opposite of what is intended.+ """+ for field in instance._meta.get_fields():+ if not getattr(field, "concrete", False):+ continue+ if field.name in skip or field.primary_key or field.is_relation:+ continue+ kind = field.get_internal_type()+ if kind not in _SEEDABLE_FIELD_TYPES:+ continue+ if getattr(field, "choices", None):+ # A choice field rejects arbitrary text and is not free-form+ # enough to hide a credential.+ continue+ setattr(+ instance,+ field.name,+ {"leak": canary} if kind == "JSONField" else f"{canary}-{field.name}",+ )+ return instance+++class ReadToolCanaryCoverageTest(SimpleTestCase):+ """Every read tool is scanned against data that *could* leak.++ ``PlatformToolLeakSweepTest`` invokes every read tool, but seeds canaries+ into only the three fixtures some of them happen to read — so the rest are+ scanned vacuously and would pass while serializing everything. This closes+ that by handing each tool a real (unsaved) model instance whose every+ free-form field carries a canary.++ Real model instances, not Mocks: these tools read named attributes off+ models, so what is under test is whether the tool names safe fields. A Mock+ would answer to any attribute and prove nothing about the model; an unsaved+ instance has exactly the fields Django says it has, and needs no database.++ The DB-backed sweep stays the stronger test — it exercises real managers,+ ``for_user`` scoping and decrypting property fields. This one is what can+ run in the unit tier, and it covers the tools that sweep leaves vacuous.+ """++ def _context(self):+ context = Mock()+ context.org_name = ORG_ID+ context.user = Mock(is_service_account=True)+ context.platform_key = Mock(name="k")+ return context++ def _assert_clean(self, result, tool_name: str) -> None:+ blob = json.dumps(result, default=str)+ assert CANARY not in blob, (+ f"Tool '{tool_name}' returned a seeded canary. It is serializing a "+ "field it does not name explicitly — build the response from named "+ "fields, never serializer.data, model_to_dict or a ** splat."+ )++ def test_list_workflows_returns_only_named_fields(self) -> None:+ from mcp_server.tools.platform import list_workflows++ workflow = seed_canaries(+ Workflow(is_active=True), CANARY, skip=("workflow_name", "description")+ )+ with patch(+ "mcp_server.tools.platform.Workflow.objects.for_user"+ ) as for_user:+ for_user.return_value.order_by.return_value = _FakeRows([workflow])+ result = list_workflows(self._context())++ self._assert_clean(result, "listWorkflows")++ def test_list_api_deployments_returns_only_named_fields(self) -> None:+ from mcp_server.tools.platform import list_api_deployments++ deployment = seed_canaries(+ APIDeployment(is_active=True, workflow=Workflow()),+ CANARY,+ skip=("display_name", "api_name", "description"),+ )+ with patch(+ "mcp_server.tools.platform.APIDeployment.objects.for_user"+ ) as for_user:+ for_user.return_value.order_by.return_value = _FakeRows([deployment])+ result = list_api_deployments(self._context())++ self._assert_clean(result, "listApiDeployments")++ def test_list_pipelines_returns_only_named_fields(self) -> None:+ from pipeline_v2.models import Pipeline++ from mcp_server.tools.platform import list_pipelines++ pipeline = seed_canaries(+ Pipeline(active=True), CANARY, skip=("pipeline_name",)+ )+ with patch("mcp_server.tools.platform.Pipeline.objects.for_user") as for_user:+ for_user.return_value.order_by.return_value = _FakeRows([pipeline])+ result = list_pipelines(self._context())++ self._assert_clean(result, "listPipelines")++ def test_list_prompt_studio_projects_returns_only_named_fields(self) -> None:+ from mcp_server.tools.platform import list_prompt_studio_projects++ project = seed_canaries(+ CustomTool(), CANARY, skip=("tool_name", "description", "author")+ )+ with patch("mcp_server.tools.platform.CustomTool.objects.for_user") as for_user:+ for_user.return_value.order_by.return_value = _FakeRows([project])+ result = list_prompt_studio_projects(self._context())++ self._assert_clean(result, "listPromptStudioProjects")++ def test_list_tool_instances_omits_tool_settings(self) -> None:+ """The highest-risk of these: a tool step's ``metadata`` is free-form+ JSON that can carry adapter ids and credential-shaped values, and the+ tool's own docstring says it omits them. This is what proves it.+ """+ from tool_instance_v2.models import ToolInstance++ from mcp_server.tools.observability import list_tool_instances++ workflow = Workflow(workflow_name="wf")+ # `tool_id` is skipped because the tool names it deliberately — it is+ # the identifier an agent needs to make sense of the step, not a+ # secret. Everything else, `metadata` above all, is seeded.+ step = seed_canaries(+ ToolInstance(workflow=workflow, step=1), CANARY, skip=("tool_id",)+ )+ assert CANARY in json.dumps(step.metadata, default=str), (+ "metadata must carry a canary or this test proves nothing"+ )+ with (+ patch(+ "mcp_server.tools.observability.Workflow.objects.for_user"+ ) as for_user,+ patch(+ "tool_instance_v2.models.ToolInstance.objects.filter"+ ) as tool_filter,+ ):+ for_user.return_value.filter.return_value.first.return_value = workflow+ tool_filter.return_value.order_by.return_value = [step]+ result = list_tool_instances(+ self._context(), workflow_id=str(workflow.id)+ )++ self._assert_clean(result, "listToolInstances")++ def test_every_read_tool_has_non_vacuous_coverage(self) -> None:+ """Walks the registry so a new read tool cannot be added unnoticed.++ The tests above cover named handlers, and the DB-backed sweep invokes+ everything — but against fixed fixtures, so a new tool reading a model+ nobody seeded would be scanned vacuously and pass. This is the guard+ that makes adding such a tool a deliberate act rather than a silent+ gap.+ """+ # Tools whose response is scanned against a canary that could actually+ # appear in it — either seeded here, or seeded into the DB fixtures the+ # other sweep builds.+ covered_here = {+ "listWorkflows",+ "listApiDeployments",+ "listPipelines",+ "listPromptStudioProjects",+ "listToolInstances",+ }+ covered_by_db_fixtures = {+ "getWorkflowEndpoints",+ "listExecutions",+ "getExecutionDetail",+ "getExecutionStatus",+ "listPrompts",+ "listPromptStudioDocuments",+ }+ # Tools that return no model-derived data at all, so there is nothing+ # for a canary to hide in. Named explicitly rather than inferred.+ no_model_data = {"readMeFirst", "whoami", "getUsageSummary", "listTags"}++ read_tools = {+ name+ for name in PLATFORM_TOOLS.names()+ if not (PLATFORM_TOOLS.get(name).writes or PLATFORM_TOOLS.get(name).billable)+ }+ uncovered = read_tools - covered_here - covered_by_db_fixtures - no_model_data++ assert uncovered == set(), (+ f"These read tools are scanned against no canary that could appear "+ f"in their output, so they pass vacuously: {sorted(uncovered)}. Add "+ f"a seeded case to ReadToolCanaryCoverageTest, or list the tool in "+ f"`no_model_data` if it returns nothing model-derived."+ )++ def test_the_seeding_helper_actually_seeds(self) -> None:+ """Guard the guard: if seeding silently wrote nothing, every test above+ would pass vacuously — which is the exact failure this class exists to+ remove.+ """+ workflow = seed_canaries(Workflow(), CANARY)++ assert CANARY in workflow.workflow_name+ assert CANARY in json.dumps(workflow.source_settings, default=str)++ def test_a_wholesale_serializer_would_be_caught(self) -> None:+ """Proves the scan can fail. A tool built the unsafe way — dumping the+ model instead of naming fields — must trip the same assertion.+ """+ from django.forms.models import model_to_dict++ workflow = seed_canaries(Workflow(), CANARY)+ unsafe_result = {"workflows": [model_to_dict(workflow)]}++ with self.assertRaises(AssertionError):+ self._assert_clean(unsafe_result, "pretendTool")+++class _FakeRows(list):+ """A list that answers the two queryset calls the list tools make.++ They do ``queryset.count()`` then slice for the page, so nothing more is+ needed — and keeping it this thin means the test fails if a tool starts+ doing something else to the queryset, rather than silently passing.+ """++ def count(self) -> int:+ return len(self)diff --git a/backend/mcp_server/tests/test_platform_auth.py b/backend/mcp_server/tests/test_platform_auth.pynew file mode 100644index 000000000..4c170ac89--- /dev/null+++ b/backend/mcp_server/tests/test_platform_auth.py@@ -0,0 +1,174 @@+"""Critical path ``mcp-platform-auth``: the organization-scoped MCP endpoint+is authenticated by ``CustomAuthMiddleware``, not by the view.++That distinction is the whole point of these tests. The deployment MCP server+owns its own auth, so testing it through ``APIRequestFactory`` is sound. This+server does not — its credential is resolved by middleware, which+``APIRequestFactory`` and direct view calls bypass entirely. A test written+that way would pass against a completely unauthenticated endpoint.++So every test here goes through ``django.test.Client`` against the real tenant+URL, exercising the full middleware stack. Needs a live DB (integration tier).+"""++from __future__ import annotations++import json+import uuid++import pytest+from account_v2.models import Organization, User+from django.test import Client, TestCase+from platform_api.models import PlatformApiKey+from tenant_account_v2.models import OrganizationMember+from utils.user_context import UserContext++ORG_ID = "org-platform-mcp"+OTHER_ORG_ID = "org-platform-other"+++def make_org_with_key(org_id: str, permission: str = "read_write"):+ """Create an organization and a platform key backed by a service account."""+ org = Organization.objects.create(+ name=org_id, display_name=org_id, organization_id=org_id+ )+ user = User.objects.create(+ username=f"svc-{org_id}",+ email=f"svc-{org_id}@platform.internal",+ user_id=f"uid-{org_id}",+ is_service_account=True,+ )+ OrganizationMember.objects.create(user=user, organization=org, role="user")+ key = PlatformApiKey.objects.create(+ name=f"key-{org_id}",+ description="test key",+ organization=org,+ api_user=user,+ permission=permission,+ )+ return org, user, key+++class PlatformMCPAuthTest(TestCase):+ def setUp(self) -> None:+ self.org, self.user, self.key = make_org_with_key(ORG_ID)+ _, _, self.other_key = make_org_with_key(OTHER_ORG_ID)+ UserContext.set_organization_identifier(ORG_ID)+ self.client = Client()+ self.url = f"/api/v1/unstract/{ORG_ID}/mcp/"++ def _post(self, auth: str | None = None, body: dict | None = None, url: str = None):+ """POST through the full URL stack so the auth middleware runs."""+ payload = body if body is not None else {+ "jsonrpc": "2.0",+ "id": 1,+ "method": "tools/list",+ }+ headers = {"HTTP_AUTHORIZATION": auth} if auth else {}+ return self.client.post(+ url or self.url,+ data=json.dumps(payload),+ content_type="application/json",+ **headers,+ )++ @pytest.mark.critical_path("mcp-platform-auth")+ def test_bad_credentials_are_rejected(self) -> None:+ """Each of these must be refused before any tool is reachable.++ The middleware answers most of them, which is exactly what is being+ verified — that this endpoint sits behind it.+ """+ cases = [+ ("no authorization header", None),+ ("not a bearer token", f"Token {self.key.key}"),+ ("empty bearer", "Bearer "),+ ("malformed uuid", "Bearer not-a-uuid"),+ ("unknown key", f"Bearer {uuid.uuid4()}"),+ ("key from another organization", f"Bearer {self.other_key.key}"),+ ]+ for label, auth in cases:+ with self.subTest(label):+ response = self._post(auth)+ assert response.status_code in (401, 403), (+ f"{label}: got {response.status_code}, body={response.content[:200]}"+ )++ @pytest.mark.critical_path("mcp-platform-auth")+ def test_inactive_key_is_rejected(self) -> None:+ self.key.is_active = False+ self.key.save()++ response = self._post(f"Bearer {self.key.key}")++ assert response.status_code == 401, response.content++ @pytest.mark.critical_path("mcp-platform-auth")+ def test_read_tier_key_cannot_reach_the_server(self) -> None:+ """A documented limitation, pinned so it cannot change silently.++ The middleware's tier check gates on HTTP method and every JSON-RPC+ message arrives as a POST whatever the tool inside it does, so a `read`+ key is refused outright — even though this server exposes plenty of+ read tools.+ """+ self.key.permission = "read"+ self.key.save()++ response = self._post(f"Bearer {self.key.key}")++ assert response.status_code == 403, response.content++ @pytest.mark.critical_path("mcp-platform-auth")+ def test_valid_key_reaches_the_tool_listing(self) -> None:+ """The inverse guard: a check that refused everything would satisfy+ every rejection case above.+ """+ response = self._post(f"Bearer {self.key.key}")++ assert response.status_code == 200, response.content+ tools = [t["name"] for t in json.loads(response.content)["result"]["tools"]]++ # readMeFirst must lead: agents weight earlier tools more heavily, and+ # it is what explains the budget and the org-wide reach of the rest.+ assert tools[0] == "readMeFirst"+ # A representative tool from each group, rather than the full ordered+ # list — pinning all 19 makes this fail on every addition without+ # telling anyone anything useful.+ assert {+ "whoami",+ "listApiDeployments",+ "listExecutions",+ "getUsageSummary",+ "setApiDeploymentActive",+ "executePipeline",+ "bulkFetchResponse",+ } <= set(tools)++ @pytest.mark.critical_path("mcp-platform-auth")+ def test_the_endpoint_is_not_whitelisted(self) -> None:+ """Guards the placement this server's security depends on.++ If these URLs were ever moved under the whitelisted ``/mcp/`` prefix,+ ``CustomAuthMiddleware`` would skip them and the endpoint would answer+ unauthenticated callers. An unauthenticated request must not get a+ JSON-RPC result.+ """+ from django.conf import settings++ assert not any(+ self.url.startswith(path) for path in settings.WHITELISTED_PATHS+ ), f"{self.url} is whitelisted — it would bypass authentication entirely"++ response = self._post(auth=None)+ assert response.status_code in (401, 403)+ assert b'"result"' not in response.content++ @pytest.mark.critical_path("mcp-platform-auth")+ def test_org_in_url_must_match_the_key(self) -> None:+ """A valid key must not reach another organization's MCP endpoint."""+ response = self._post(+ f"Bearer {self.key.key}", url=f"/api/v1/unstract/{OTHER_ORG_ID}/mcp/"+ )++ assert response.status_code in (401, 403), response.contentdiff --git a/backend/mcp_server/tests/test_platform_tier_guard.py b/backend/mcp_server/tests/test_platform_tier_guard.pynew file mode 100644index 000000000..8f6836aa3--- /dev/null+++ b/backend/mcp_server/tests/test_platform_tier_guard.py@@ -0,0 +1,253 @@+"""Per-tool authorization on the platform server.++The auth middleware checks the key's permission tier against the request's HTTP+method — but every JSON-RPC message arrives as an HTTP POST whatever the tool+inside it does, so the middleware's verdict cannot distinguish+``listWorkflows`` from ``executePipeline``. This guard re-applies the tier+against the method each tool *declares*, and is the only thing standing between+a low-tier key and a write tool.+"""++from __future__ import annotations++from unittest.mock import Mock++from django.test import SimpleTestCase++from platform_api.models import ApiKeyPermission++from mcp_server.context import PlatformMCPContext+from mcp_server.platform_views import PlatformMCPServerView+from mcp_server.registry import PLATFORM_TOOLS, MCPTool+++def a_tool(required_method: str, writes: bool = True) -> MCPTool:+ return MCPTool(+ name="doThing",+ description="d",+ input_schema={"type": "object", "properties": {}},+ handler=lambda ctx: None,+ writes=writes,+ required_method=required_method,+ )+++def context_for(tier: str) -> PlatformMCPContext:+ return PlatformMCPContext(+ user=Mock(is_service_account=True),+ platform_key=Mock(permission=tier),+ org_name="org-tier",+ )+++class PlatformTierGuardTest(SimpleTestCase):+ def setUp(self) -> None:+ self.view = PlatformMCPServerView()++ def test_tier_matrix(self) -> None:+ """The whole authorization model in one table.++ Mirrors ``ApiKeyPermission.allows`` deliberately: if that mapping ever+ changes, this fails and forces the MCP surface to be reconsidered+ rather than silently inheriting a wider grant.+ """+ cases = [+ # tier, method, allowed+ ("read", "GET", True),+ ("read", "POST", False),+ ("read", "DELETE", False),+ ("read_write", "GET", True),+ ("read_write", "POST", True),+ # read_write must NOT reach destructive tools.+ ("read_write", "DELETE", False),+ ("full_access", "GET", True),+ ("full_access", "POST", True),+ ("full_access", "DELETE", True),+ ]+ for tier, method, allowed in cases:+ with self.subTest(f"{tier} -> {method}"):+ refusal = self.view.check_tool_allowed(a_tool(method), context_for(tier))+ assert (refusal is None) == allowed, (+ f"{tier} {method}: expected allowed={allowed}, got {refusal!r}"+ )++ def test_refusal_names_the_tool_and_the_tier(self) -> None:+ """The refusal is read by an agent, which should be able to tell its+ operator what to change rather than just retrying.+ """+ refusal = self.view.check_tool_allowed(+ a_tool("DELETE"), context_for("read_write")+ )++ assert "doThing" in refusal+ assert "DELETE" in refusal+ assert "read_write" in refusal++ def test_unrecognised_tier_is_refused(self) -> None:+ refusal = self.view.check_tool_allowed(a_tool("GET"), context_for("wat"))++ assert refusal is not None++ def test_every_write_tool_declares_a_non_get_method(self) -> None:+ """The invariant that keeps the guard meaningful.++ A ``writes=True`` tool left at the default ``required_method="GET"``+ would be reachable by any key that can reach the server at all — the+ guard would pass it silently. This is what makes registering a write+ tool safe by default.+ """+ unguarded = [+ name+ for name in PLATFORM_TOOLS.names()+ if PLATFORM_TOOLS.get(name).writes+ and PLATFORM_TOOLS.get(name).required_method == "GET"+ ]++ assert unguarded == [], (+ f"These platform tools mutate state but declare required_method="+ f"'GET', so the tier guard will not protect them: {unguarded}"+ )++ def test_registering_an_unguarded_write_tool_is_refused(self) -> None:+ """The invariant above is now enforced at registration, not only here.++ A CI assertion can be deselected, or simply not run on the branch that+ adds the tool; a registration-time refusal means a wrongly declared+ tool cannot reach a running server at all.+ """+ from mcp_server.registry import MCPToolRegistry++ registry = MCPToolRegistry()++ with self.assertRaises(ValueError) as caught:+ registry.register(+ MCPTool(+ name="deleteEverything",+ description="",+ input_schema={},+ handler=lambda context: None,+ writes=True,+ )+ )++ assert "required_method='GET'" in str(caught.exception)++ def test_a_truthfully_declared_write_tool_registers(self) -> None:+ """The invariant is satisfiable, not merely restrictive.++ It applies to both registries. The deployment server has no tiers — it+ does not override ``check_tool_allowed``, so ``required_method`` is+ unread there — but declaring it truthfully costs nothing and keeps the+ check unconditional rather than opt-in.+ """+ from mcp_server.registry import MCPToolRegistry++ registry = MCPToolRegistry()++ registry.register(+ MCPTool(+ name="extractSomething",+ description="",+ input_schema={},+ handler=lambda context: None,+ writes=True,+ required_method="POST",+ )+ )++ assert registry.get("extractSomething") is not None++ def test_no_credential_tools_are_exposed(self) -> None:+ """API key creation and rotation return the secret in their response.++ Exposing either as an MCP tool would hand an agent — one that may be+ processing untrusted document content — a way to mint or exfiltrate+ credentials. Named explicitly so adding one is a deliberate act.+ """+ forbidden = ("key", "credential", "secret", "token", "rotate")+ offenders = [+ name+ for name in PLATFORM_TOOLS.names()+ if any(word in name.lower() for word in forbidden)+ ]++ assert offenders == [], (+ f"Platform MCP server must not expose credential tools: {offenders}"+ )+++class TierGuardIsProspectiveTest(SimpleTestCase):+ """The per-tool tier guard refuses nothing today — deliberately.++ Raised in review as a design question worth answering explicitly rather+ than leaving implicit: is the tier model doing what we think it does?++ It is not, *yet*. A `read` key cannot reach the view (the middleware+ rejects POST on tier before dispatch), and both remaining tiers allow POST+ while every tool declares GET or POST. So `check_tool_allowed` never+ refuses a request that reaches it. That is defense-in-depth for the day a+ DELETE-tier tool is added, or the day the middleware grows an MCP carve-out.++ Pinned here so the fact is a tripwire rather than something a future+ reader re-derives: when this test starts failing, the guard has begun doing+ real work and the docstrings claiming otherwise need updating.+ """++ def test_no_tool_declares_delete_today(self) -> None:+ methods = {+ PLATFORM_TOOLS.get(name).required_method for name in PLATFORM_TOOLS.names()+ }++ assert methods <= {"GET", "POST"}, (+ f"A tool now declares {sorted(methods - {'GET', 'POST'})}. The tier "+ "guard has started refusing real calls — update the docstrings in "+ "platform_views.check_tool_allowed that say it refuses nothing."+ )++ def test_the_guard_refuses_nothing_for_tiers_that_can_reach_it(self) -> None:+ """Both halves of the reviewer's trace, asserted rather than argued."""+ for permission in ApiKeyPermission:+ if not permission.allows("POST"):+ # Cannot reach the view at all; the middleware stops it first.+ continue+ refused = [+ name+ for name in PLATFORM_TOOLS.names()+ if not permission.allows(PLATFORM_TOOLS.get(name).required_method)+ ]+ with self.subTest(permission.value):+ assert refused == [], (+ f"Tier '{permission.value}' now refuses {refused}. The guard "+ "is doing real work — see the note above."+ )++ def test_a_read_key_cannot_reach_the_server_at_all(self) -> None:+ """The documented minimum tier, pinned to the permission model.++ `whoami` and the README both state `read_write` is required *including*+ for read tools. That claim is only true while `read` disallows POST.+ """+ assert ApiKeyPermission("read").allows("POST") is False+ assert ApiKeyPermission("read_write").allows("POST") is True++ def test_whoami_states_the_minimum_tier(self) -> None:+ """An agent holding a read key never reaches whoami, but one holding a+ read_write key needs to know why its tier is the floor — otherwise+ `can_use_write_tools: false` reads as "read tools still work", which+ would be wrong for a read key.+ """+ from unittest.mock import Mock, patch++ from mcp_server.tools.platform import whoami++ context = Mock()+ context.org_name = "org-mcp"+ context.platform_key = Mock(name="k", permission="read_write")+ context.user = Mock(is_service_account=True)++ with patch("mcp_server.spend_guard.peek") as peek:+ peek.return_value = Mock(used=0, limit=50, window_seconds=3600)+ result = whoami(context)++ assert result["minimum_tier"] == "read_write"+ assert "read_write" in result["tier_note"]diff --git a/backend/mcp_server/tests/test_platform_tools.py b/backend/mcp_server/tests/test_platform_tools.pynew file mode 100644index 000000000..6ec102f8b--- /dev/null+++ b/backend/mcp_server/tests/test_platform_tools.py@@ -0,0 +1,531 @@+"""Behaviour of the organization-scoped read-only tools.++These run against a live DB because the whole point of each tool is the+queryset it delegates to — mocking that away would leave nothing worth+asserting. Needs a live DB (integration tier).+"""++from __future__ import annotations++import json+from typing import Any+from unittest.mock import Mock, patch++from account_v2.models import Organization, User+from api_v2.models import APIDeployment+from django.test import TestCase+from pipeline_v2.models import Pipeline+from platform_api.models import PlatformApiKey+from prompt_studio.prompt_studio_core_v2.models import CustomTool+from prompt_studio.prompt_studio_document_manager_v2.models import DocumentManager+from prompt_studio.prompt_studio_v2.models import ToolStudioPrompt+from tenant_account_v2.models import OrganizationMember+from utils.user_context import UserContext+from workflow_manager.workflow_v2.models.execution import WorkflowExecution+from workflow_manager.workflow_v2.models.workflow import Workflow++from mcp_server.context import PlatformMCPContext+from mcp_server.exceptions import MCPToolError+from mcp_server.tools.platform_execution import (+ get_platform_execution_status,+ platform_extract_document,+)+from mcp_server.tools.platform import (+ execute_pipeline,+ list_api_deployments,+ list_prompt_studio_projects,+ list_workflows,+ platform_read_me_first,+ set_api_deployment_active,+ set_pipeline_active,+ whoami,+)+from mcp_server.tools.prompt_studio import (+ list_prompt_studio_documents,+ list_prompts,+)++ORG_ID = "org-tools"+++class PlatformToolsTest(TestCase):+ def setUp(self) -> None:+ self.org = Organization.objects.create(+ name=ORG_ID, display_name="Tools Org", organization_id=ORG_ID+ )+ UserContext.set_organization_identifier(ORG_ID)+ self.user = User.objects.create(+ username="svc-tools",+ email="[email protected]",+ user_id="uid-tools",+ is_service_account=True,+ )+ OrganizationMember.objects.create(+ user=self.user, organization=self.org, role="user"+ )+ self.key = PlatformApiKey.objects.create(+ name="tools-key",+ description="d",+ organization=self.org,+ api_user=self.user,+ permission="read_write",+ )++ self.workflow = Workflow.objects.create(+ workflow_name="invoice-wf", description="Invoices", is_active=True+ )+ self.api = APIDeployment.objects.create(+ api_name="invoice-api",+ display_name="Invoice API",+ description="Extracts invoices",+ workflow=self.workflow,+ )+ self.tool = CustomTool.objects.create(+ tool_name="Invoice Prompts", description="Prompt project", author="acme"+ )+ self.pipeline = Pipeline.objects.create(+ pipeline_name="etl-pipeline", workflow=self.workflow+ )++ self.context = PlatformMCPContext(+ user=self.user,+ platform_key=self.key,+ org_name=ORG_ID,+ request=Mock(data={}),+ )++ def test_list_api_deployments_exposes_the_api_name(self) -> None:+ """api_name is what an agent needs to open a deployment-scoped MCP+ session, so it must be in the payload — not just the display name.+ """+ result = list_api_deployments(self.context)++ assert result["count"] == 1+ row = result["api_deployments"][0]+ assert row["api_name"] == "invoice-api"+ assert row["display_name"] == "Invoice API"+ assert row["is_active"] is True+ assert "truncated" not in result++ def test_list_workflows_returns_the_org_workflow(self) -> None:+ result = list_workflows(self.context)++ assert result["count"] == 1+ assert result["workflows"][0]["workflow_name"] == "invoice-wf"++ def test_list_prompt_studio_projects_returns_the_org_project(self) -> None:+ result = list_prompt_studio_projects(self.context)++ assert result["count"] == 1+ assert result["prompt_studio_projects"][0]["tool_name"] == "Invoice Prompts"++ def test_listings_do_not_leak_across_organizations(self) -> None:+ """The single most important property of these tools. A platform key+ must never surface another tenant's resources.+ """+ other = Organization.objects.create(+ name="org-other", display_name="Other", organization_id="org-other"+ )+ UserContext.set_organization_identifier("org-other")+ other_wf = Workflow.objects.create(workflow_name="secret-wf", is_active=True)+ APIDeployment.objects.create(+ api_name="secret-api", display_name="Secret API", workflow=other_wf+ )+ CustomTool.objects.create(+ tool_name="Secret Prompts", description="d", author="other"+ )++ # Back to the original org: its key must see only its own resources.+ UserContext.set_organization_identifier(ORG_ID)++ names = [r["api_name"] for r in list_api_deployments(self.context)["api_deployments"]]+ assert names == ["invoice-api"]++ wf_names = [r["workflow_name"] for r in list_workflows(self.context)["workflows"]]+ assert "secret-wf" not in wf_names++ tool_names = [+ r["tool_name"]+ for r in list_prompt_studio_projects(self.context)["prompt_studio_projects"]+ ]+ assert "Secret Prompts" not in tool_names+ assert other.organization_id == "org-other" # sanity: the other org exists++ def test_whoami_reports_tier_and_service_account_scope(self) -> None:+ """An agent that gets more results than a human would should be able to+ find out why.+ """+ result = whoami(self.context)++ assert result["organization"] == ORG_ID+ assert result["permission_tier"] == "read_write"+ assert result["is_service_account"] is True+ # read_write may write but must not reach destructive tools.+ assert result["can_use_write_tools"] is True+ assert result["can_use_destructive_tools"] is False+ assert "modify" in result["visibility"]++ def test_set_api_deployment_active_toggles_and_reports_the_change(self) -> None:+ result = set_api_deployment_active(+ self.context, api_name="invoice-api", is_active=False+ )++ assert result["changed"] is True+ assert result["previous_state"] is True+ assert result["is_active"] is False+ self.api.refresh_from_db()+ assert self.api.is_active is False++ def test_set_api_deployment_active_is_a_noop_when_already_in_state(self) -> None:+ """Reporting `changed: False` rather than silently succeeding lets an+ agent tell "I did this" from "this was already so".+ """+ result = set_api_deployment_active(+ self.context, api_name="invoice-api", is_active=True+ )++ assert result["changed"] is False++ def test_set_api_deployment_active_rejects_an_unknown_name(self) -> None:+ with self.assertRaises(MCPToolError) as caught:+ set_api_deployment_active(+ self.context, api_name="no-such-api", is_active=False+ )++ # The message must point at the tool that lists valid names.+ assert "listApiDeployments" in str(caught.exception)++ def test_set_pipeline_active_toggles_the_schedule(self) -> None:+ result = set_pipeline_active(+ self.context, pipeline_id=str(self.pipeline.id), active=True+ )++ assert result["changed"] is True+ self.pipeline.refresh_from_db()+ assert self.pipeline.active is True++ def test_execute_pipeline_delegates_to_the_platform_manager(self) -> None:+ """Guards the tool-kwarg -> PipelineManager mapping.++ Nothing else executes this call, so a renamed kwarg would pass every+ other test and fail on the first real pipeline run.+ """+ fake_response = Mock(status_code=200, data={"execution": "started"})+ with patch(+ "pipeline_v2.manager.PipelineManager.execute_pipeline",+ return_value=fake_response,+ ) as execute:+ result = execute_pipeline(self.context, pipeline_id=str(self.pipeline.id))++ execute.assert_called_once_with(+ request=self.context.request, pipeline_id=str(self.pipeline.id)+ )+ assert result["status"] == 200+ assert result["pipeline_name"] == "etl-pipeline"++ def test_execute_pipeline_rejects_an_unknown_id(self) -> None:+ with patch("pipeline_v2.manager.PipelineManager.execute_pipeline") as execute:+ with self.assertRaises(MCPToolError):+ execute_pipeline(+ self.context, pipeline_id="99999999-9999-9999-9999-999999999999"+ )++ execute.assert_not_called()++ def test_writes_do_not_cross_organizations(self) -> None:+ """The most important property of the write tools.++ A platform key must not be able to deactivate, pause or run another+ tenant's resources — the read-side isolation test has a write-side+ equivalent because the consequences here are not merely disclosure.+ """+ UserContext.set_organization_identifier("org-write-other")+ Organization.objects.create(+ name="org-write-other",+ display_name="Other",+ organization_id="org-write-other",+ )+ other_wf = Workflow.objects.create(workflow_name="other-wf", is_active=True)+ other_api = APIDeployment.objects.create(+ api_name="other-api", display_name="Other API", workflow=other_wf+ )+ other_pipeline = Pipeline.objects.create(+ pipeline_name="other-pipeline", workflow=other_wf+ )++ UserContext.set_organization_identifier(ORG_ID)++ with self.assertRaises(MCPToolError):+ set_api_deployment_active(+ self.context, api_name="other-api", is_active=False+ )+ with self.assertRaises(MCPToolError):+ set_pipeline_active(+ self.context, pipeline_id=str(other_pipeline.id), active=True+ )+ with patch("pipeline_v2.manager.PipelineManager.execute_pipeline") as execute:+ with self.assertRaises(MCPToolError):+ execute_pipeline(self.context, pipeline_id=str(other_pipeline.id))+ execute.assert_not_called()++ # And nothing was touched.+ other_api.refresh_from_db()+ other_pipeline.refresh_from_db()+ assert other_api.is_active is True+ assert other_pipeline.active is False++ def test_read_me_first_states_the_scope_warning(self) -> None:+ """The service-account visibility rule is surprising enough that the+ guide must say it outright, not leave it to be discovered.+ """+ result = platform_read_me_first(self.context)++ assert "service account" in result["scope_warning"].lower()+ assert result["organization"] == ORG_ID+ # It must also point at the other server, since this one cannot extract.+ assert "extractDocument" in result["to_run_an_extraction"]+++class PromptStudioProducerToolsTest(TestCase):+ """The two tools that make the billable Prompt Studio tools reachable.++ ``indexDocument``, ``fetchResponse``, ``bulkFetchResponse`` and+ ``singlePassExtraction`` all require ids an agent has no way to invent.+ These tools are the only source of them, so their payload shape is load+ bearing in a way a listing's usually is not — a missing ``document_id``+ field here silently disables every billable tool on the server.+ """++ def setUp(self) -> None:+ self.org = Organization.objects.create(+ name="org-producers",+ display_name="Producers Org",+ organization_id="org-producers",+ )+ UserContext.set_organization_identifier("org-producers")+ self.user = User.objects.create(+ username="svc-producers",+ email="[email protected]",+ user_id="uid-producers",+ is_service_account=True,+ )+ OrganizationMember.objects.create(+ user=self.user, organization=self.org, role="user"+ )+ self.key = PlatformApiKey.objects.create(+ name="producers-key",+ description="d",+ organization=self.org,+ api_user=self.user,+ permission="read_write",+ )+ self.project = CustomTool.objects.create(+ tool_name="Invoice Prompts", description="Prompt project", author="acme"+ )+ self.document = DocumentManager.objects.create(+ document_name="invoice-001.pdf", tool=self.project+ )+ self.prompt = ToolStudioPrompt.objects.create(+ prompt_key="invoice_number",+ prompt="What is the invoice number?",+ tool_id=self.project,+ sequence_number=1,+ prompt_type="Text",+ enforce_type="text",+ # Set deliberately: the redaction assertion below is only meaningful+ # if there is a webhook URL that *could* have leaked.+ postprocessing_webhook_url="https://hooks.internal/acme?token=s3cr3t",+ )+ self.context = PlatformMCPContext(+ user=self.user,+ platform_key=self.key,+ org_name="org-producers",+ request=Mock(data={}),+ )++ def test_documents_are_listed_with_the_id_billable_tools_need(self) -> None:+ result = list_prompt_studio_documents(+ self.context, project_id=str(self.project.tool_id)+ )++ assert result["documents"] == [+ {+ "document_id": str(self.document.document_id),+ "document_name": "invoice-001.pdf",+ }+ ]++ def test_prompts_are_listed_with_the_id_fetch_response_needs(self) -> None:+ result = list_prompts(self.context, project_id=str(self.project.tool_id))++ assert len(result["prompts"]) == 1+ row = result["prompts"][0]+ assert row["prompt_id"] == str(self.prompt.prompt_id)+ assert row["prompt_key"] == "invoice_number"+ assert row["prompt"] == "What is the invoice number?"++ def test_listing_prompts_does_not_leak_adapters_or_webhooks(self) -> None:+ """The promise the README makes, pinned.++ ``ToolStudioPrompt`` carries a ``profile_manager`` FK — the LLM,+ embedding and vector-store adapters behind the prompt — and a+ webhook URL that may embed a token. A serializer would have carried+ both out to the agent, which is why the handler builds its dict by+ hand. This test is what stops someone swapping it back.+ """+ result = list_prompts(self.context, project_id=str(self.project.tool_id))++ payload = json.dumps(result)+ assert "profile_manager" not in payload+ assert "postprocessing_webhook_url" not in payload+ assert "s3cr3t" not in payload, "a webhook token reached the agent"+ assert "hooks.internal" not in payload++ def test_an_unknown_project_is_refused_with_a_usable_message(self) -> None:+ with self.assertRaises(MCPToolError) as caught:+ list_prompts(self.context, project_id="00000000-0000-0000-0000-000000000000")++ assert "listPromptStudioProjects" in str(caught.exception)++ def test_documents_do_not_leak_across_projects(self) -> None:+ """Documents are resolved through the project, never by a bare id+ lookup. The platform key is a service account, for which ``for_user``+ returns everything, so the project join is the scoping that holds.+ """+ other = CustomTool.objects.create(+ tool_name="Other Project", description="d", author="acme"+ )+ DocumentManager.objects.create(document_name="secret.pdf", tool=other)++ result = list_prompt_studio_documents(+ self.context, project_id=str(self.project.tool_id)+ )++ names = [row["document_name"] for row in result["documents"]]+ assert names == ["invoice-001.pdf"], "a sibling project's document leaked"+++class PlatformExtractionScopeTest(TestCase):+ """The organization boundary on platform-side extraction.++ On the deployment server the API key narrowed a caller to one deployment,+ so an execution id from elsewhere was unreachable by construction. Nothing+ narrows a platform caller, and ``get_status_of_async_task`` resolves a bare+ ``WorkflowExecution.objects.get(id=...)`` with no tenant filter — this+ deployment runs one shared schema (``TENANT_APPS`` is empty), so there is+ no per-tenant database to fall back on. These tests pin the check that+ replaces the one the credential used to provide.+ """++ def setUp(self) -> None:+ self.org = Organization.objects.create(+ name="org-extract",+ display_name="Extract Org",+ organization_id="org-extract",+ )+ UserContext.set_organization_identifier("org-extract")+ self.user = User.objects.create(+ username="svc-extract",+ email="[email protected]",+ user_id="uid-extract",+ is_service_account=True,+ )+ OrganizationMember.objects.create(+ user=self.user, organization=self.org, role="user"+ )+ self.key = PlatformApiKey.objects.create(+ name="extract-key",+ description="d",+ organization=self.org,+ api_user=self.user,+ permission="read_write",+ )+ self.workflow = Workflow.objects.create(+ workflow_name="extract-wf", is_active=True+ )+ self.api = APIDeployment.objects.create(+ api_name="extract-api", display_name="Extract API", workflow=self.workflow+ )+ self.context = PlatformMCPContext(+ user=self.user,+ platform_key=self.key,+ org_name="org-extract",+ request=Mock(data={}),+ )++ def _execution_for(self, workflow) -> Any:+ with patch(+ "workflow_manager.workflow_v2.models.execution."+ "WorkflowExecution._handle_execution_cache"+ ):+ return WorkflowExecution.objects.create(+ workflow_id=workflow.id, status="PENDING"+ )++ def test_another_organizations_execution_is_not_readable(self) -> None:+ """The boundary this change actually creates. A platform key must not+ be able to poll an execution belonging to another tenant.+ """+ UserContext.set_organization_identifier("org-extract-other")+ Organization.objects.create(+ name="org-extract-other",+ display_name="Other",+ organization_id="org-extract-other",+ )+ other_wf = Workflow.objects.create(workflow_name="other-wf", is_active=True)+ APIDeployment.objects.create(+ api_name="other-api", display_name="Other API", workflow=other_wf+ )+ foreign = self._execution_for(other_wf)++ UserContext.set_organization_identifier("org-extract")++ with self.assertRaises(MCPToolError) as caught:+ get_platform_execution_status(self.context, execution_id=str(foreign.id))++ # The message must not confirm the id exists elsewhere.+ assert "org-extract-other" not in str(caught.exception)++ def test_an_unknown_execution_is_refused(self) -> None:+ with self.assertRaises(MCPToolError):+ get_platform_execution_status(+ self.context, execution_id="00000000-0000-0000-0000-000000000000"+ )++ def test_a_pipeline_run_is_refused_with_a_pointer_to_the_right_tool(self) -> None:+ """The second gate. A workflow in this org that is not behind an API+ deployment has no deployment to build a context from, so it must be+ refused rather than fail deeper with an unhelpful error.+ """+ pipeline_wf = Workflow.objects.create(+ workflow_name="pipeline-only-wf", is_active=True+ )+ execution = self._execution_for(pipeline_wf)++ with self.assertRaises(MCPToolError) as caught:+ get_platform_execution_status(self.context, execution_id=str(execution.id))++ assert "getExecutionDetail" in str(caught.exception)++ def test_extraction_refuses_a_deployment_outside_the_organization(self) -> None:+ UserContext.set_organization_identifier("org-extract-far")+ Organization.objects.create(+ name="org-extract-far",+ display_name="Far",+ organization_id="org-extract-far",+ )+ far_wf = Workflow.objects.create(workflow_name="far-wf", is_active=True)+ APIDeployment.objects.create(+ api_name="far-api", display_name="Far API", workflow=far_wf+ )++ UserContext.set_organization_identifier("org-extract")++ with self.assertRaises(MCPToolError) as caught:+ platform_extract_document(+ self.context,+ api_name="far-api",+ document_urls=["https://s3.example.com/doc.pdf?X-Amz-Signature=x"],+ )++ assert "listApiDeployments" in str(caught.exception)diff --git a/backend/mcp_server/tests/test_prompt_studio_dispatch.py b/backend/mcp_server/tests/test_prompt_studio_dispatch.pynew file mode 100644index 000000000..0c0b6918d--- /dev/null+++ b/backend/mcp_server/tests/test_prompt_studio_dispatch.py@@ -0,0 +1,433 @@+"""The Prompt Studio billable tools actually deliver their payload.++These four tools were previously executed by no test — they appeared only in+registry name-lists and spend-guard assertions — which is how the dispatch bug+survived: ``_dispatch`` routed through ``PromptStudioCoreView.as_view()``, and+``as_view()`` calls ``initialize_request()``, building a *new* DRF ``Request``+with no ``_full_data``. ``.data`` then re-parsed ``request.body``, so the view+received the JSON-RPC envelope instead of ``document_id``/``prompt_id`` — and+because these tools are billable, budget was consumed before the failure.++Two things are pinned here, and they are different. The payload assertions pin+*argument construction* — that ``fetch_response`` maps ``prompt_id`` onto the+view's ``id`` field, and so on. ``test_the_action_attribute_is_set`` and+``test_get_object_is_reachable`` pin the *dispatch mechanism*, and they are the+ones that fail if the bare-instance path is missing anything DRF needs.++**These tests drive a real ``PromptStudioCoreView`` subclass, not a mock.** An+earlier version of this file patched the view class wholesale, which made every+payload assertion pass against a dispatch that raised ``AttributeError`` in+production: the actions call ``get_object()`` → ``get_queryset()``, which reads+``self.action``, and ``self.action`` is set only by ``initialize_request`` —+i.e. only by ``as_view()``. Mocking the collaborator whose behaviour is at issue+is exactly how the bug this file exists to prevent got through a green suite.++``SimpleTestCase``: ``get_object`` and the four action bodies are stubbed, so no+database is needed. What is *not* stubbed is the bare-instance setup itself —+the stub reads ``self.action`` and ``self.kwargs["pk"]`` exactly as the real+``get_object`` chain does, which is what makes the missing-attribute regression+observable. The action bodies are out of scope here; they are the delegated+view's own responsibility and are covered by its tests.+"""++from __future__ import annotations++import json+from unittest.mock import MagicMock, patch++from django.http import Http404+from django.test import SimpleTestCase+from prompt_studio.prompt_studio_core_v2.views import PromptStudioCoreView+from rest_framework.exceptions import PermissionDenied+from rest_framework.parsers import JSONParser+from rest_framework.request import Request+from rest_framework.response import Response+from rest_framework.test import APIRequestFactory++from mcp_server.tools.prompt_studio import (+ bulk_fetch_response,+ fetch_response,+ index_document,+ single_pass_extraction,+)++PROJECT_ID = "55555555-5555-5555-5555-555555555555"+DOCUMENT_ID = "66666666-6666-6666-6666-666666666666"+PROMPT_ID = "77777777-7777-7777-7777-777777777777"+OTHER_PROMPT_ID = "99999999-9999-9999-9999-999999999999"+PROFILE_ID = "aaaaaaaa-aaaa-aaaa-aaaa-aaaaaaaaaaaa"++# What a real MCP request carries in request.data: the JSON-RPC envelope, not+# the tool's arguments. If dispatch regresses, this is what the view sees.+ENVELOPE = {+ "jsonrpc": "2.0",+ "id": 1,+ "method": "tools/call",+ "params": {"name": "indexDocument", "arguments": {"document_id": DOCUMENT_ID}},+}+++class FakeProject:+ tool_id = PROJECT_ID+ tool_name = "Invoice Prompts"+++def build_request(data=None):+ """A real DRF Request whose body is the JSON-RPC envelope.++ Built the way the transport builds one — the envelope really is what+ ``.data`` re-parses to if dispatch loses ``_full_data``, so a regression+ shows up as the envelope arriving at the action rather than as a mock+ quietly returning whatever it was told to.+ """+ django_request = APIRequestFactory().post(+ "/api/v1/unstract/org-mcp/mcp/",+ data=json.dumps(data if data is not None else ENVELOPE),+ content_type="application/json",+ )+ return Request(django_request, parsers=[JSONParser()])+++class RecordingView(PromptStudioCoreView):+ """A real ``PromptStudioCoreView`` subclass with two things stubbed.++ ``get_object`` is overridden because the real one runs ``for_user``+ querysets and DRF filter backends that need a live DB; the four action+ bodies are replaced by ``_record`` because their real bodies dispatch Celery+ work. Neither stub is the point — both deliberately read ``self.action`` and+ ``self.kwargs["pk"]`` first, the attributes a bare instance is missing, so+ the setup ``_dispatch`` performs is exercised for real and a regression in+ it fails here exactly as it would in production.+ """++ seen: dict = {}++ def get_object(self):+ # Real get_object() → get_queryset() → `self.action == "list"`, and+ # → self.kwargs[lookup]. Touch both so a bare instance missing either+ # raises here just as it does in production.+ _ = self.action == "list"+ _ = self.kwargs["pk"]+ return FakeProject()++ def _record(self, request, pk=None):+ RecordingView.seen = {+ "payload": request.data,+ "pk": pk,+ "action": self.action,+ "project": self.get_object(),+ }+ return Response({"ok": True}, status=200)++ index_document = _record+ fetch_response = _record+ bulk_fetch_response = _record+ single_pass_extraction = _record+++class PromptStudioDispatchTest(SimpleTestCase):+ def _run(self, tool, view_class=None, **kwargs):+ """Invoke a tool against a real view subclass.++ ``view_class`` swaps in a stub whose action raises, for the error-path+ tests; everything else about the dispatch is unchanged. Returns what the+ action observed alongside the tool's own result.+ """+ RecordingView.seen = {}+ request = build_request()+ self.request = request++ context = MagicMock()+ context.request = request+ context.org_name = "org-mcp"++ with (+ patch(+ "mcp_server.tools.prompt_studio._resolve_project",+ return_value=FakeProject(),+ ),+ patch(+ "mcp_server.tools.prompt_studio.PromptStudioCoreView",+ view_class or RecordingView,+ ),+ ):+ result = tool(context, project_id=PROJECT_ID, **kwargs)++ return dict(RecordingView.seen), result++ def test_index_document_delivers_document_id(self) -> None:+ seen, result = self._run(index_document, document_id=DOCUMENT_ID)++ assert seen["payload"] == {"document_id": DOCUMENT_ID}+ assert seen["payload"] != ENVELOPE, "the view received the JSON-RPC envelope"+ assert seen["pk"] == PROJECT_ID+ assert result["ok"] is True++ def test_fetch_response_delivers_prompt_and_document(self) -> None:+ seen, _ = self._run(fetch_response, document_id=DOCUMENT_ID, prompt_id=PROMPT_ID)++ # The view names this field `id`, not `prompt_id`; the mapping is the+ # kind of thing only an executed test catches.+ assert seen["payload"] == {"document_id": DOCUMENT_ID, "id": PROMPT_ID}++ def test_fetch_response_passes_optional_profile(self) -> None:+ seen, _ = self._run(+ fetch_response,+ document_id=DOCUMENT_ID,+ prompt_id=PROMPT_ID,+ profile_manager_id=PROFILE_ID,+ )++ assert seen["payload"]["profile_manager"] == PROFILE_ID++ def test_bulk_fetch_response_delivers_prompt_ids(self) -> None:+ seen, _ = self._run(+ bulk_fetch_response,+ document_id=DOCUMENT_ID,+ prompt_ids=[PROMPT_ID, OTHER_PROMPT_ID],+ )++ assert seen["payload"] == {+ "document_id": DOCUMENT_ID,+ "prompt_ids": [PROMPT_ID, OTHER_PROMPT_ID],+ }++ def test_single_pass_extraction_delivers_document_id(self) -> None:+ seen, _ = self._run(single_pass_extraction, document_id=DOCUMENT_ID)++ assert seen["payload"] == {"document_id": DOCUMENT_ID}++ def test_dispatch_restores_request_data_afterwards(self) -> None:+ """The swap is scoped to the call.++ ``_full_data`` belongs to the live request that the transport is still+ holding; leaving a tool's payload on it would corrupt anything reading+ ``request.data`` later in the same MCP call.+ """+ self._run(index_document, document_id=DOCUMENT_ID)++ assert self.request._full_data == ENVELOPE++ def test_the_action_attribute_is_set(self) -> None:+ """``self.action`` is what ``get_queryset`` branches on.++ ``ViewSetMixin.initialize_request`` normally sets it, and only+ ``as_view()`` calls that — so a bare instance without it raises+ AttributeError inside ``get_object()``. Every one of these tools calls+ ``get_object()``, and the budget is claimed before the handler runs, so+ omitting this fails all four *after* charging for them.+ """+ seen, _ = self._run(index_document, document_id=DOCUMENT_ID)++ assert seen["action"] == "index_document"++ def test_get_object_is_reachable(self) -> None:+ """The action can resolve its project — the whole DRF path works.++ This is the assertion a mocked view cannot make: it is what fails when+ `action`, `kwargs` or `request` is missing from the bare instance.+ """+ seen, _ = self._run(index_document, document_id=DOCUMENT_ID)++ assert isinstance(seen["project"], FakeProject)++ def test_dispatch_restores_request_data_after_a_failure(self) -> None:+ """Restoration is in a finally, so a raising view does not leak it."""++ class Exploding(RecordingView):+ def index_document(self, req, pk=None):+ raise RuntimeError("view exploded")++ with self.assertRaises(RuntimeError):+ self._run(index_document, view_class=Exploding, document_id=DOCUMENT_ID)++ assert self.request._full_data == ENVELOPE++ def test_a_view_refusal_is_returned_as_data(self) -> None:+ """Http404/APIException must not escape as a raw exception.++ ``dispatch()`` would have converted these to a non-2xx Response; the+ bare-instance path bypasses it. ``_result`` documents non-2xx as+ returned-as-data — the agent needs the reason to fix its next call —+ so an escaping exception would become an opaque TOOL_EXECUTION_ERROR+ with the budget already spent.+ """++ class Refusing(RecordingView):+ def index_document(self, req, pk=None):+ raise Http404++ _, result = self._run(+ index_document, view_class=Refusing, document_id=DOCUMENT_ID+ )++ assert result["ok"] is False+ assert result["status"] == 404++ def test_a_permission_denial_is_returned_as_data(self) -> None:+ """Same for check_object_permissions' PermissionDenied."""++ class Denying(RecordingView):+ def index_document(self, req, pk=None):+ raise PermissionDenied("nope")++ _, result = self._run(index_document, view_class=Denying, document_id=DOCUMENT_ID)++ assert result["ok"] is False+ assert result["status"] == 403++ def test_missing_request_is_an_agent_error(self) -> None:+ """A context with no live request refuses rather than raising."""+ from mcp_server.exceptions import MCPToolError++ context = MagicMock()+ context.request = None+ context.org_name = "org-mcp"++ with patch(+ "mcp_server.tools.prompt_studio._resolve_project",+ return_value=FakeProject(),+ ):+ with self.assertRaises(MCPToolError):+ index_document(context, project_id=PROJECT_ID, document_id=DOCUMENT_ID)++ def test_a_server_fault_is_logged_and_not_sold_as_an_argument_error(self) -> None:+ """A 5xx from the view must not read to the agent as "your fault".++ IndexingAPIError, AnswerFetchError and ExtractionAPIError are+ APIException subclasses carrying status_code 500. Catching them turned+ a genuine server fault into an ordinary tool result — with no operator+ log, and a note telling the agent to fix its arguments and try again on+ a tool that charges for every attempt.+ """+ from prompt_studio.prompt_studio_core_v2.exceptions import IndexingAPIError++ class Failing(RecordingView):+ def index_document(self, req, pk=None):+ raise IndexingAPIError("vector store unreachable")++ with patch("mcp_server.tools.prompt_studio.logger") as log:+ _, result = self._run(+ index_document, view_class=Failing, document_id=DOCUMENT_ID+ )++ assert result["status"] == 500+ assert result["ok"] is False+ # The operator signal the transport used to provide before this arm.+ # Asserted as "something was logged at error level" rather than naming+ # logger.exception: the message is routed through log_exception, which+ # deliberately uses logger.error and omits the traceback so a+ # credential in the exception text cannot reach log aggregation.+ log.error.assert_called_once()+ # And the agent is told not to retry-and-pay.+ assert "not because of your arguments" in result["note"]++ def test_a_4xx_still_reads_as_a_correctable_argument_error(self) -> None:+ """The distinction has to cut both ways, or it is just a rename."""+ from rest_framework.exceptions import ValidationError++ class Rejecting(RecordingView):+ def index_document(self, req, pk=None):+ raise ValidationError("document_id is required")++ with patch("mcp_server.tools.prompt_studio.logger") as log:+ _, result = self._run(+ index_document, view_class=Rejecting, document_id=DOCUMENT_ID+ )++ assert result["status"] == 400+ assert "usually a missing or invalid argument" in result["note"]+ log.error.assert_not_called()+ log.exception.assert_not_called()++ def test_a_malformed_document_id_is_refused_before_the_view(self) -> None:+ """Sibling ids reach the same UUID lookups the project id does.++ A Django ValidationError from `DocumentManager.objects.get(pk=...)` is+ neither APIException nor Http404, so it would escape as an opaque+ failure with the budget already spent.+ """+ from mcp_server.exceptions import MCPToolError++ with self.assertRaises(MCPToolError) as caught:+ self._run(index_document, document_id="not-a-uuid")++ assert "not a valid document id" in str(caught.exception)++ def test_a_malformed_prompt_id_is_refused(self) -> None:+ from mcp_server.exceptions import MCPToolError++ with self.assertRaises(MCPToolError):+ self._run(fetch_response, document_id=DOCUMENT_ID, prompt_id="oops")++ def test_a_malformed_id_in_prompt_ids_is_refused(self) -> None:+ """BulkFetchResponse takes a list, so each element needs the guard."""+ from mcp_server.exceptions import MCPToolError++ with self.assertRaises(MCPToolError):+ self._run(+ bulk_fetch_response,+ document_id=DOCUMENT_ID,+ prompt_ids=[PROMPT_ID, "not-a-uuid"],+ )++ def test_a_missing_document_is_returned_as_data(self) -> None:+ """A well-formed but unknown id must not escape as an opaque failure.++ The four actions do not resolve their arguments the same way:+ `bulk_fetch_response` catches `DocumentManager.DoesNotExist` and answers+ 404, but `index_document`, `fetch_response` and `single_pass_extraction`+ call a bare `.objects.get(pk=...)`. That raises `ObjectDoesNotExist`,+ which is neither `APIException` nor `Http404` — so without this arm it+ reached the client as "failed unexpectedly", with the non-refundable+ billable budget already spent on a call that did no work.+ """+ from django.core.exceptions import ObjectDoesNotExist++ class Missing(RecordingView):+ def index_document(self, req, pk=None):+ raise ObjectDoesNotExist++ _, result = self._run(+ index_document, view_class=Missing, document_id=DOCUMENT_ID+ )++ assert result["ok"] is False+ assert result["status"] == 404+ # `str(DoesNotExist())` is empty, so the agent needs to be told what to+ # do rather than handed a blank detail.+ assert "listPromptStudioDocuments" in result["result"]["detail"]++ def test_a_missing_object_is_not_logged_as_a_fault(self) -> None:+ """An agent naming an id that does not exist is ordinary. Logging it at+ error level would make a caller's typo look like a server problem.+ """+ from django.core.exceptions import ObjectDoesNotExist++ class Missing(RecordingView):+ def index_document(self, req, pk=None):+ raise ObjectDoesNotExist++ with patch("mcp_server.tools.prompt_studio.logger") as log:+ self._run(index_document, view_class=Missing, document_id=DOCUMENT_ID)++ log.error.assert_not_called()+ log.exception.assert_not_called()++ def test_a_model_specific_does_not_exist_is_caught(self) -> None:+ """The actions raise `DocumentManager.DoesNotExist`, a subclass — the+ arm must catch the base rather than a specific model's exception.+ """+ from prompt_studio.prompt_studio_document_manager_v2.models import (+ DocumentManager,+ )++ class Missing(RecordingView):+ def index_document(self, req, pk=None):+ raise DocumentManager.DoesNotExist++ _, result = self._run(+ index_document, view_class=Missing, document_id=DOCUMENT_ID+ )++ assert result["status"] == 404diff --git a/backend/mcp_server/tests/test_redaction.py b/backend/mcp_server/tests/test_redaction.pynew file mode 100644index 000000000..9e6f723a7--- /dev/null+++ b/backend/mcp_server/tests/test_redaction.py@@ -0,0 +1,727 @@+"""Redaction of credential-shaped text in execution errors.++Structured fields are safe because they are named explicitly. Free-form error+text is not: a connector that fails to connect reports the connection string it+tried, and a provider client echoes the key it authenticated with. That text+goes straight to the agent, so it is scrubbed first.++This was found by ``test_no_credential_leak`` rather than by review — the+canary seeded in an execution's ``error_message`` came back through+``listExecutions``.+"""++from __future__ import annotations++import json+from unittest.mock import MagicMock++from django.test import SimpleTestCase++from mcp_server.sanitize import _is_secret_key, redact_secrets, redact_structure+++class RedactSecretsTest(SimpleTestCase):+ def test_key_value_secrets_are_masked(self) -> None:+ cases = [+ "connect failed: password=hunter2",+ "connect failed: PASSWORD: hunter2",+ "auth error api_key=sk-abc123",+ "auth error api-key = sk-abc123",+ "bad token=eyJhbGciOi",+ "secret_access_key=AKIAIOSFODNN7EXAMPLE",+ ]+ for text in cases:+ with self.subTest(text):+ out = redact_secrets(text)+ assert "[REDACTED]" in out+ for leaked in ("hunter2", "sk-abc123", "eyJhbGciOi", "AKIA"):+ assert leaked not in out++ def test_connection_string_password_is_masked(self) -> None:+ out = redact_secrets(+ "could not connect to postgresql://admin:[email protected]:5432/x"+ )++ assert "s3cr3tpw" not in out+ assert "[REDACTED]" in out+ # The rest of the URL survives, or the message stops being diagnosable.+ assert "db.internal:5432" in out+ assert "admin" in out++ def test_a_url_with_no_username_is_masked(self) -> None:+ """Redis URLs conventionally omit the username: `redis://:pw@host`.++ The pattern required at least one character before the colon, so the+ most common credential-bearing URL in this codebase was the one shape+ it did not match — and a django-redis connection error renders exactly+ that string.+ """+ out = redact_secrets(+ "Error 111 connecting to redis://:[email protected]:6379/0"+ )++ assert "s3cr3tpw" not in out+ assert "[REDACTED]" in out+ # Still diagnosable: an operator needs to know which host failed.+ assert "cache.internal:6379" in out++ def test_ordinary_urls_are_not_touched(self) -> None:+ """Widening the userinfo must not start mangling credential-free URLs,+ including the pre-signed S3 links these tools legitimately return.+ """+ for url in (+ "https://example.com/a.pdf",+ "redis://localhost:6379/0",+ "https://b.s3.us-east-1.amazonaws.com/x.pdf?X-Amz-Signature=abc",+ "see http://docs.internal/guide#section",+ ):+ with self.subTest(url):+ assert redact_secrets(url) == url++ def test_bearer_token_is_masked(self) -> None:+ out = redact_secrets("401 from provider, sent Authorization: Bearer abcdef123456")++ assert "abcdef123456" not in out+ assert "[REDACTED]" in out++ def test_surrounding_message_is_preserved(self) -> None:+ """Redaction that eats the whole message would trade one problem for+ another — the agent still has to be able to diagnose the failure.+ """+ out = redact_secrets(+ "Failed after 3 attempts on table `invoices`: password=hunter2 (code 28P01)"+ )++ assert "Failed after 3 attempts" in out+ assert "invoices" in out+ assert "28P01" in out+ assert "hunter2" not in out++ def test_ordinary_error_text_is_untouched(self) -> None:+ text = "File not found: invoice-2024-03.pdf (no such object in bucket)"++ assert redact_secrets(text) == text++ def test_empty_and_none_are_safe(self) -> None:+ assert redact_secrets(None) is None+ assert redact_secrets("") == ""++ def test_multiple_secrets_in_one_message_are_all_masked(self) -> None:+ out = redact_secrets("password=one and api_key=two and token=three")++ for leaked in ("one", "two", "three"):+ assert f"={leaked}" not in out+ assert out.count("[REDACTED]") == 3++ def test_prefixed_credential_names_are_masked(self) -> None:+ """The names credentials actually travel under carry a prefix.++ These are the shapes an env dump or a boto/SDK error puts on the wire,+ and every one of them was missed while the pattern anchored on ``\\b``:+ underscore is a word character, so there is no boundary inside+ ``aws_secret_access_key`` for ``\\b`` to find, and the match never+ started. The bare names below were the only ones ever covered.+ """+ cases = {+ # Deliberately not an AKIA-shaped placeholder: what this case pins+ # is the *key name's* prefix, and a realistic-looking value only+ # trips the repo's own secret scanner on the way in.+ "aws_secret_access_key=notarealsecret9999": "notarealsecret9999",+ "db_password=hunter2": "hunter2",+ "x-api-key: sk-abc123": "sk-abc123",+ "X_API_KEY=sk-live-999": "sk-live-999",+ "my_token=ghp_aaaaaaaa": "ghp_aaaaaaaa",+ "client_secret: shhh123": "shhh123",+ "aws_access_key=AKIA1234": "AKIA1234",+ }+ for text, leaked in cases.items():+ with self.subTest(text):+ out = redact_secrets(text)+ assert leaked not in out+ assert "[REDACTED]" in out++ def test_the_two_credential_vocabularies_agree(self) -> None:+ """A name listed in ``_SECRET_KEY_NAMES`` is also matched in free text.++ The two lists claim to share a vocabulary — ``_SECRET_KEY_NAMES``'s own+ comment says "same vocabulary as the key=value pattern above" — and did+ not: these four were redacted as a dict key and left in place in free+ text, which is the same secret reaching an agent by a different route.++ ``private_key`` additionally needed an entry in ``_SECRET_ANCHORS``: the+ literal pre-filter short-circuits before the regexes run, so a keyword+ that appears in no anchor is matchable in principle and unreachable in+ practice.+ """+ cases = {+ "secret_key=sk_live_51": "sk_live_51",+ "private_key=-----BEGINRSA": "BEGINRSA",+ "access_key_id=AKIA9999": "AKIA9999",+ "client_secret=GOCSPX-x": "GOCSPX-x",+ }+ for text, leaked in cases.items():+ with self.subTest(text):+ out = redact_secrets(text)+ assert leaked not in out+ assert "[REDACTED]" in out++ def test_the_plural_credentials_is_deliberately_not_matched(self) -> None:+ """``has_credentials: true`` is an ordinary field and must survive.++ Adding a ``credentials?`` alternative would redact it, because the+ lookbehind permits the ``has_`` prefix and only the trailing ``s``+ currently stops the delimiter lining up. This pattern also runs over+ extracted document output via ``redact_structure``, where a false+ positive destroys real data rather than merely being noisy — so the+ plural is left to ``_is_secret_key``, which can demand a whole-key+ match.+ """+ assert redact_secrets("has_credentials: true") == "has_credentials: true"+ assert _is_secret_key("credentials")++ def test_a_credential_word_inside_a_larger_word_is_not_a_match(self) -> None:+ """Widening the prefix must not widen it to *any* preceding character.++ A ``_``/``-`` separated prefix names a credential; an alphanumeric one+ is a different word that merely ends in the same letters.+ """+ text = "hastoken=notasecret"++ assert redact_secrets(text) == text++ def test_credential_named_fields_without_a_value_are_untouched(self) -> None:+ """The delimiter, not the word boundary, is what stops false positives.++ These are ordinary fields an agent needs in order to act, and they stay+ readable because the keyword is not immediately followed by ``=``/``:``.+ """+ for text in ("token_count=5", "has_credentials: true", "the tokenizer failed"):+ with self.subTest(text):+ assert redact_secrets(text) == text++ def test_bearer_is_matched_case_insensitively(self) -> None:+ """The keyword folds case; the token charset is matched as written.++ Pinned because the two were entangled: a pattern-wide ``(?i)`` made the+ token class ``[A-Za-z...]`` a redundant restatement of itself, and+ scoping the flag to the keyword is only safe if every spelling of+ ``Bearer`` still matches.+ """+ for word in ("Bearer", "bearer", "BEARER", "BeArEr"):+ with self.subTest(word):+ out = redact_secrets(f"sent {word} abcdef123456")++ assert "abcdef123456" not in out+ assert word in out, "the scheme name itself stays, only the token goes"+++class SecretAnchorTest(SimpleTestCase):+ """The literal pre-filter must never change what redaction produces.++ ``redact_secrets`` skips its regex passes when none of ``_SECRET_ANCHORS``+ appears in the text. That is a pure speed optimisation — it walks every+ string in an execution result, including raw OCR text — so it is only safe+ while every pattern genuinely requires one of those literals. A new pattern+ added without its anchor would silently stop matching.+ """++ def _unfiltered(self, text: str) -> str:+ from mcp_server.sanitize import _SECRET_PATTERNS++ out = text+ for pattern, replacement in _SECRET_PATTERNS:+ out = pattern.sub(replacement, out)+ return out++ def test_every_pattern_is_reachable_through_the_anchors(self) -> None:+ """One representative match per pattern, asserted equal both ways."""+ cases = [+ "Authorization: Bearer abcdef123456",+ "connect failed: password=hunter2",+ "auth error api_key=sk-abc123",+ "api-key = sk-abc123",+ "secret_access_key=AKIAIOSFODNN7EXAMPLE",+ "bad token=eyJhbGciOi",+ "could not connect to postgresql://admin:s3cr3tpw@db:5432/x",+ "PASSWORD: hunter2",+ ]+ for text in cases:+ with self.subTest(text):+ assert redact_secrets(text) == self._unfiltered(text)+ assert "[REDACTED]" in redact_secrets(text)++ def test_anchorless_text_is_returned_untouched(self) -> None:+ text = "File not found: invoice-2024-03.pdf (no such object in bucket)"++ assert redact_secrets(text) == self._unfiltered(text) == text++ def test_unicode_case_folding_does_not_slip_past_the_filter(self) -> None:+ """The pre-filter must not disagree with the regex about case.++ ``(?i)`` and ``str.lower()`` are different functions: the regex engine+ folds U+017F (ſ) to "s" and U+212A (K) to "k"; ``.lower()`` leaves both+ alone. So these strings match a pattern while containing no anchor once+ lowercased. An ASCII-only gate on the fast path is what keeps them+ going through the regexes — without it, each is a live credential leak.+ """+ cases = [+ "pa\u017f\u017fword=hunter2",+ "\u017fecret: abc123",+ "\u017fecret_acce\u017f\u017f_key=AKIA1234",+ "PA\u017f\u017fWORD=hunter2",+ "to\u212aen=zzz",+ ]+ for text in cases:+ with self.subTest(text):+ assert redact_secrets(text) == self._unfiltered(text)+ assert "[REDACTED]" in redact_secrets(text), (+ f"{text!r} matches a secret pattern but was not redacted"+ )++ def test_the_filter_never_changes_output_over_a_generated_corpus(self) -> None:+ """The invariant as a property, not a hand-kept list.++ A hand-listed corpus only catches what someone thought to add — which+ is exactly the step a change that adds a pattern forgets. This builds+ inputs from the pattern vocabulary itself, including the case-folding+ characters, so a new pattern without its anchor fails here.+ """+ import itertools++ keys = [+ "password", "passwd", "pwd", "secret", "access_key",+ "secret_access_key", "apikey", "api_key", "api-key", "token",+ "authorization", "credential", "bearer",+ ]+ # Case variants, including the two characters the regex folds and+ # str.lower() does not.+ def variants(word: str):+ yield word+ yield word.upper()+ yield word.replace("s", "\u017f")+ yield word.replace("k", "\u212a")++ seps = ["=", ": ", " = "]+ corpus = []+ for key, sep in itertools.product(+ (v for k in keys for v in variants(k)), seps+ ):+ corpus.append(f"connect failed: {key}{sep}s3cr3t (code 28P01)")+ corpus += [+ "postgresql://admin:pw@host:5432/db",+ "POSTGRESQL://ADMIN:PW@HOST/DB",+ "ordinary text with no secrets",+ "invoice-2024.pdf not found",+ "",+ ]++ for text in corpus:+ with self.subTest(text[:48]):+ assert redact_secrets(text) == self._unfiltered(text)+++class RedactStructureTest(SimpleTestCase):+ """Redaction of upstream payloads the billable and write tools return.++ The read tools serialize named fields, so what they return is known. The+ billable and write tools hand back whatever the delegated view or execution+ helper produced — ``response.data`` including non-2xx error bodies, and raw+ execution results — which is exactly where a failed connector's error text+ arrives verbatim. Previously only free-form error *messages* were scrubbed,+ so ``redact_secrets``'s docstring claimed a coverage it did not have.+ """++ def test_nested_dict_values_are_masked(self) -> None:+ out = redact_structure(+ {+ "error": {+ "detail": "connect failed: password=hunter2",+ "code": "28P01",+ }+ }+ )++ assert "hunter2" not in out["error"]["detail"]+ assert out["error"]["code"] == "28P01"++ def test_secrets_inside_lists_are_masked(self) -> None:+ out = redact_structure(+ [+ {"file": "a.pdf", "error": "api_key=sk-abc123"},+ {"file": "b.pdf", "error": None},+ ]+ )++ assert "sk-abc123" not in out[0]["error"]+ assert out[0]["file"] == "a.pdf"+ assert out[1]["error"] is None++ def test_non_string_leaves_are_preserved(self) -> None:+ """Redaction must not coerce types — an agent reads these as data."""+ payload = {"count": 3, "ok": True, "ratio": 1.5, "missing": None}++ assert redact_structure(payload) == payload++ def test_the_input_is_not_mutated(self) -> None:+ """Containers are rebuilt, so reporting a cached dict or a queryset row+ must not scrub the caller's own copy as a side effect.+ """+ original = {"error": "password=hunter2"}++ redact_structure(original)++ assert original["error"] == "password=hunter2"++ def test_sequences_become_plain_lists(self) -> None:+ """Sequence subclasses are normalised, not reconstructed.++ Rebuilding via ``type(value)(...)`` raises for anything whose __init__+ is not "an iterable" — which on this path means a crash on the return+ of a billable tool, after the money was already spent.+ """+ out = redact_structure(("password=hunter2", "plain"))++ assert isinstance(out, list)+ assert "hunter2" not in out[0]+ assert out[1] == "plain"++ def test_a_namedtuple_does_not_raise(self) -> None:+ from typing import NamedTuple++ class Row(NamedTuple):+ secret: str+ name: str++ out = redact_structure(Row(secret="password=hunter2", name="a.pdf"))++ assert "hunter2" not in out[0]+ assert out[1] == "a.pdf"++ def test_a_drf_return_list_does_not_raise(self) -> None:+ """``ReturnList.__init__`` requires a ``serializer`` kwarg.++ Reachable because the write paths return whatever a delegated+ ``many=True`` serializer produced.+ """+ from rest_framework.utils.serializer_helpers import ReturnList++ payload = ReturnList([{"error": "password=hunter2"}], serializer=object())++ out = redact_structure({"results": payload})++ assert "hunter2" not in json.dumps(out, default=str)++ def test_nested_sequence_subclass_inside_a_dict(self) -> None:+ """The dict branch recurses into the sequence branch, so a nested+ subclass hits the same path a top-level one would.+ """+ from rest_framework.utils.serializer_helpers import ReturnList++ payload = {+ "outer": {"inner": ReturnList(["api_key=sk-abc"], serializer=object())}+ }++ out = redact_structure(payload)++ assert "sk-abc" not in json.dumps(out, default=str)+++class SecretKeyNameTest(SimpleTestCase):+ """A credential-named key with a bare value is redacted.++ The text patterns anchor on a delimiter — `password=x`, `Bearer x`,+ `scheme://u:p@h` — so `{"api_key": "sk-abc"}` matched none of them and+ passed through untouched. That is the ordinary shape of serializer output,+ i.e. exactly what the delegating tools return, so it was the most likely+ leak rather than an edge case. Found by the write-tool leak sweep.+ """++ def test_a_credential_named_key_is_redacted(self) -> None:+ for key in (+ "password",+ "api_key",+ "api-key",+ "apiKey",+ "token",+ "access_token",+ "client_secret",+ "private_key",+ "secret_access_key",+ "CREDENTIALS",+ ):+ with self.subTest(key):+ out = redact_structure({key: "sk-abc123"})+ assert out[key] == "[REDACTED]"++ def test_it_reaches_nested_and_listed_values(self) -> None:+ out = redact_structure(+ {"profile": {"api_key": "sk-abc"}, "rows": [{"token": "t-1"}]}+ )++ assert out["profile"]["api_key"] == "[REDACTED]"+ assert out["rows"][0]["token"] == "[REDACTED]"++ def test_camel_case_credential_keys_are_redacted(self) -> None:+ """AWS-style and JS-style names arrive camelCased in upstream JSON.++ `.lower()` alone compresses `secretAccessKey` to `secretaccesskey`,+ which matches nothing — so the camel boundary has to be split *before*+ lowercasing. These were live leaks until that was fixed.+ """+ for key in (+ "secretAccessKey",+ "accessKeyId",+ "clientSecret",+ "authToken",+ "refreshToken",+ "privateKey",+ "apiKey",+ ):+ with self.subTest(key):+ assert redact_structure({key: "sk-abc"})[key] == "[REDACTED]"++ def test_mixed_separators_normalise_the_same_way(self) -> None:+ for key in ("Api-Key", "API_KEY", "Access Key", "refresh-token"):+ with self.subTest(key):+ assert redact_structure({key: "sk-abc"})[key] == "[REDACTED]"++ def test_ordinary_fields_are_not_redacted(self) -> None:+ """The false-positive guard. Over-redacting makes output unusable to an+ agent without making it safer, so the match is exact rather than a+ substring test.+ """+ payload = {+ "token_count": 1280,+ "has_credentials": True,+ "secret_santa": "Bob",+ "password_hint": "your usual one",+ "tokens_used": 42,+ "authorization_status": "approved",+ }++ assert redact_structure(payload) == payload++ def test_non_string_values_are_left_alone(self) -> None:+ """A credential-named key holding a non-string is not a leaked secret,+ and blanking it would destroy structure an agent reads.+ """+ payload = {"token": None, "api_key": 0, "secret": {"nested": "value"}}+ out = redact_structure(payload)++ assert out["token"] is None+ assert out["api_key"] == 0+ assert out["secret"] == {"nested": "value"}++ def test_key_and_text_redaction_compose(self) -> None:+ out = redact_structure(+ {"api_key": "sk-abc", "log": "connect failed: password=hunter2"}+ )++ assert out["api_key"] == "[REDACTED]"+ assert "hunter2" not in out["log"]+++class WriteToolRedactionCallSitesTest(SimpleTestCase):+ """The billable and write paths actually apply the net.++ ``test_no_credential_leak`` sweeps only the read tools — invoking the+ others would start real executions and spend real budget — so without+ these, dropping a ``redact_structure`` call from a write path would be+ caught by nothing. Each test drives the real function with the upstream+ boundary stubbed to return a canary.+ """++ CANARY = "connect failed: password=hunter2"++ def test_prompt_studio_result_is_redacted(self) -> None:+ from unittest.mock import MagicMock++ from mcp_server.tools.prompt_studio import _result++ response = MagicMock()+ response.status_code = 400+ # Non-2xx bodies are returned as data rather than raised, which is+ # exactly where a delegated view's error text arrives verbatim.+ response.data = {"detail": self.CANARY}+ project = MagicMock()+ project.tool_id = "p1"+ project.tool_name = "P"++ out = _result(response, project)++ assert "hunter2" not in json.dumps(out, default=str)++ def test_execution_status_result_is_redacted(self) -> None:+ from unittest.mock import MagicMock, patch++ from workflow_manager.workflow_v2.dto import ExecutionResponse++ from mcp_server.tools.execution import get_execution_status++ response = ExecutionResponse(+ workflow_id="wf",+ execution_id="e1",+ execution_status="COMPLETED",+ result=[{"file": "a.pdf", "error": self.CANARY}],+ )+ context = MagicMock()+ context.api.workflow_id = "wf"++ with (+ patch("mcp_server.tools.execution.WorkflowExecution.objects.filter") as f,+ patch(+ "mcp_server.tools.execution.ExecutionQuerySerializer.is_valid",+ return_value=True,+ ),+ patch(+ "mcp_server.tools.execution.ExecutionQuerySerializer.validated_data",+ {+ "execution_id": "11111111-1111-1111-1111-111111111111",+ "include_metadata": False,+ "include_metrics": False,+ "include_extracted_text": False,+ },+ ),+ patch(+ "mcp_server.tools.execution.DeploymentHelper.get_execution_status",+ return_value=response,+ ),+ patch(+ "mcp_server.tools.execution.DeploymentHelper.process_completed_execution"+ ),+ ):+ f.return_value.only.return_value.first.return_value = object()+ out = get_execution_status(+ context, execution_id="11111111-1111-1111-1111-111111111111"+ )++ assert "hunter2" not in json.dumps(out, default=str)++ def test_extract_document_result_is_redacted(self) -> None:+ from unittest.mock import MagicMock, patch++ from mcp_server.tools.execution import extract_document++ context = MagicMock()+ context.api.is_active = True++ with (+ patch(+ "mcp_server.tools.execution.ExecutionRequestSerializer.is_valid",+ return_value=True,+ ),+ patch(+ "mcp_server.tools.execution.ExecutionRequestSerializer.validated_data",+ {+ "presigned_urls": [+ "https://x.s3.amazonaws.com/a.pdf?X-Amz-Signature=z"+ ]+ },+ ),+ patch(+ "mcp_server.tools.execution.APIDeploymentRateLimiter.check_and_acquire",+ return_value=(True, {}),+ ),+ patch("mcp_server.tools.execution.DeploymentHelper.load_presigned_files"),+ patch(+ "mcp_server.tools.execution.DeploymentHelper.execute_workflow",+ return_value={"error": self.CANARY},+ ),+ ):+ out = extract_document(+ context,+ document_urls=["https://x.s3.amazonaws.com/a.pdf?X-Amz-Signature=z"],+ )++ assert "hunter2" not in json.dumps(out, default=str)++ def test_execute_pipeline_result_is_redacted(self) -> None:+ from unittest.mock import MagicMock, patch++ from mcp_server.tools.platform import execute_pipeline++ context = MagicMock()+ context.org_name = "org-mcp"+ response = MagicMock()+ response.status_code = 200+ response.data = {"detail": self.CANARY}++ pipeline = MagicMock()+ pipeline.id = "p1"+ pipeline.pipeline_name = "P"++ with (+ patch("mcp_server.tools.platform._resolve_pipeline", return_value=pipeline),+ patch(+ "pipeline_v2.manager.PipelineManager.execute_pipeline",+ return_value=response,+ ),+ ):+ out = execute_pipeline(context, pipeline_id="p1")++ assert "hunter2" not in json.dumps(out, default=str)+++class LogRedactionTest(SimpleTestCase):+ """Upstream exceptions must not carry credentials into the logs.++ `logger.exception(f"...: {error}")` writes both the exception text and a+ traceback, unredacted. On these paths the exception comes from connectors,+ provider clients and the execution stack — a failed connection reports the+ string it tried — and a log line outlives the request, reaching log+ aggregation and an audience the credential was never scoped to.+ """++ def test_the_message_is_redacted(self) -> None:+ from mcp_server.sanitize import log_exception++ log = MagicMock()+ error = RuntimeError("connect failed: password=hunter2")++ log_exception(log, "extraction failed", error)++ log.error.assert_called_once()+ written = log.error.call_args.args[0]+ assert "hunter2" not in written+ assert "[REDACTED]" in written++ def test_the_useful_parts_survive(self) -> None:+ """Redaction that ate the diagnosis would trade one problem for another+ — an operator still has to be able to tell what broke.+ """+ from mcp_server.sanitize import log_exception++ log = MagicMock()+ error = ConnectionError("could not reach db.internal:5432, password=x")++ log_exception(log, "MCP extractDocument failed for api 'live-api'", error)++ written = log.error.call_args.args[0]+ assert "MCP extractDocument failed for api 'live-api'" in written+ assert "ConnectionError" in written+ assert "db.internal:5432" in written++ def test_no_traceback_is_attached(self) -> None:+ """`redact_secrets` can only clean the string it is given; a traceback+ renders frame locals, which is exactly where a connection string sits.+ Losing the stack is the deliberate cost.+ """+ from mcp_server.sanitize import log_exception++ log = MagicMock()++ log_exception(log, "boom", ValueError("x"))++ log.exception.assert_not_called()+ assert log.error.call_args.kwargs.get("exc_info") in (None, False)++ def test_a_structured_credential_in_the_message_is_caught_too(self) -> None:+ """Exception text can carry a serialized dict, not just key=value."""+ from mcp_server.sanitize import log_exception++ log = MagicMock()+ error = RuntimeError("upstream rejected Bearer abcdef123456")++ log_exception(log, "failed", error)++ assert "abcdef123456" not in log.error.call_args.args[0]diff --git a/backend/mcp_server/tests/test_registry_reachability.py b/backend/mcp_server/tests/test_registry_reachability.pynew file mode 100644index 000000000..ca4b510bb--- /dev/null+++ b/backend/mcp_server/tests/test_registry_reachability.py@@ -0,0 +1,467 @@+"""Every id a tool asks for must be obtainable from another tool.++An MCP client sees only ``tools/list``. It has no database, no UI and no way to+guess a UUID, so a tool whose required argument no other tool ever returns is+dead on arrival — it will be listed, attempted, and fail every time.++This shipped once already: the four billable Prompt Studio tools all required a+``document_id`` while nothing in the registry produced one, which made the most+expensive tools on the server the only unreachable ones. The failure was+invisible because each tool was correct in isolation; only the registry as a+whole was wrong. So the invariant is asserted over the registry, not per tool.+"""++from __future__ import annotations++from unittest.mock import Mock++from django.test import SimpleTestCase++from mcp_server.registry import DEPLOYMENT_TOOLS, PLATFORM_TOOLS++# Which tool an agent is expected to call to obtain each opaque id.+#+# Written out rather than inferred from response shapes: handlers return plain+# dicts built at runtime, so inference would mean executing them, and a map+# that guesses would quietly bless the next unreachable id. Adding an entry+# here is a deliberate statement that the named tool really does return it.+PLATFORM_ID_PRODUCERS = {+ "project_id": "listPromptStudioProjects",+ "document_id": "listPromptStudioDocuments",+ "prompt_id": "listPrompts",+ "prompt_ids": "listPrompts",+ "pipeline_id": "listPipelines",+ "workflow_id": "listWorkflows",+ "api_id": "listApiDeployments",+ # listExecutions produces ids for past runs; extractDocument returns the id+ # of the run it just started. Either is a valid handle for polling.+ "execution_id": "listExecutions",+ # Not an id by name, but just as unguessable: an agent cannot invent a+ # deployment's api_name any more than it can invent a UUID.+ "api_name": "listApiDeployments",+}++# Argument names that identify a specific resource the caller must already+# know. Extends past the ``_id`` suffix because ``api_name`` is an opaque+# handle in every way that matters here.+_OPAQUE_SUFFIXES = ("_id", "_ids", "_name")++DEPLOYMENT_ID_PRODUCERS = {+ # The deployment server is scoped to a single API, and its execution id is+ # handed back by the extraction call itself rather than by a listing.+ "execution_id": "extractDocument",+}+++def _required_ids(registry) -> dict[str, list[str]]:+ """Map each id-shaped argument to the tools that require it."""+ consumed: dict[str, list[str]] = {}+ for name in registry.names():+ schema = registry.get(name).input_schema+ required = set(schema.get("required", []))+ for argument in schema.get("properties", {}):+ if argument in required and argument.endswith(_OPAQUE_SUFFIXES):+ consumed.setdefault(argument, []).append(name)+ return consumed+++class PlatformRegistryReachabilityTest(SimpleTestCase):+ def test_every_required_id_has_a_declared_producer(self) -> None:+ consumed = _required_ids(PLATFORM_TOOLS)++ unproducible = {+ argument: tools+ for argument, tools in consumed.items()+ if argument not in PLATFORM_ID_PRODUCERS+ }++ assert unproducible == {}, (+ "These tools require an id no tool is declared to produce, so an "+ f"agent can never call them: {unproducible}. Either add the tool "+ "that returns the id, or record its producer in "+ "PLATFORM_ID_PRODUCERS."+ )++ def test_every_declared_producer_is_actually_registered(self) -> None:+ """The map is only as good as its referents.++ A producer that was renamed or dropped would leave the test above+ passing while the id became unobtainable again.+ """+ missing = sorted(+ {+ producer+ for producer in PLATFORM_ID_PRODUCERS.values()+ if PLATFORM_TOOLS.get(producer) is None+ }+ )++ assert missing == [], f"Declared id producers are not registered: {missing}"++ def test_the_billable_prompt_studio_chain_is_walkable(self) -> None:+ """The specific path that was broken, pinned end to end.++ Named separately from the general invariant because this is the+ sequence an agent must actually perform to spend money: project, then+ document, then prompt, then the billable call.+ """+ for step in (+ "listPromptStudioProjects",+ "listPromptStudioDocuments",+ "listPrompts",+ "fetchResponse",+ ):+ assert PLATFORM_TOOLS.get(step) is not None, f"{step} is missing"++ for producer in ("listPromptStudioDocuments", "listPrompts"):+ tool = PLATFORM_TOOLS.get(producer)+ assert tool.billable is False, (+ f"{producer} exists so an agent can find ids before spending; "+ "making it billable would put the discovery step behind the "+ "budget it is meant to help the agent respect."+ )+ assert tool.required_method == "GET", (+ f"{producer} only reads, so a read-tier key must reach it — "+ "otherwise the billable tools stay unreachable for that key."+ )+++class DeploymentRegistryReachabilityTest(SimpleTestCase):+ def test_every_required_id_has_a_declared_producer(self) -> None:+ consumed = _required_ids(DEPLOYMENT_TOOLS)++ unproducible = {+ argument: tools+ for argument, tools in consumed.items()+ if argument not in DEPLOYMENT_ID_PRODUCERS+ }++ assert unproducible == {}, f"Unreachable on the deployment server: {unproducible}"+++class ReadMeFirstMatchesTheRegistryTest(SimpleTestCase):+ """``readMeFirst`` must name every tool the server actually exposes.++ This is the tool whose entire job is telling an agent how to reach the+ others, and its listing is hand-maintained prose rather than a projection+ of the registry — so it drifts silently. It already did: the two tools that+ produce ``document_id`` and ``prompt_id`` were added to the registry and+ not to the guide, leaving an agent that trusts the guide able to see the+ billable tools with no way to call them.++ Kept as a listing rather than generated from the registry on purpose. The+ ``purpose`` text is written for an agent deciding *whether* to call a tool+ — which is worth more than the tool's own description — so the value is in+ a human writing it. This test only enforces that the set is complete.+ """++ # Keys in the platform guide that enumerate tools. Named explicitly so+ # adding a new grouping is a deliberate act rather than a silent hole.+ PLATFORM_LISTING_KEYS = (+ "discovery_tools",+ "observability_tools",+ "state_change_tools",+ "billable_tools",+ )++ def _named_in(self, guide: dict, keys) -> set[str]:+ return {+ entry["name"]+ for key in keys+ for entry in guide[key]+ if isinstance(entry, dict) and "name" in entry+ }++ def test_platform_guide_names_every_platform_tool(self) -> None:+ from mcp_server.tools.platform import platform_read_me_first++ guide = platform_read_me_first(+ Mock(org_name="org-guide", platform_key=Mock(permission="read_write"))+ )+ listed = self._named_in(guide, self.PLATFORM_LISTING_KEYS)++ # readMeFirst does not list itself; an agent has already called it.+ registered = set(PLATFORM_TOOLS.names()) - {"readMeFirst"}++ assert registered - listed == set(), (+ f"readMeFirst does not mention {sorted(registered - listed)}. An "+ "agent that trusts the guide will never call them — add each to "+ "the appropriate listing in platform_read_me_first."+ )+ assert listed - registered == set(), (+ f"readMeFirst advertises tools that do not exist: "+ f"{sorted(listed - registered)}."+ )++ def test_billable_tools_are_listed_as_billable(self) -> None:+ """A costly tool listed under discovery reads as free to call.++ The grouping is the guide's main signal about consequence, so a tool+ in the wrong group actively misleads rather than merely omitting.+ """+ from mcp_server.tools.platform import platform_read_me_first++ guide = platform_read_me_first(+ Mock(org_name="org-guide", platform_key=Mock(permission="read_write"))+ )+ listed_billable = self._named_in(guide, ("billable_tools",))+ actually_billable = {+ name for name in PLATFORM_TOOLS.names() if PLATFORM_TOOLS.get(name).billable+ }++ assert listed_billable == actually_billable, (+ "readMeFirst's billable_tools must match the registry's billable "+ f"flags exactly. Listed: {sorted(listed_billable)}, actual: "+ f"{sorted(actually_billable)}."+ )++ def test_free_tools_are_not_listed_as_billable(self) -> None:+ """The producers exist so an agent can find ids *before* spending.++ Listing them as billable would push an agent to avoid the very step+ that makes the billable tools callable.+ """+ from mcp_server.tools.platform import platform_read_me_first++ guide = platform_read_me_first(+ Mock(org_name="org-guide", platform_key=Mock(permission="read_write"))+ )+ listed_discovery = self._named_in(guide, ("discovery_tools",))++ for producer in ("listPromptStudioDocuments", "listPrompts"):+ assert producer in listed_discovery, (+ f"{producer} produces the ids the billable tools need, so it "+ "belongs in discovery_tools where an agent looks first."+ )++ def test_deployment_guide_names_every_deployment_tool(self) -> None:+ from mcp_server.tools.info import read_me_first++ guide = read_me_first(+ Mock(api=Mock(display_name="d", description="x", is_active=True))+ )+ listed = self._named_in(guide, ("tools",))+ registered = set(DEPLOYMENT_TOOLS.names()) - {"readMeFirst"}++ assert listed == registered, (+ f"Deployment readMeFirst is out of sync. Missing: "+ f"{sorted(registered - listed)}, phantom: {sorted(listed - registered)}."+ )+++class PlatformExtractionShapeTest(SimpleTestCase):+ """Invariants of extraction on the organization-scoped server.++ Both servers run the same extraction. What differs is the credential, and+ two consequences of that are enforced here because neither is visible at+ the call site.+ """++ def test_platform_extraction_does_not_offer_an_llm_profile(self) -> None:+ """The coupling that keeps a missing api_key from ever being read.++ ``ExecutionRequestSerializer.validate_llm_profile_id`` resolves the+ profile against the API key's owner. A platform caller has no+ deployment key — ``MCPContext.api_key`` is None — and the only reason+ that is safe is that this argument is never offered, so DRF never runs+ that validator. Adding the field back would surface as "Unable to+ validate LLM profile ownership" from a code path with no obvious link+ to this schema.+ """+ schema = PLATFORM_TOOLS.get("extractDocument").input_schema++ assert "llm_profile_id" not in schema["properties"], (+ "llm_profile_id is validated against the API key's owner, which a "+ "platform key does not have. Offering it here would either break "+ "at runtime or require borrowing another principal's key."+ )++ def test_platform_extraction_still_mirrors_the_deployment_limits(self) -> None:+ """The platform schema is derived from the deployment one, so the file+ cap and tag rules cannot drift between the two servers.+ """+ platform = PLATFORM_TOOLS.get("extractDocument").input_schema["properties"]+ deployment = DEPLOYMENT_TOOLS.get("extractDocument").input_schema["properties"]++ assert platform["document_urls"] == deployment["document_urls"]+ assert platform["timeout"] == deployment["timeout"]+ assert platform["tags"] == deployment["tags"]++ def test_platform_extraction_requires_naming_a_deployment(self) -> None:+ """The deployment server gets its target from the credential; here the+ agent must name one, so it is required rather than optional.+ """+ schema = PLATFORM_TOOLS.get("extractDocument").input_schema++ assert "api_name" in schema["required"]+ assert "document_urls" in schema["required"]++ def test_polling_is_free_but_extracting_is_not(self) -> None:+ """An agent told that polling costs money will avoid polling, which is+ the one thing it should do freely while waiting.+ """+ assert PLATFORM_TOOLS.get("extractDocument").billable is True+ assert PLATFORM_TOOLS.get("getExecutionStatus").billable is False+ assert PLATFORM_TOOLS.get("getExecutionStatus").required_method == "GET"+++class ToolAnnotationsTest(SimpleTestCase):+ """`tools/list` advertises behaviour hints a host can act on.++ Without these every tool looks equally dangerous, so a host cannot+ auto-approve `listWorkflows` while prompting on `executePipeline` — which+ is what makes a 23-tool surface tolerable.++ Spec defaults are the trap (`ToolAnnotations`, rev 2025-06-18):+ `readOnlyHint` defaults to false and `destructiveHint` defaults to *true*,+ so silence is the unsafe answer in both directions and every value here is+ stated explicitly.+ """++ def test_every_tool_advertises_annotations_and_a_title(self) -> None:+ for registry in (DEPLOYMENT_TOOLS, PLATFORM_TOOLS):+ for schema in registry.list_schemas():+ with self.subTest(schema["name"]):+ assert schema["title"], "a host shows this in its UI"+ hints = schema["annotations"]+ assert hints["title"] == schema["title"]+ assert isinstance(hints["readOnlyHint"], bool)+ assert hints["openWorldHint"] is True++ def test_read_tools_are_marked_read_only(self) -> None:+ """The whole point: a host can auto-approve these."""+ for registry in (DEPLOYMENT_TOOLS, PLATFORM_TOOLS):+ for name in registry.names():+ tool = registry.get(name)+ if tool.writes or tool.billable or tool.consumes_result:+ continue+ with self.subTest(name):+ assert tool.annotations()["readOnlyHint"] is True++ def test_writing_tools_are_never_marked_read_only(self) -> None:+ """The inverse, and the one that matters for safety: a tool that+ spends money, changes state, or consumes a one-shot result must not be+ advertised as inert.+ """+ for registry in (DEPLOYMENT_TOOLS, PLATFORM_TOOLS):+ for name in registry.names():+ tool = registry.get(name)+ if not (tool.writes or tool.billable or tool.consumes_result):+ continue+ with self.subTest(name):+ assert tool.annotations()["readOnlyHint"] is False++ def test_result_consuming_tools_are_not_advertised_as_read_only(self) -> None:+ """`getExecutionStatus` reads, but reading is destructive.++ The result store is one-shot: a successful read of a COMPLETED+ execution acknowledges it and the payload cannot be fetched again. It+ sets neither `writes` nor `billable` — nothing is written, nothing is+ spent — so without `consumes_result` the derivation would call it+ read-only and a host would auto-approve it, which is how a result gets+ silently lost to a polling loop.++ Named explicitly rather than left to the loops above, because it is the+ only tool on either server in this category and a future refactor that+ drops the flag would otherwise just make one subtest quietly stop+ running.+ """+ for registry in (DEPLOYMENT_TOOLS, PLATFORM_TOOLS):+ tool = registry.get("getExecutionStatus")+ with self.subTest(registry=tool.name):+ assert tool.consumes_result is True+ assert tool.annotations()["readOnlyHint"] is False++ def test_read_tools_omit_the_write_only_hints(self) -> None:+ """`destructiveHint`/`idempotentHint` are defined as meaningful only+ when `readOnlyHint` is false, so emitting them for a read tool would+ publish a value the client is told to ignore.+ """+ tool = PLATFORM_TOOLS.get("listWorkflows")+ hints = tool.annotations()++ assert "destructiveHint" not in hints+ assert "idempotentHint" not in hints++ def test_no_tool_is_advertised_as_destructive(self) -> None:+ """Every write tool here is reversible and none deletes anything —+ a deliberate property of this surface, stated rather than left to the+ spec default of true.+ """+ for registry in (DEPLOYMENT_TOOLS, PLATFORM_TOOLS):+ for name in registry.names():+ tool = registry.get(name)+ with self.subTest(name):+ assert tool.annotations().get("destructiveHint") is not True++ def test_state_changes_are_idempotent_and_executions_are_not(self) -> None:+ """The distinction a host needs to decide whether a retry is safe."""+ for name in ("setApiDeploymentActive", "setPipelineActive"):+ with self.subTest(name):+ assert PLATFORM_TOOLS.get(name).annotations()["idempotentHint"] is True++ for name in ("executePipeline", "indexDocument", "singlePassExtraction"):+ with self.subTest(name):+ assert PLATFORM_TOOLS.get(name).annotations()["idempotentHint"] is False++ def test_deployment_extract_is_not_advertised_as_repeatable(self) -> None:+ """The case that broke a derived-from-`billable` implementation.++ Deployment `extractDocument` is not billable — the deployment server+ has no spend guard, its cost is bounded by the rate limiter — but it+ still starts a fresh execution per call, so it must not be advertised+ as safe to repeat.+ """+ tool = DEPLOYMENT_TOOLS.get("extractDocument")++ assert tool.billable is False+ assert tool.annotations()["idempotentHint"] is False+++class ToolCountIsDocumentedTest(SimpleTestCase):+ """The advertised tool count matches the registry.++ Review found the PR description claiming 19 platform tools against 23+ registered — the listing had not kept up as tools were added. That is the+ kind of drift nobody notices, because the docs are read by people and the+ registry is read by machines.++ This pins the number so adding a tool forces a deliberate decision about+ where it is announced. If it fails, update the count *and* the tool tables+ in README.md and the PR description, not just this constant.+ """++ EXPECTED_PLATFORM_TOOLS = 23+ EXPECTED_DEPLOYMENT_TOOLS = 4++ def test_platform_tool_count(self) -> None:+ actual = len(PLATFORM_TOOLS.names())++ assert actual == self.EXPECTED_PLATFORM_TOOLS, (+ f"Platform registry has {actual} tools, docs say "+ f"{self.EXPECTED_PLATFORM_TOOLS}. Update README.md's tool tables "+ "and the PR/release notes, then this constant."+ )++ def test_deployment_tool_count(self) -> None:+ actual = len(DEPLOYMENT_TOOLS.names())++ assert actual == self.EXPECTED_DEPLOYMENT_TOOLS, (+ f"Deployment registry has {actual} tools, docs say "+ f"{self.EXPECTED_DEPLOYMENT_TOOLS}. Update README.md's tool table, "+ "then this constant."+ )++ def test_the_readme_lists_every_platform_tool(self) -> None:+ """Counting alone would pass if one tool were swapped for another."""+ from pathlib import Path++ readme = (Path(__file__).resolve().parent.parent / "README.md").read_text()++ missing = [name for name in PLATFORM_TOOLS.names() if f"`{name}`" not in readme]++ assert missing == [], (+ f"These platform tools are registered but absent from README.md: "+ f"{missing}. An agent's operator reads that table to know what the "+ "credential can reach."+ )diff --git a/backend/mcp_server/tests/test_spend_guard.py b/backend/mcp_server/tests/test_spend_guard.pynew file mode 100644index 000000000..68007bfeb--- /dev/null+++ b/backend/mcp_server/tests/test_spend_guard.py@@ -0,0 +1,558 @@+"""The billable-call budget.++The guard is what stands between an agent in a retry loop and an unbounded LLM+bill, so its edges are pinned here — particularly the one that contradicts the+neighbouring rate-limiter pattern.+"""++from __future__ import annotations++import json+from unittest.mock import Mock, patch++import redis+from django.core.cache import cache+from django.test import SimpleTestCase, override_settings++from mcp_server import spend_guard+from mcp_server.context import PlatformMCPContext+from mcp_server.exceptions import MCPToolError+from mcp_server.platform_views import PlatformMCPServerView+from mcp_server.registry import PLATFORM_TOOLS, MCPTool++ORG = "org-budget"+++def a_tool(billable: bool) -> MCPTool:+ return MCPTool(+ name="costly" if billable else "cheap",+ description="d",+ input_schema={"type": "object", "properties": {}},+ handler=lambda ctx: None,+ billable=billable,+ required_method="POST" if billable else "GET",+ )+++def context() -> PlatformMCPContext:+ return PlatformMCPContext(+ user=Mock(is_service_account=True),+ platform_key=Mock(permission="read_write"),+ org_name=ORG,+ )+++# The guard's behaviour is what is under test, not Redis's. A local-memory+# cache exercises the same django.core.cache API deterministically and without+# requiring live infra, keeping these in the unit tier.+LOCMEM = {+ "default": {+ "BACKEND": "django.core.cache.backends.locmem.LocMemCache",+ "LOCATION": "mcp-spend-guard-tests",+ }+}+++@override_settings(+ MCP_BILLABLE_CALL_LIMIT=3, MCP_BILLABLE_WINDOW_SECONDS=60, CACHES=LOCMEM+)+class SpendGuardTest(SimpleTestCase):+ def setUp(self) -> None:+ cache.delete(f"mcp:billable:{ORG}")++ def tearDown(self) -> None:+ cache.delete(f"mcp:billable:{ORG}")++ def test_budget_allows_up_to_the_limit_then_refuses(self) -> None:+ for expected_used in (1, 2, 3):+ state = spend_guard.consume(ORG)+ assert state.allowed is True+ assert state.used == expected_used++ exhausted = spend_guard.consume(ORG)+ assert exhausted.allowed is False++ def test_exhaustion_message_reads_as_temporary_not_forbidden(self) -> None:+ """An agent that reads this as a permission error stops retrying; one+ that reads it as a wait comes back later. The wording is the only thing+ that distinguishes them.+ """+ for _ in range(4):+ state = spend_guard.consume(ORG)++ message = state.message()+ assert "temporary" in message.lower()+ assert "retry" in message.lower()+ assert "3" in message # the limit is named++ def test_peek_does_not_consume(self) -> None:+ """Whoami calls peek; surfacing the budget must not spend it."""+ spend_guard.consume(ORG)++ before = spend_guard.peek(ORG)+ after = spend_guard.peek(ORG)++ assert before.used == 1+ assert after.used == 1++ def test_budget_is_per_organization(self) -> None:+ for _ in range(4):+ spend_guard.consume(ORG)++ other = spend_guard.consume("org-other-budget")+ try:+ assert other.allowed is True, "one org's spend must not block another's"+ finally:+ cache.delete("mcp:billable:org-other-budget")++ def test_cache_failure_fails_open(self) -> None:+ """This guard bounds runaway loops; it is not a licence check. Taking+ the whole MCP surface offline because Redis blipped would be the worse+ failure, so it allows and logs.+ """+ broken = Mock()+ broken.add.side_effect = redis.ConnectionError("redis down")+ with patch("mcp_server.spend_guard.get_cache", return_value=broken):+ state = spend_guard.consume(ORG)++ assert state.allowed is True++ def test_cache_errors_covers_what_django_redis_actually_raises(self) -> None:+ """Pin the hierarchy, because it is not the obvious one.++ django-redis's ``omit_exception`` unwraps ``_main_exceptions`` and+ re-raises the underlying cause. Those are redis-py's own exception+ types, and they derive from ``Exception`` — **not** ``OSError``. So a+ ``CACHE_ERRORS`` built from stdlib socket errors alone would silently+ stop catching every real Redis failure, turning documented fail-open+ into fail-closed on a billable call.+ """+ from django_redis.client.default import _main_exceptions++ for exc in _main_exceptions:+ with self.subTest(exc.__name__):+ assert issubclass(exc, spend_guard.CACHE_ERRORS), (+ f"{exc.__name__} is raised by django-redis but not caught "+ "by CACHE_ERRORS — the spend guard would fail closed."+ )++ def test_a_bug_in_the_cache_path_is_not_swallowed(self) -> None:+ """Fail-open covers the cache being unreachable, not every error.++ A bare ``except Exception`` here would also absorb a TypeError or any+ future defect in this module and return allowed=True — silently+ unbudgeting every billable call in the deployment, with a log line as+ the only signal. A real bug must surface.+ """+ broken = Mock()+ broken.add.side_effect = TypeError("bad key type")+ with patch("mcp_server.spend_guard.get_cache", return_value=broken):+ with self.assertRaises(TypeError):+ spend_guard.consume(ORG)++ def test_reseed_failure_also_fails_open(self) -> None:+ """The add/incr expiry race must not invert the documented behaviour.++ ``incr`` raising ValueError means the key expired between the two+ calls; if Redis then drops during the re-seed, an unguarded+ ``cache.set`` would raise out of ``consume`` and fail *closed* — a 500+ on a billable call, the opposite of everything above.+ """+ broken = Mock()+ broken.incr.side_effect = ValueError("key expired")+ broken.set.side_effect = redis.ConnectionError("redis down")+ with patch("mcp_server.spend_guard.get_cache", return_value=broken):+ state = spend_guard.consume(ORG)++ assert state.allowed is True++ def test_peek_also_fails_open(self) -> None:+ """Whoami reports the budget, so an unreachable cache must not break it."""+ broken = Mock()+ broken.get.side_effect = redis.ConnectionError("redis down")+ with patch("mcp_server.spend_guard.get_cache", return_value=broken):+ state = spend_guard.peek(ORG)++ assert state.allowed is True+ assert state.used == 0+++@override_settings(+ MCP_BILLABLE_CALL_LIMIT=2, MCP_BILLABLE_WINDOW_SECONDS=60, CACHES=LOCMEM+)+class SpendGuardEnforcementTest(SimpleTestCase):+ def setUp(self) -> None:+ cache.delete(f"mcp:billable:{ORG}")+ self.view = PlatformMCPServerView()++ def tearDown(self) -> None:+ cache.delete(f"mcp:billable:{ORG}")++ def test_only_billable_tools_consume_budget(self) -> None:+ for _ in range(5):+ assert (+ self.view.check_spend_allowed(a_tool(billable=False), context()) is None+ )++ assert spend_guard.peek(ORG).used == 0++ def test_billable_tool_is_refused_once_the_budget_is_spent(self) -> None:+ assert self.view.check_spend_allowed(a_tool(billable=True), context()) is None+ assert self.view.check_spend_allowed(a_tool(billable=True), context()) is None++ refusal = self.view.check_spend_allowed(a_tool(billable=True), context())++ assert refusal is not None+ assert "budget" in refusal.lower()++ def test_budget_is_not_refunded_when_a_tool_then_fails(self) -> None:+ """Deliberately the opposite of the rate-limit slot in tools/execution.++ That slot models concurrency, so releasing it on failure is right. This+ counter models money already spent: a Prompt Studio call that fails+ partway may have burned tokens upstream, and refunding would let an+ agent spend without limit by failing in a loop.+ """+ self.view.check_spend_allowed(a_tool(billable=True), context())+ used_before = spend_guard.peek(ORG).used++ # Simulate the handler blowing up after the budget was claimed.+ try:+ raise RuntimeError("tool exploded")+ except RuntimeError:+ pass++ assert spend_guard.peek(ORG).used == used_before, (+ "budget must not be refunded on failure"+ )++ def test_probing_after_exhaustion_does_not_reset_the_counter(self) -> None:+ for _ in range(2):+ self.view.check_spend_allowed(a_tool(billable=True), context())++ for _ in range(3):+ refusal = self.view.check_spend_allowed(a_tool(billable=True), context())+ assert refusal is not None+++@override_settings(+ MCP_BILLABLE_CALL_LIMIT=1, MCP_BILLABLE_WINDOW_SECONDS=60, CACHES=LOCMEM+)+class SpendGuardDispatchTest(SimpleTestCase):+ """The budget as an MCP client actually experiences it.++ The enforcement tests above call the hook directly; this drives the real+ JSON-RPC dispatch, which is where the distinction between "refused+ permanently" and "retry later" is actually made.+ """++ def setUp(self) -> None:+ cache.delete(f"mcp:billable:{ORG}")+ self.view = PlatformMCPServerView()+ self.context = context()++ def tearDown(self) -> None:+ cache.delete(f"mcp:billable:{ORG}")++ def _call_billable(self):+ from dataclasses import replace++ tool = replace(+ PLATFORM_TOOLS.get("executePipeline"),+ handler=lambda ctx, **kw: {"ran": True},+ # The real preflight resolves a pipeline_id against the DB, which+ # this unit-tier test has no fixture for. Budget behaviour is what+ # is under test here; preflight has its own tests below.+ preflight=None,+ )+ with patch.object(PLATFORM_TOOLS, "get", return_value=tool):+ response = self.view._call_tool(+ request_id=1,+ params={"name": "executePipeline", "arguments": {}},+ context=self.context,+ )+ return json.loads(response.content)++ def test_first_call_runs_and_second_is_refused_retryably(self) -> None:+ first = self._call_billable()+ assert first["result"]["isError"] is False++ second = self._call_billable()++ # Crucially a *result*, not a JSON-RPC error: clients treat protocol+ # errors as unrecoverable, and this condition clears on its own.+ assert "error" not in second+ assert second["result"]["isError"] is True+ text = second["result"]["content"][0]["text"]+ assert "temporary" in text.lower()+ assert "retry" in text.lower()++ def test_non_billable_tools_still_work_once_the_budget_is_gone(self) -> None:+ """Spending the budget must not take the whole server down with it —+ an agent should still be able to look around and report what happened.+ """+ self._call_billable()+ self._call_billable() # exhausts++ response = self.view._call_tool(+ request_id=2,+ params={"name": "whoami", "arguments": {}},+ context=self.context,+ )+ body = json.loads(response.content)++ assert body["result"]["isError"] is False+++class CacheSelectionTest(SimpleTestCase):+ """The budget lives in the project's shared Django cache.++ There is no MCP-specific DB knob: it was unreachable at its shipped default+ and built a fresh connection pool per billable call when set. Using the+ shared cache is also what keeps ``override_settings(CACHES=...)`` working+ in the tests above.+ """++ def test_budget_uses_the_shared_django_cache(self) -> None:+ from django.core.cache import cache as django_cache++ assert spend_guard.get_cache() is django_cache+++class BillableRegistryInvariantTest(SimpleTestCase):+ def test_every_costly_platform_tool_is_marked_billable(self) -> None:+ """The flag is the only thing wiring a tool into the budget, so a+ costly tool that forgets it is unguarded. Named explicitly rather than+ inferred, so adding one to this list is a deliberate act.+ """+ must_be_billable = {+ "executePipeline",+ "indexDocument",+ "fetchResponse",+ "bulkFetchResponse",+ "singlePassExtraction",+ # The most expensive tool on the server: it runs a whole+ # extraction workflow and consumes the org's extraction quota.+ "extractDocument",+ }++ unguarded = [+ name+ for name in must_be_billable+ if PLATFORM_TOOLS.get(name) is None or not PLATFORM_TOOLS.get(name).billable+ ]++ assert unguarded == [], (+ f"These tools cost money but are not budgeted: {unguarded}"+ )++ def test_billable_tools_are_also_write_gated(self) -> None:+ """Spending money is a write in every sense that matters, so a billable+ tool must never be reachable by a key that cannot write.+ """+ wrong = [+ name+ for name in PLATFORM_TOOLS.names()+ if PLATFORM_TOOLS.get(name).billable+ and PLATFORM_TOOLS.get(name).required_method == "GET"+ ]++ assert wrong == [], f"Billable tools must not be GET-tier: {wrong}"+++@override_settings(+ MCP_BILLABLE_CALL_LIMIT=2, MCP_BILLABLE_WINDOW_SECONDS=60, CACHES=LOCMEM+)+class PreflightProtectsTheBudgetTest(SimpleTestCase):+ """A call that could never have run must not cost a billable slot.++ The budget is consumed on invocation and never refunded, which is right+ for a call that may have spent tokens upstream — but wrong for one refused+ on an argument the server could check for free. ``extractDocument`` is the+ case that motivated this: ``api_name`` is caller-supplied prose rather than+ an id copied from a listing, so an agent guessing at names could exhaust+ the window without ever reaching an LLM.+ """++ def setUp(self) -> None:+ cache.delete(f"mcp:billable:{ORG}")+ self.view = PlatformMCPServerView()+ self.context = context()++ def tearDown(self) -> None:+ cache.delete(f"mcp:billable:{ORG}")++ def _call(self, tool: MCPTool, arguments: dict):+ with patch.object(PLATFORM_TOOLS, "get", return_value=tool):+ response = self.view._call_tool(+ request_id=1,+ params={"name": tool.name, "arguments": arguments},+ context=self.context,+ )+ return json.loads(response.content)++ def _billable_tool(self, preflight) -> MCPTool:+ from dataclasses import replace++ return replace(+ PLATFORM_TOOLS.get("extractDocument"),+ handler=lambda ctx, **kw: {"ran": True},+ preflight=preflight,+ )++ def test_a_failed_preflight_does_not_consume_budget(self) -> None:+ def refuse(ctx, **kwargs):+ raise MCPToolError("No API deployment named 'typo'.")++ body = self._call(self._billable_tool(refuse), {"api_name": "typo"})++ assert body["result"]["isError"] is True+ assert "typo" in body["result"]["content"][0]["text"]+ assert spend_guard.peek(ORG).used == 0, (+ "a call refused before it could run must not cost a billable slot"+ )++ def test_repeated_bad_names_never_exhaust_the_window(self) -> None:+ """The failure mode this exists to prevent: an agent guessing at+ deployment names locking the organization out of real extractions.+ """++ def refuse(ctx, **kwargs):+ raise MCPToolError("No such deployment.")++ for _ in range(5):+ self._call(self._billable_tool(refuse), {"api_name": "guess"})++ assert spend_guard.peek(ORG).used == 0++ # The budget is still intact for a call that does resolve.+ body = self._call(self._billable_tool(lambda ctx, **kw: None), {})+ assert body["result"]["isError"] is False+ assert spend_guard.peek(ORG).used == 1++ def test_a_passing_preflight_still_consumes_budget(self) -> None:+ """Preflight guards the budget; it must not become a way around it."""+ body = self._call(self._billable_tool(lambda ctx, **kw: None), {})++ assert body["result"]["isError"] is False+ assert spend_guard.peek(ORG).used == 1++ def test_preflight_runs_before_the_budget_not_merely_instead_of_it(self) -> None:+ """Ordering, pinned directly.++ If the budget were claimed first and the preflight merely reported the+ refusal, the counter would still have moved. Exhausting the budget and+ then calling with a bad name distinguishes the two: with correct+ ordering the caller is told the name is wrong, not that it is broke.+ """++ def refuse(ctx, **kwargs):+ raise MCPToolError("No API deployment named 'typo'.")++ for _ in range(2):+ self._call(self._billable_tool(lambda ctx, **kw: None), {})+ assert spend_guard.peek(ORG).used == 2 # exhausted++ body = self._call(self._billable_tool(refuse), {"api_name": "typo"})++ text = body["result"]["content"][0]["text"]+ assert "typo" in text, f"budget refusal masked the real problem: {text}"++ def test_every_billable_tool_has_a_preflight(self) -> None:+ """Each billable tool resolves something before it spends, so each can+ refuse for free. A new one without a preflight silently reintroduces+ the burn-a-slot-on-a-typo behaviour.+ """+ unguarded = [+ name+ for name in PLATFORM_TOOLS.names()+ if PLATFORM_TOOLS.get(name).billable+ and PLATFORM_TOOLS.get(name).preflight is None+ ]++ assert unguarded == [], (+ "These billable tools consume budget before validating their "+ f"arguments: {unguarded}. Give each a preflight that resolves its "+ "target, or state here why it cannot fail cheaply."+ )+++class CacheErrorLoggingTest(SimpleTestCase):+ """Cache failures must not write the Redis URL into the logs.++ A django-redis connection error renders the URL it tried, which carries the+ password in any deployment that sets one. `peek` was converted to+ `log_exception` first and both `consume` branches were missed — the kind of+ gap that survives review because the three sites are 50 lines apart and+ catch the same `CACHE_ERRORS`.+ """++ URL_ERROR = "Error 111 connecting to redis://:[email protected]:6379/0"++ def _assert_no_secret(self, log) -> None:+ written = " ".join(+ str(call.args[0]) for call in log.error.call_args_list + log.warning.call_args_list+ )+ assert "s3cr3tpw" not in written, (+ f"a Redis password reached the logs: {written}"+ )+ assert written, "nothing was logged; the failure would be invisible"++ def test_consume_does_not_log_the_redis_url(self) -> None:+ broken = Mock()+ broken.add.side_effect = redis.ConnectionError(self.URL_ERROR)++ with (+ patch("mcp_server.spend_guard.get_cache", return_value=broken),+ patch("mcp_server.spend_guard.logger") as log,+ ):+ state = spend_guard.consume(ORG)++ assert state.allowed is True, "must still fail open"+ self._assert_no_secret(log)++ def test_the_reseed_branch_does_not_log_the_redis_url(self) -> None:+ """The add/incr expiry race — a separate handler, missed the first time."""+ broken = Mock()+ broken.incr.side_effect = ValueError("key expired")+ broken.set.side_effect = redis.ConnectionError(self.URL_ERROR)++ with (+ patch("mcp_server.spend_guard.get_cache", return_value=broken),+ patch("mcp_server.spend_guard.logger") as log,+ ):+ state = spend_guard.consume(ORG)++ assert state.allowed is True+ self._assert_no_secret(log)++ def test_peek_does_not_log_the_redis_url(self) -> None:+ broken = Mock()+ broken.get.side_effect = redis.ConnectionError(self.URL_ERROR)++ with (+ patch("mcp_server.spend_guard.get_cache", return_value=broken),+ patch("mcp_server.spend_guard.logger") as log,+ ):+ spend_guard.peek(ORG)++ self._assert_no_secret(log)++ def test_no_handler_in_this_module_logs_a_raw_exception(self) -> None:+ """The invariant, rather than three examples of it.++ Catches a *new* cache handler added with `logger.error(f"...{error}")`,+ which is how the two `consume` branches came to differ from `peek`.+ """+ import inspect+ import re++ source = inspect.getsource(spend_guard)+ offenders = re.findall(+ r"logger\.(?:error|warning|exception)\([^)]*\{error\}", source, re.S+ )++ assert offenders == [], (+ "These log calls interpolate a raw exception; route them through "+ f"sanitize.log_exception so a Redis URL cannot leak: {offenders}"+ )diff --git a/backend/mcp_server/tests/test_tool_errors.py b/backend/mcp_server/tests/test_tool_errors.pynew file mode 100644index 000000000..b365a08f0--- /dev/null+++ b/backend/mcp_server/tests/test_tool_errors.py@@ -0,0 +1,147 @@+"""Tool-layer error messages, which are read by an agent rather than a human.++A tool error is the agent's next prompt: if it does not name the offending+argument in plain text, the agent's most likely next move is to retry the same+call — and for `extractDocument` a retry spends the organization's quota.+"""++from __future__ import annotations++from django.test import SimpleTestCase+from rest_framework.exceptions import ValidationError++from mcp_server.tools.execution import _format_validation_error+++class FormatValidationErrorTest(SimpleTestCase):+ def test_flattens_nested_detail_without_errordetail_repr(self) -> None:+ """DRF nests per-item errors under the field name and wraps each in an+ ``ErrorDetail``; both would otherwise reach the agent verbatim.+ """+ error = ValidationError({"presigned_urls": {0: ["Enter a valid URL."]}})++ message = _format_validation_error(error)++ assert message == "presigned_urls.0: Enter a valid URL."+ assert "ErrorDetail" not in message++ def test_reports_every_failing_field(self) -> None:+ """Surfacing only the first failure would send the agent round the loop+ once per bad argument.+ """+ error = ValidationError(+ {"timeout": ["Ensure this value is at most 300."], "tags": ["Invalid tag."]}+ )++ message = _format_validation_error(error)++ assert "timeout: Ensure this value is at most 300." in message+ assert "tags: Invalid tag." in message++ def test_plain_message_passes_through_unchanged(self) -> None:+ error = ValidationError("Something was wrong.")++ assert _format_validation_error(error) == "Something was wrong."+++class DocumentUrlContractTest(SimpleTestCase):+ """What `extractDocument`'s docs promise vs what the serializer enforces.++ These are pinned together because they had already drifted apart. Every+ description said a URL without a signature is rejected; the validator only+ ever checked the *host*. The drift was not theoretical — two independent+ agent runs read that sentence, concluded a working call would be refused,+ and declined to make it, one of them noting the tool's own warning about+ speculative calls. A wrong doc string on an MCP tool is not cosmetic: the+ description *is* the interface, and it talked a valid call out of existing.+ """++ def _validate(self, url: str) -> str | None:+ """Return the rejection message, or None when the URL is accepted."""+ from api_v2.serializers import ExecutionRequestSerializer++ serializer = ExecutionRequestSerializer()+ try:+ serializer._validate_presigned_url(url)+ except ValidationError as error:+ return str(error.detail)+ return None++ def test_an_unsigned_public_s3_url_is_accepted(self) -> None:+ """The case the docs used to deny.++ A publicly-readable object needs no signature, and nothing in the+ validator asks for one. Confirmed end to end against a live deployment:+ an unsigned public S3 URL extracted successfully.+ """+ assert self._validate("https://s3.amazonaws.com/some-bucket/doc.pdf") is None++ def test_a_pre_signed_url_is_accepted(self) -> None:+ """The ordinary case: a private object reached with a signature."""+ signed = (+ "https://b.s3.us-east-1.amazonaws.com/doc.pdf"+ "?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Signature=abc"+ )++ assert self._validate(signed) is None++ def test_a_non_s3_host_is_rejected(self) -> None:+ """The restriction that *is* real, and the one the docs should state."""+ message = self._validate("https://evil.example.com/doc.pdf")++ assert message is not None+ assert "S3" in message++ def test_plain_http_is_rejected_even_on_an_s3_host(self) -> None:+ message = self._validate("http://s3.amazonaws.com/some-bucket/doc.pdf")++ assert message is not None+ assert "HTTPS" in message++ def test_no_description_claims_an_unsigned_url_is_rejected(self) -> None:+ """The docs must not re-acquire the claim the validator does not make.++ Asserted over the three places a client actually reads — the tool+ description, the argument schema, and the readMeFirst guide — because+ the wording was wrong in all three and fixing one would have left an+ agent reading either of the others.+ """+ from unittest.mock import MagicMock++ from mcp_server.registry import build_deployment_registry+ from mcp_server.tools.execution import extract_document_schema+ from mcp_server.tools.info import read_me_first++ context = MagicMock()+ context.api.display_name = "d"+ context.api.description = "d"+ context.api.is_active = True++ schema = extract_document_schema()+ extract_tool = build_deployment_registry().get("extractDocument")+ texts = {+ "document_urls schema": schema["properties"]["document_urls"]["description"],+ "extractDocument description": extract_tool.description,+ "readMeFirst notes": " ".join(read_me_first(context)["notes"]),+ }+ # Guarding the exact sentences that were wrong, not word proximity:+ # the corrected text legitimately contains both "publicly readable" and+ # "rejected" (of a non-S3 host), so a co-occurrence check fails on+ # correct wording.+ false_claims = (+ "only s3 pre-signed urls are accepted",+ "ordinary public link is rejected",+ "ordinary public links are rejected",+ "ordinary public http(s) link is rejected",+ "public links are rejected",+ )+ for label, text in texts.items():+ with self.subTest(label):+ lowered = " ".join(text.lower().split())+ # The restriction that is real must be stated.+ assert "s3" in lowered+ for claim in false_claims:+ assert claim not in lowered, (+ f"{label} still says {claim!r}; the validator checks the "+ "host only, and accepts an unsigned public S3 URL"+ )diff --git a/backend/mcp_server/tests/test_tool_execution.py b/backend/mcp_server/tests/test_tool_execution.pynew file mode 100644index 000000000..94c2872e8--- /dev/null+++ b/backend/mcp_server/tests/test_tool_execution.py@@ -0,0 +1,455 @@+"""The success path of the extraction tools.++The transport and auth tests never reach these helpers on a successful call, so+without this file the argument mapping from tool kwargs into+``execute_workflow`` is executed by nothing — a renamed or mis-mapped kwarg+would pass every other test and fail on the first real extraction. The+``assert_called_once_with`` below is the point of these tests.++Mocked at the ``DeploymentHelper`` boundary so no database or Celery is needed.+"""++from __future__ import annotations++from unittest.mock import patch++from django.test import SimpleTestCase+from rest_framework.exceptions import ValidationError+from workflow_manager.workflow_v2.dto import ExecutionResponse++from mcp_server.context import MCPContext+from mcp_server.exceptions import MCPToolError+from mcp_server.tools.execution import extract_document, get_execution_status++# The execution serializer accepts only HTTPS S3 pre-signed URLs; using a+# realistic one here keeps these tests honest about what the tool accepts.+DOC_URL = "https://my-bucket.s3.us-east-1.amazonaws.com/invoice.pdf?X-Amz-Signature=abc"+EXECUTION_ID = "33333333-3333-3333-3333-333333333333"+WORKFLOW_ID = "44444444-4444-4444-4444-444444444444"+# The deployment's own id. Executions record it as `pipeline_id`, and that —+# not workflow_id — is what scopes a poll to one deployment.+DEPLOYMENT_ID = "88888888-8888-8888-8888-888888888888"+++class FakeOrganization:+ organization_id = "org-mcp"+++class FakeAPI:+ display_name = "Invoice Extractor"+ api_name = "live-api"+ is_active = True+ organization = FakeOrganization()+ # get_execution_status scopes the polled id to this deployment itself.+ id = DEPLOYMENT_ID+ workflow_id = WORKFLOW_ID+++def make_context(active: bool = True) -> MCPContext:+ api = FakeAPI()+ api.is_active = active+ return MCPContext(api=api, api_key="test-key", org_name="org-mcp")+++class ExtractDocumentTest(SimpleTestCase):+ def setUp(self) -> None:+ self.context = make_context()++ def _run(self, execute_return=None, **kwargs):+ """Drive extract_document with the whole execution stack stubbed."""+ with (+ patch(+ "mcp_server.tools.execution.DeploymentHelper.load_presigned_files"+ ) as load_files,+ patch(+ "mcp_server.tools.execution.APIDeploymentRateLimiter.check_and_acquire",+ return_value=(True, {}),+ ),+ patch(+ "mcp_server.tools.execution.APIDeploymentRateLimiter.release_slot"+ ) as release,+ patch(+ "mcp_server.tools.execution.DeploymentHelper.execute_workflow",+ return_value=execute_return+ if execute_return is not None+ else {"execution_status": "COMPLETED", "message": "ok"},+ ) as execute,+ ):+ result = extract_document(self.context, **kwargs)+ return result, execute, load_files, release++ def test_arguments_map_onto_execute_workflow(self) -> None:+ """Guards the tool-kwarg -> execute_workflow mapping. A rename on+ either side breaks here rather than in production.+ """+ result, execute, load_files, _ = self._run(+ document_urls=[DOC_URL],+ timeout=45,+ include_metadata=True,+ include_metrics=True,+ include_extracted_text=True,+ tags=["alpha"],+ )++ load_files.assert_called_once()+ assert load_files.call_args.args[0] == [DOC_URL]++ execute.assert_called_once()+ kwargs = execute.call_args.kwargs+ assert kwargs["organization_name"] == "org-mcp"+ assert kwargs["api"] is self.context.api+ assert kwargs["timeout"] == 45+ assert kwargs["include_metadata"] is True+ assert kwargs["include_metrics"] is True+ assert kwargs["include_extracted_text"] is True+ # Agents send a JSON array; the execution layer expects parsed names.+ assert kwargs["tag_names"] == ["alpha"]+ assert kwargs["execution_id"] == result["execution_id"]++ def test_result_always_carries_execution_id(self) -> None:+ """On the pending path the id is the agent's only handle for polling,+ and execute_workflow does not guarantee it in the body.+ """+ result, execute, _, _ = self._run(+ execute_return={"execution_status": "PENDING"}, document_urls=[DOC_URL]+ )++ assert result["execution_status"] == "PENDING"+ assert result["execution_id"] == execute.call_args.kwargs["execution_id"]++ def test_existing_execution_id_in_response_is_preserved(self) -> None:+ """Never overwrite an id the execution layer reported itself."""+ result, _, _, _ = self._run(+ execute_return={"execution_status": "COMPLETED", "execution_id": "from-core"},+ document_urls=[DOC_URL],+ )++ assert result["execution_id"] == "from-core"++ def test_defaults_are_conservative(self) -> None:+ """Defaults must not silently opt an agent into large or costly output."""+ _, execute, _, _ = self._run(document_urls=[DOC_URL])++ kwargs = execute.call_args.kwargs+ assert kwargs["include_metadata"] is False+ assert kwargs["include_metrics"] is False+ assert kwargs["include_extracted_text"] is False+ assert kwargs["tag_names"] == []++ def test_inactive_deployment_rejected_before_quota_is_spent(self) -> None:+ with patch(+ "mcp_server.tools.execution.DeploymentHelper.execute_workflow"+ ) as execute:+ with self.assertRaises(MCPToolError):+ extract_document(make_context(active=False), document_urls=[DOC_URL])++ execute.assert_not_called()++ def test_non_s3_url_rejected_with_an_explanation(self) -> None:+ """Only S3 pre-signed URLs are accepted. The tool description says so,+ and this pins the behaviour the description promises — an agent handed+ an ordinary link must be told why, not just that it failed.+ """+ with patch(+ "mcp_server.tools.execution.DeploymentHelper.execute_workflow"+ ) as execute:+ with self.assertRaises(MCPToolError) as caught:+ extract_document(+ self.context, document_urls=["https://example.com/invoice.pdf"]+ )++ execute.assert_not_called()+ assert "S3" in str(caught.exception)++ def test_invalid_url_rejected_before_quota_is_spent(self) -> None:+ with (+ patch(+ "mcp_server.tools.execution.APIDeploymentRateLimiter.check_and_acquire"+ ) as acquire,+ patch(+ "mcp_server.tools.execution.DeploymentHelper.execute_workflow"+ ) as execute,+ ):+ with self.assertRaises(MCPToolError):+ extract_document(self.context, document_urls=["not-a-url"])++ acquire.assert_not_called()+ execute.assert_not_called()++ def test_rate_limited_call_downloads_nothing(self) -> None:+ """The slot is taken before the documents are fetched, so a rejected+ call must not pull every document over the network first.+ """+ with (+ patch(+ "mcp_server.tools.execution.DeploymentHelper.load_presigned_files"+ ) as load_files,+ patch(+ "mcp_server.tools.execution.APIDeploymentRateLimiter.check_and_acquire",+ return_value=(+ False,+ {"current_usage": 5, "limit": 5, "limit_type": "organization"},+ ),+ ),+ patch(+ "mcp_server.tools.execution.DeploymentHelper.execute_workflow"+ ) as execute,+ ):+ with self.assertRaises(MCPToolError) as caught:+ extract_document(self.context, document_urls=[DOC_URL])++ load_files.assert_not_called()+ execute.assert_not_called()+ assert "5/5" in str(caught.exception)++ def test_failed_download_releases_the_rate_limit_slot(self) -> None:+ """The fetch happens after the slot is taken, so a fetch failure has to+ give the slot back or the org silently loses throughput.+ """+ with (+ patch(+ "mcp_server.tools.execution.DeploymentHelper.load_presigned_files",+ side_effect=RuntimeError("403 Forbidden"),+ ),+ patch(+ "mcp_server.tools.execution.APIDeploymentRateLimiter.check_and_acquire",+ return_value=(True, {}),+ ),+ patch(+ "mcp_server.tools.execution.APIDeploymentRateLimiter.release_slot"+ ) as release,+ patch(+ "mcp_server.tools.execution.DeploymentHelper.execute_workflow"+ ) as execute,+ ):+ with self.assertRaises(MCPToolError):+ extract_document(self.context, document_urls=[DOC_URL])++ release.assert_called_once()+ execute.assert_not_called()++ def test_failed_execution_releases_the_rate_limit_slot(self) -> None:+ """A leaked slot degrades the org's throughput permanently, so the+ release must survive the failure path.+ """+ with (+ patch("mcp_server.tools.execution.DeploymentHelper.load_presigned_files"),+ patch(+ "mcp_server.tools.execution.APIDeploymentRateLimiter.check_and_acquire",+ return_value=(True, {}),+ ),+ patch(+ "mcp_server.tools.execution.APIDeploymentRateLimiter.release_slot"+ ) as release,+ patch(+ "mcp_server.tools.execution.DeploymentHelper.execute_workflow",+ side_effect=RuntimeError("boom"),+ ),+ ):+ with self.assertRaises(MCPToolError):+ extract_document(self.context, document_urls=[DOC_URL])++ release.assert_called_once()+++class GetExecutionStatusTest(SimpleTestCase):+ def setUp(self) -> None:+ self.context = make_context()++ def _run(self, response: ExecutionResponse, scoped: bool = True, **kwargs):+ """Drive get_execution_status with the ORM scope check stubbed.++ ``scoped=False`` models the id belonging to another deployment: the+ filter finds nothing, which is what the guard keys off.+ """+ with (+ patch(+ "mcp_server.tools.execution.WorkflowExecution.objects.filter"+ ) as scope_filter,+ patch(+ "mcp_server.tools.execution.ExecutionQuerySerializer.is_valid",+ return_value=True,+ ),+ patch(+ "mcp_server.tools.execution.ExecutionQuerySerializer.validated_data",+ {+ "execution_id": EXECUTION_ID,+ "include_metadata": kwargs.get("include_metadata", False),+ "include_metrics": kwargs.get("include_metrics", False),+ "include_extracted_text": kwargs.get("include_extracted_text", False),+ },+ ),+ patch(+ "mcp_server.tools.execution.DeploymentHelper.get_execution_status",+ return_value=response,+ ),+ patch(+ "mcp_server.tools.execution.DeploymentHelper.process_completed_execution"+ ) as process,+ ):+ scope_filter.return_value.only.return_value.first.return_value = (+ object() if scoped else None+ )+ result = get_execution_status(+ self.context, execution_id=EXECUTION_ID, **kwargs+ )+ self.scope_filter = scope_filter+ return result, process++ def test_completed_execution_is_enriched_and_returned(self) -> None:+ response = ExecutionResponse(+ workflow_id="wf",+ execution_id=EXECUTION_ID,+ execution_status="COMPLETED",+ result=[{"file": "invoice.pdf", "result": {"output": {"total": 42}}}],+ )++ result, process = self._run(response, include_metadata=True)++ # Enrichment is what turns the raw execution into the API-shaped+ # result; skipping it would silently return a thinner payload.+ process.assert_called_once()+ assert process.call_args.kwargs["include_metadata"] is True+ assert result["execution_status"] == "COMPLETED"+ assert result["result"][0]["result"]["output"]["total"] == 42+ assert result["execution_id"] == EXECUTION_ID++ def test_pending_execution_is_not_enriched(self) -> None:+ response = ExecutionResponse(+ workflow_id="wf",+ execution_id=EXECUTION_ID,+ execution_status="EXECUTING",+ result=None,+ )++ result, process = self._run(response)++ process.assert_not_called()+ assert result["execution_status"] == "EXECUTING"++ def test_polled_id_is_scoped_to_this_deployment(self) -> None:+ """The scope filter is anchored to the deployment, not just the id.++ A filter on ``id`` alone would pass this test's sibling below while+ leaving every execution in the shared schema pollable.++ It must anchor on ``pipeline_id`` (the deployment that started the run)+ rather than ``workflow_id``: ``APIDeployment.workflow`` is a plain FK+ with no uniqueness constraint, so two deployments can share a workflow+ and a workflow-scoped filter would let either poll the other's runs.+ """+ response = ExecutionResponse(+ workflow_id="wf",+ execution_id=EXECUTION_ID,+ execution_status="EXECUTING",+ result=None,+ )++ self._run(response)++ self.scope_filter.assert_called_once_with(+ id=EXECUTION_ID, pipeline_id=DEPLOYMENT_ID+ )++ def test_two_deployments_sharing_a_workflow_cannot_poll_each_other(self) -> None:+ """The case a workflow-scoped filter would have missed.++ `APIDeployment.workflow` is a plain FK with `related_name="apis"` and no+ uniqueness constraint, so two deployments can point at one workflow.+ Filtering on `workflow_id` would then match both deployments' runs; the+ filter must name the deployment itself.+ """+ response = ExecutionResponse(+ workflow_id=WORKFLOW_ID,+ execution_id=EXECUTION_ID,+ execution_status="EXECUTING",+ result=None,+ )++ self._run(response)++ kwargs = self.scope_filter.call_args.kwargs+ assert kwargs["pipeline_id"] == DEPLOYMENT_ID+ assert "workflow_id" not in kwargs, (+ "scoping on workflow_id lets a sibling deployment sharing the same "+ "workflow poll this deployment's executions"+ )++ def test_execution_from_another_deployment_is_refused(self) -> None:+ """Polling another deployment's execution must not reach the helper.++ DeploymentHelper.get_execution_status resolves the id with no tenant+ filter and acknowledging a completed result drops it from cache, so+ reaching it at all would both disclose the result and destroy it for+ the caller it belongs to.+ """+ response = ExecutionResponse(+ workflow_id="wf",+ execution_id=EXECUTION_ID,+ execution_status="COMPLETED",+ result=[{"file": "secret.pdf", "result": {"output": {"total": 42}}}],+ )++ with self.assertRaises(MCPToolError) as caught:+ self._run(response, scoped=False)++ assert "No execution with id" in str(caught.exception)++ def test_foreign_execution_never_reaches_the_helper(self) -> None:+ """The guard runs before the fetch, not alongside it."""+ with (+ patch(+ "mcp_server.tools.execution.WorkflowExecution.objects.filter"+ ) as scope_filter,+ patch(+ "mcp_server.tools.execution.ExecutionQuerySerializer.is_valid",+ return_value=True,+ ),+ patch(+ "mcp_server.tools.execution.ExecutionQuerySerializer.validated_data",+ {+ "execution_id": EXECUTION_ID,+ "include_metadata": False,+ "include_metrics": False,+ "include_extracted_text": False,+ },+ ),+ patch(+ "mcp_server.tools.execution.DeploymentHelper.get_execution_status"+ ) as fetch,+ ):+ scope_filter.return_value.only.return_value.first.return_value = None+ with self.assertRaises(MCPToolError):+ get_execution_status(self.context, execution_id=EXECUTION_ID)++ fetch.assert_not_called()++ def test_acknowledged_result_explains_itself(self) -> None:+ """The REST surface answers 406 here; an agent cannot act on a status+ code, so the tool must say the result is gone and not retryable.+ """+ response = ExecutionResponse(+ workflow_id="wf",+ execution_id=EXECUTION_ID,+ execution_status="COMPLETED",+ result=None,+ )+ response.result_acknowledged = True++ result, process = self._run(response)++ process.assert_not_called()+ assert "already acknowledged" in result["message"]++ def test_unknown_execution_id_is_an_agent_error(self) -> None:+ """A bad id comes back as an actionable tool error, not a server+ fault — the agent may simply have mistyped it.+ """+ with patch(+ "mcp_server.tools.execution.ExecutionQuerySerializer.is_valid",+ side_effect=ValidationError({"execution_id": ["Invalid execution_id."]}),+ ):+ with self.assertRaises(MCPToolError) as caught:+ get_execution_status(self.context, execution_id="nope")++ assert "execution_id" in str(caught.exception)diff --git a/backend/mcp_server/tools/__init__.py b/backend/mcp_server/tools/__init__.pynew file mode 100644index 000000000..e69de29bbdiff --git a/backend/mcp_server/tools/execution.py b/backend/mcp_server/tools/execution.pynew file mode 100644index 000000000..5ac468091--- /dev/null+++ b/backend/mcp_server/tools/execution.py@@ -0,0 +1,396 @@+"""Tools that run the connected API deployment and read back its results."""++import logging+import uuid+from typing import Any++from api_v2.constants import ApiExecution+from api_v2.deployment_helper import DeploymentHelper+from api_v2.dto import DeploymentExecutionDTO+from api_v2.exceptions import RateLimitExceeded+from api_v2.rate_limiter import APIDeploymentRateLimiter+from api_v2.serializers import ExecutionQuerySerializer, ExecutionRequestSerializer+from rest_framework.exceptions import ValidationError+from tags.serializers import TagParamsSerializer+from utils.enums import CeleryTaskState+from workflow_manager.workflow_v2.dto import ExecutionResponse+from workflow_manager.workflow_v2.exceptions import ExecutionDoesNotExistError+from workflow_manager.workflow_v2.models.execution import WorkflowExecution++from mcp_server.context import MCPContext+from mcp_server.exceptions import MCPToolError+from mcp_server.sanitize import (+ log_exception,+ redact_structure,+ valid_uuid,+)++logger = logging.getLogger(__name__)++# Mirrors the serializer limits into the JSON schema, so a well-behaved client+# is stopped before it spends a round trip on a request that cannot validate.+# Sourced from the serializers rather than copied, so the advertised schema+# tracks the limits actually enforced.+MAX_DOCUMENTS = ExecutionRequestSerializer.MAX_FILES_ALLOWED+MAX_TAGS = TagParamsSerializer.MAX_TAGS_ALLOWED++# An agent polling with getExecutionStatus does not benefit from holding the+# HTTP request open for the full 300s the REST API permits, and a long-held+# MCP call reads as a hang. Default to a short wait and let the agent poll.+DEFAULT_TIMEOUT_SEC = 30+++def _format_validation_error(error: ValidationError) -> str:+ """Flatten DRF validation detail into a sentence an agent can act on.++ ``error.detail`` stringifies with ``ErrorDetail(string=..., code=...)``+ wrappers and nested dict/list structure. That is noise the agent has to+ parse around before it can see which argument was wrong, so flatten it to+ ``field: message`` pairs.+ """++ def walk(node: Any, path: str = "") -> list[str]:+ if isinstance(node, dict):+ return [+ message+ for key, value in node.items()+ for message in walk(value, f"{path}.{key}" if path else str(key))+ ]+ if isinstance(node, list):+ return [message for item in node for message in walk(item, path)]+ return [f"{path}: {node}" if path else str(node)]++ return "; ".join(walk(error.detail))+++def extract_document_schema() -> dict[str, Any]:+ return {+ "type": "object",+ "properties": {+ "document_urls": {+ "type": "array",+ "items": {"type": "string"},+ "minItems": 1,+ "maxItems": MAX_DOCUMENTS,+ "description": (+ "HTTPS URLs of the documents to extract, which Unstract "+ "fetches server-side. The host must be an AWS S3 endpoint "+ "(*.amazonaws.com); links to any other host are rejected. "+ "For a private object, pass a pre-signed URL — that is the "+ "usual case. An object that is already publicly readable "+ "needs no signature."+ ),+ },+ "timeout": {+ "type": "integer",+ "minimum": -1,+ "maximum": ApiExecution.MAXIMUM_TIMEOUT_IN_SEC,+ "default": DEFAULT_TIMEOUT_SEC,+ "description": (+ "Seconds to wait for the extraction to finish before "+ "returning a pending result. Use -1 to return immediately "+ "and poll with getExecutionStatus."+ ),+ },+ "include_metadata": {+ "type": "boolean",+ "default": False,+ "description": "Include extraction metadata in the result.",+ },+ "include_metrics": {+ "type": "boolean",+ "default": False,+ "description": "Include token and cost metrics in the result.",+ },+ "include_extracted_text": {+ "type": "boolean",+ "default": False,+ "description": (+ "Include the full raw text extracted from each document, "+ "alongside the structured output. This can be very large."+ ),+ },+ "tags": {+ "type": "array",+ "items": {"type": "string"},+ "maxItems": MAX_TAGS,+ "description": (+ f"Tags to associate with this execution (at most "+ f"{MAX_TAGS}). Each tag must start with a letter and "+ "contain only letters, numbers, underscores and hyphens."+ ),+ },+ "llm_profile_id": {+ "type": "string",+ "description": (+ "UUID of an LLM profile to override the deployment's "+ "default. Omit to use the configured default."+ ),+ },+ },+ "required": ["document_urls"],+ }+++def extract_document(+ context: MCPContext,+ document_urls: list[str],+ timeout: int = DEFAULT_TIMEOUT_SEC,+ include_metadata: bool = False,+ include_metrics: bool = False,+ include_extracted_text: bool = False,+ tags: list[str] | None = None,+ llm_profile_id: str | None = None,+) -> dict[str, Any]:+ """Run the deployment's workflow over the supplied document URLs.++ Deliberately routed through ``ExecutionRequestSerializer`` rather than+ calling the execution helper directly: that serializer owns URL validation+ (HTTPS/endpoint restrictions) and the file-count cap, and reimplementing+ those here would let the MCP surface drift away from the REST surface it+ is meant to mirror.+ """+ if not context.api.is_active:+ raise MCPToolError(+ f"API deployment '{context.api.display_name}' is not active. "+ "Activate it in Unstract before extracting."+ )++ # `tags` is a comma-separated string on the REST surface; MCP clients+ # produce JSON arrays, so convert rather than exposing the wire format.+ payload: dict[str, Any] = {+ ApiExecution.PRESIGNED_URLS: document_urls,+ ApiExecution.TIMEOUT_FORM_DATA: timeout,+ ApiExecution.INCLUDE_METADATA: include_metadata,+ ApiExecution.INCLUDE_METRICS: include_metrics,+ ApiExecution.INCLUDE_EXTRACTED_TEXT: include_extracted_text,+ }+ if tags:+ payload[ApiExecution.TAGS] = ",".join(tags)+ if llm_profile_id:+ payload[ApiExecution.LLM_PROFILE_ID] = llm_profile_id++ serializer = ExecutionRequestSerializer(+ data=payload, context={"api": context.api, "api_key": context.api_key}+ )+ try:+ serializer.is_valid(raise_exception=True)+ except ValidationError as error:+ raise MCPToolError(+ f"Invalid arguments: {_format_validation_error(error)}"+ ) from error++ validated = serializer.validated_data+ presigned_urls = validated.get(ApiExecution.PRESIGNED_URLS, [])++ organization = context.api.organization+ execution_id = str(uuid.uuid4())++ # Take the rate-limit slot before downloading anything. The REST view+ # fetches first because it already holds the uploaded files in the request+ # body; here the fetch is ours to make, and doing it for a call we are+ # about to reject would pull every document over the network for nothing.+ can_proceed, limit_info = APIDeploymentRateLimiter.check_and_acquire(+ organization, execution_id+ )+ if not can_proceed:+ raise MCPToolError(+ "Rate limit exceeded for this organization "+ f"({limit_info['current_usage']}/{limit_info['limit']} "+ f"{limit_info['limit_type']}). Retry once running extractions finish."+ )++ try:+ file_objs: list[Any] = []+ DeploymentHelper.load_presigned_files(presigned_urls, file_objs)+ response = DeploymentHelper.execute_workflow(+ organization_name=context.org_name,+ api=context.api,+ file_objs=file_objs,+ timeout=validated.get(ApiExecution.TIMEOUT_FORM_DATA),+ include_metadata=validated.get(ApiExecution.INCLUDE_METADATA),+ include_metrics=validated.get(ApiExecution.INCLUDE_METRICS),+ include_extracted_text=validated.get(ApiExecution.INCLUDE_EXTRACTED_TEXT),+ use_file_history=validated.get(ApiExecution.USE_FILE_HISTORY, False),+ tag_names=validated.get(ApiExecution.TAGS, []),+ llm_profile_id=validated.get(ApiExecution.LLM_PROFILE_ID),+ execution_id=execution_id,+ )+ except RateLimitExceeded as error:+ # A limit raised from deeper in the stack is a different limit than the+ # one checked above, but it is still the agent's to wait out — so+ # convert it rather than letting it fall through to the generic+ # "failed unexpectedly" branch below.+ APIDeploymentRateLimiter.release_slot(organization, execution_id)+ raise MCPToolError(+ f"Rate limit exceeded: {error.detail}. Retry once running extractions finish."+ ) from error+ except Exception as error:+ APIDeploymentRateLimiter.release_slot(organization, execution_id)+ # Redacted, and without a traceback: this exception comes from the+ # connector and execution stack, so it can carry a connection string or+ # a signed URL — and a log line outlives the request and is shipped to+ # aggregation, reaching an audience the credential was never scoped to.+ log_exception(+ logger, f"MCP extractDocument failed for api '{context.api.api_name}'", error+ )+ # The upstream exception text is NOT returned to the caller. Redaction+ # strips the credential shapes it recognises, but an arbitrary exception+ # from a connector, a provider client or the execution stack also+ # carries internal hostnames, paths, row ids and stack detail that an+ # MCP client has no business seeing — and "recognised" is doing a lot of+ # work in that sentence. The full exception is in the log line above,+ # where an operator can correlate it by api name and timestamp.+ raise MCPToolError(+ "Extraction failed unexpectedly. The failure has been logged; "+ "contact your Unstract administrator if this persists."+ ) from error++ # Raw upstream response, not assembled field by field here, so it goes+ # through the same redaction net as the error messages.+ result = redact_structure(dict(response))+ # Surface the execution id unconditionally. On the pending path it is the+ # agent's only handle for polling, and execute_workflow does not guarantee+ # it is present in the response body.+ result.setdefault("execution_id", execution_id)+ return result+++def get_execution_status_schema() -> dict[str, Any]:+ return {+ "type": "object",+ "properties": {+ "execution_id": {+ "type": "string",+ "description": (+ "The execution_id returned by a previous extractDocument call."+ ),+ },+ "include_metadata": {+ "type": "boolean",+ "default": False,+ "description": "Include extraction metadata in the result.",+ },+ "include_metrics": {+ "type": "boolean",+ "default": False,+ "description": "Include token and cost metrics in the result.",+ },+ "include_extracted_text": {+ "type": "boolean",+ "default": False,+ "description": (+ "Include the full raw text extracted from each document. "+ "This can be very large."+ ),+ },+ },+ "required": ["execution_id"],+ }+++def get_execution_status(+ context: MCPContext,+ execution_id: str,+ include_metadata: bool = False,+ include_metrics: bool = False,+ include_extracted_text: bool = False,+) -> dict[str, Any]:+ """Return the status, and once ready the result, of an extraction."""+ serializer = ExecutionQuerySerializer(+ data={+ ApiExecution.EXECUTION_ID: execution_id,+ ApiExecution.INCLUDE_METADATA: include_metadata,+ ApiExecution.INCLUDE_METRICS: include_metrics,+ ApiExecution.INCLUDE_EXTRACTED_TEXT: include_extracted_text,+ }+ )+ try:+ serializer.is_valid(raise_exception=True)+ except ValidationError as error:+ raise MCPToolError(+ f"Invalid arguments: {_format_validation_error(error)}"+ ) from error+ except ExecutionDoesNotExistError as error:+ # `validate_execution_id` raises TWO different types: ValidationError+ # for a malformed UUID, and this — an APIException, not a+ # ValidationError — for a well-formed id that matches no execution.+ # Uncaught it escapes to the transport catch-all, so the agent gets+ # "Tool execution failed", and the deployment-scoped not-found message+ # further down is never reached because the serializer runs first.+ #+ # Answered with that same wording deliberately: whether an id belongs+ # to another deployment or to no execution at all is not something a+ # caller should be able to tell apart, and the next step is identical.+ raise MCPToolError(+ f"No execution with id '{execution_id}' for this deployment. "+ "Poll only ids returned by extractDocument."+ ) from error++ validated = serializer.validated_data+ validated_id = validated.get(ApiExecution.EXECUTION_ID)++ # Scope the id to this *deployment* before fetching anything.+ # DeploymentHelper.get_execution_status resolves a bare+ # WorkflowExecution.objects.get(id=...) with no tenant or deployment+ # filter, and this deployment runs one shared schema rather than a schema+ # per tenant. The API key narrows the caller to a deployment, not to an+ # execution id within it, so without this check a key for deployment A+ # could poll deployment B's execution — and, because a completed status+ # acknowledges the result and drops it from cache, destroy it for the+ # caller it belongs to.+ #+ # Filtered on `pipeline_id`, which holds the APIDeployment that started the+ # run (`DeploymentHelper` sets `pipeline_id = api.id`), NOT on+ # `workflow_id`. `APIDeployment.workflow` is a plain FK with no uniqueness+ # constraint — `related_name="apis"` is plural — so two deployments can+ # share one workflow, and a workflow-scoped filter would let either poll+ # the other's executions.+ #+ # The serializer validates shape, not that the id parses as a UUID; a+ # malformed one would raise ValidationError from the field below rather+ # than the actionable message this function already writes.+ valid_uuid(validated_id, "execution id", "Poll only ids returned by extractDocument.")+ execution = (+ WorkflowExecution.objects.filter(id=validated_id, pipeline_id=context.api.id)+ .only("id")+ .first()+ )+ if execution is None:+ raise MCPToolError(+ f"No execution with id '{validated_id}' for this deployment. "+ "Poll only ids returned by extractDocument."+ )++ response: ExecutionResponse = DeploymentHelper.get_execution_status(validated_id)++ if response.result_acknowledged:+ # The REST surface answers 406 here. An agent cannot act on a status+ # code, so say plainly that the result is gone and not retrievable.+ return {+ "execution_status": response.execution_status,+ "message": (+ "This result was already acknowledged and is no longer "+ "retrievable. Start a new extraction if you need it again."+ ),+ }++ if response.execution_status == CeleryTaskState.COMPLETED.value:+ deployment_execution_dto = DeploymentExecutionDTO(+ api=context.api, api_key=context.api_key+ )+ DeploymentHelper.process_completed_execution(+ response=response,+ deployment_execution_dto=deployment_execution_dto,+ include_metadata=validated.get(ApiExecution.INCLUDE_METADATA),+ include_metrics=validated.get(ApiExecution.INCLUDE_METRICS),+ include_extracted_text=validated.get(ApiExecution.INCLUDE_EXTRACTED_TEXT),+ )++ return {+ "execution_id": execution_id,+ "execution_status": response.execution_status,+ # Raw upstream result, not assembled field by field here: a failed+ # connector's error text can embed the connection string it tried.+ "result": redact_structure(response.result),+ }diff --git a/backend/mcp_server/tools/info.py b/backend/mcp_server/tools/info.pynew file mode 100644index 000000000..e5bbbc280--- /dev/null+++ b/backend/mcp_server/tools/info.py@@ -0,0 +1,86 @@+"""Read-only tools describing the connected API deployment."""++import logging+from typing import Any++from mcp_server.context import MCPContext++logger = logging.getLogger(__name__)+++def read_me_first(context: MCPContext) -> dict[str, Any]:+ """Return the orientation guide for a coding agent.++ Deliberately built from the live deployment rather than a static string —+ the agent should learn the name of the deployment it is actually talking+ to, not a generic description of Unstract.+ """+ return {+ "server": "Unstract MCP Server",+ "connected_deployment": {+ "name": context.api.display_name,+ "description": context.api.description or None,+ "is_active": context.api.is_active,+ },+ "what_this_does": (+ "Unstract runs LLM-driven extraction over unstructured documents "+ "(PDFs, scans, images) and returns structured JSON. This MCP "+ "server is scoped to a single API deployment, which encapsulates "+ "one extraction workflow — the prompts and output schema are fixed "+ "by that deployment, so you supply documents, not instructions."+ ),+ "tools": [+ {+ "name": "getApiInfo",+ "purpose": "Learn what the connected deployment extracts.",+ },+ {+ "name": "extractDocument",+ "purpose": "Run extraction over document URLs. Consumes quota.",+ },+ {+ "name": "getExecutionStatus",+ "purpose": "Poll for the result of a pending extraction.",+ },+ ],+ "recommended_workflow": [+ "1. Call getApiInfo to confirm the deployment is active and see "+ "what it is meant to process.",+ "2. Call extractDocument with the URL(s) of the document(s).",+ "3. If the response has execution_status COMPLETED, the result is "+ "already in the response — you are done.",+ "4. Otherwise poll getExecutionStatus with the returned "+ "execution_id until the status is COMPLETED or ERROR, pausing a "+ "few seconds between polls.",+ ],+ "notes": [+ "Documents are fetched server-side from the URLs you provide. The "+ "URL must be HTTPS on an AWS S3 host; any other host is rejected. "+ "Pass a pre-signed URL for a private object — the usual case — or "+ "a plain URL if the object is already publicly readable. Upload to "+ "S3 first if the document is not there yet.",+ "extractDocument consumes the organization's extraction quota. "+ "Never call it speculatively, and never retry a call that already "+ "returned an execution_id — poll instead.",+ ],+ }+++def get_api_info_schema() -> dict[str, Any]:+ return {"type": "object", "properties": {}, "required": []}+++def get_api_info(context: MCPContext) -> dict[str, Any]:+ """Return metadata about the connected API deployment."""+ api = context.api+ return {+ "id": str(api.id),+ "display_name": api.display_name,+ "api_name": api.api_name,+ "description": api.description or None,+ "is_active": api.is_active,+ "workflow": {+ "id": str(api.workflow_id),+ "name": api.workflow.workflow_name,+ },+ }diff --git a/backend/mcp_server/tools/observability.py b/backend/mcp_server/tools/observability.pynew file mode 100644index 000000000..9997c9b61--- /dev/null+++ b/backend/mcp_server/tools/observability.py@@ -0,0 +1,380 @@+"""Read-only tools for execution history, usage and workflow structure.++Every response here is built **field by field from named model attributes**.+That is a security boundary, not a style preference: several of this project's+serializers return decrypted credentials in their normal output — connector+metadata carries database passwords and object-store keys, adapter metadata+carries provider API keys — and a workflow endpoint references a connector+instance. Passing a model through ``serializer.data``, ``model_to_dict`` or a+``**`` splat would pipe those secrets into an LLM's context and defeat the+whole exclusion list in this app's README.++So: name every field you return. ``test_no_credentials_in_tool_output``+enforces this across the whole registry.+"""++import logging+from typing import Any++from usage_v2.models import Usage+from workflow_manager.endpoint_v2.models import WorkflowEndpoint+from workflow_manager.workflow_v2.models.execution import WorkflowExecution+from workflow_manager.workflow_v2.models.workflow import Workflow++from mcp_server.context import PlatformMCPContext+from mcp_server.exceptions import MCPToolError+from mcp_server.sanitize import (+ LIST_LIMIT,+ log_exception,+ redact_secrets,+ truncation_note,+ valid_uuid,+)++logger = logging.getLogger(__name__)++# Executions are the highest-cardinality thing here, and an agent asking "what+# happened" wants the recent tail rather than the archive.+EXECUTION_LIMIT = 25+LOG_LIMIT = 100+++def _org_workflow_ids(context: PlatformMCPContext) -> list[Any]:+ """Workflow ids the caller may see, for scoping execution queries.++ WorkflowExecution has no ``for_user`` manager, so visibility is derived+ from the workflows the caller can reach rather than assumed.+ """+ return list(Workflow.objects.for_user(context.user).values_list("id", flat=True))+++def list_executions_schema() -> dict[str, Any]:+ return {+ "type": "object",+ "properties": {+ "workflow_id": {+ "type": "string",+ "description": (+ "Optional UUID to show executions of one workflow only. "+ "Omit for the organization's most recent executions."+ ),+ },+ "status": {+ "type": "string",+ "description": (+ "Optional status filter, e.g. COMPLETED, ERROR, EXECUTING, PENDING."+ ),+ },+ },+ "required": [],+ }+++def list_executions(+ context: PlatformMCPContext,+ workflow_id: str | None = None,+ status: str | None = None,+) -> dict[str, Any]:+ """List recent workflow executions, newest first.++ This is the tool for "why did that fail" and "did it run" — the questions+ an agent actually has after triggering work.+ """+ visible = _org_workflow_ids(context)+ queryset = WorkflowExecution.objects.filter(workflow_id__in=visible)++ if workflow_id:+ if not any(str(wid) == str(workflow_id) for wid in visible):+ raise MCPToolError(+ f"No workflow with id '{workflow_id}' in organization "+ f"'{context.org_name}'. Call listWorkflows for valid ids."+ )+ queryset = queryset.filter(workflow_id=workflow_id)+ if status:+ queryset = queryset.filter(status=status.upper())++ queryset = queryset.order_by("-created_at")+ total = queryset.count()+ rows = queryset[:EXECUTION_LIMIT]++ return {+ "count": total,+ "showing": min(total, EXECUTION_LIMIT),+ "executions": [+ {+ "id": str(row.id),+ "workflow_id": str(row.workflow_id),+ "pipeline_id": str(row.pipeline_id) if row.pipeline_id else None,+ "status": row.status,+ "execution_mode": row.execution_mode,+ "total_files": row.total_files,+ "successful_files": row.successful_files,+ "failed_files": row.failed_files,+ # Truncated: an error message can be a full stack trace, and a+ # page of them would crowd out everything else.+ "error_message": redact_secrets((row.error_message or "")[:500]) or None,+ "attempts": row.attempts,+ "execution_time_seconds": row.execution_time,+ "created_at": row.created_at,+ }+ for row in rows+ ],+ **(+ {+ "truncated": True,+ "note": (+ f"Showing the {EXECUTION_LIMIT} most recent of {total}. "+ "Filter by workflow_id or status to narrow."+ ),+ }+ if total > EXECUTION_LIMIT+ else {}+ ),+ }+++def get_execution_detail_schema() -> dict[str, Any]:+ return {+ "type": "object",+ "properties": {+ "execution_id": {+ "type": "string",+ "description": "UUID of the execution, from listExecutions.",+ },+ },+ "required": ["execution_id"],+ }+++def get_execution_detail(+ context: PlatformMCPContext, execution_id: str+) -> dict[str, Any]:+ """Per-file detail for one execution — which files failed, and why."""+ valid_uuid(execution_id, "execution id", "Call listExecutions for valid ids.")++ visible = _org_workflow_ids(context)+ execution = WorkflowExecution.objects.filter(+ id=execution_id, workflow_id__in=visible+ ).first()+ if execution is None:+ raise MCPToolError(+ f"No execution with id '{execution_id}' in organization "+ f"'{context.org_name}'. Call listExecutions for valid ids."+ )++ files = []+ files_complete = True+ try:+ for fe in execution.file_executions.all()[:LOG_LIMIT]:+ files.append(+ {+ "id": str(fe.id),+ "file_name": fe.file_name,+ "status": fe.status,+ "error": redact_secrets((fe.execution_error or "")[:500]) or None,+ }+ )+ except Exception as error: # pragma: no cover - defensive+ # Partial results are kept — some detail beats none — but the caller+ # must be told, because the failure mode here is silent and wrong in a+ # specific way: an agent asking "which files failed" reads a short or+ # empty list as "none did", which is the opposite of what happened.+ files_complete = False+ log_exception(+ logger, f"Could not load file executions for '{execution_id}'", error+ )++ result = {+ "id": str(execution.id),+ "workflow_id": str(execution.workflow_id),+ "status": execution.status,+ "total_files": execution.total_files,+ "successful_files": execution.successful_files,+ "failed_files": execution.failed_files,+ "error_message": redact_secrets((execution.error_message or "")[:1000]) or None,+ "execution_time_seconds": execution.execution_time,+ "created_at": execution.created_at,+ "files": files,+ }+ if not files_complete:+ result["files_complete"] = False+ result["files_note"] = (+ "Per-file detail could not be fully loaded, so `files` is "+ "incomplete — do not read it as the full set of failures. "+ "`failed_files` above is authoritative for the count. Retry, or "+ "use the Unstract UI."+ )+ elif len(files) == LOG_LIMIT and execution.total_files > LOG_LIMIT:+ # The same wrong premise from a different cause: a capped list also+ # reads as complete unless it says otherwise.+ result["files_complete"] = False+ result["files_note"] = (+ f"Showing the first {LOG_LIMIT} of {execution.total_files} files."+ )+ return result+++def get_usage_summary_schema() -> dict[str, Any]:+ return {"type": "object", "properties": {}, "required": []}+++def get_usage_summary(context: PlatformMCPContext) -> dict[str, Any]:+ """Aggregate token and cost usage for the organization.++ Aggregated in the database rather than by pulling rows: a busy+ organization has far more usage records than belong in an agent's context,+ and the totals are what the question is actually about.+ """+ from django.db.models import Count, Sum++ totals = Usage.objects.aggregate(+ records=Count("id"),+ prompt_tokens=Sum("prompt_tokens"),+ completion_tokens=Sum("completion_tokens"),+ total_tokens=Sum("total_tokens"),+ embedding_tokens=Sum("embedding_tokens"),+ cost_in_dollars=Sum("cost_in_dollars"),+ )++ return {+ "organization": context.org_name,+ "records": totals["records"] or 0,+ "prompt_tokens": totals["prompt_tokens"] or 0,+ "completion_tokens": totals["completion_tokens"] or 0,+ "total_tokens": totals["total_tokens"] or 0,+ "embedding_tokens": totals["embedding_tokens"] or 0,+ "cost_in_dollars": float(totals["cost_in_dollars"] or 0),+ "note": (+ "Totals across all recorded usage for this organization. This is "+ "historical accounting, not a spending limit."+ ),+ }+++def list_tool_instances_schema() -> dict[str, Any]:+ return {+ "type": "object",+ "properties": {+ "workflow_id": {+ "type": "string",+ "description": "UUID of the workflow, from listWorkflows.",+ },+ },+ "required": ["workflow_id"],+ }+++def list_tool_instances(context: PlatformMCPContext, workflow_id: str) -> dict[str, Any]:+ """List the tool steps inside a workflow, in execution order.++ Tool settings are omitted deliberately: they are free-form JSON that can+ carry adapter ids and, in some tools, credential-shaped values. The step+ order and identity answer "what does this workflow do" without that risk.+ """+ from tool_instance_v2.models import ToolInstance++ valid_uuid(workflow_id, "workflow id", "Call listWorkflows for valid ids.")+ workflow = Workflow.objects.for_user(context.user).filter(id=workflow_id).first()+ if workflow is None:+ raise MCPToolError(+ f"No workflow with id '{workflow_id}' in organization "+ f"'{context.org_name}'. Call listWorkflows for valid ids."+ )++ rows = ToolInstance.objects.filter(workflow=workflow).order_by("step")++ return {+ "workflow_id": str(workflow.id),+ "workflow_name": workflow.workflow_name,+ "tool_instances": [+ {+ "id": str(row.id),+ "tool_id": row.tool_id,+ "step": row.step,+ }+ for row in rows+ ],+ "note": (+ "Tool settings are not exposed — they can reference adapters and "+ "credential-shaped values. Use the Unstract UI to inspect them."+ ),+ }+++def list_tags(context: PlatformMCPContext) -> dict[str, Any]:+ """List the organization's execution tags."""+ from tags.models import Tag++ queryset = Tag.objects.all().order_by("name")+ total = queryset.count()+ rows = queryset[:LIST_LIMIT]++ return {+ "count": total,+ "tags": [+ {"id": str(row.id), "name": row.name, "description": row.description or None}+ for row in rows+ ],+ # Without this a capped listing reads to the agent as the complete set,+ # which is exactly the wrong premise for it to build on.+ **truncation_note(len(rows), total),+ }+++def get_workflow_endpoints_schema() -> dict[str, Any]:+ return {+ "type": "object",+ "properties": {+ "workflow_id": {+ "type": "string",+ "description": "UUID of the workflow, from listWorkflows.",+ },+ },+ "required": ["workflow_id"],+ }+++def get_workflow_endpoints(+ context: PlatformMCPContext, workflow_id: str+) -> dict[str, Any]:+ """Describe a workflow's source and destination wiring.++ Returns the *shape* of the connection — endpoint type, connection type, and+ the connector's name — and deliberately never its configuration. A+ connector instance holds decrypted credentials in ``connector_metadata``,+ so only named, non-secret attributes are read here. Do not extend this to+ return ``configuration`` or ``connector_metadata``.+ """+ valid_uuid(workflow_id, "workflow id", "Call listWorkflows for valid ids.")+ workflow = Workflow.objects.for_user(context.user).filter(id=workflow_id).first()+ if workflow is None:+ raise MCPToolError(+ f"No workflow with id '{workflow_id}' in organization "+ f"'{context.org_name}'. Call listWorkflows for valid ids."+ )++ endpoints = []+ for endpoint in WorkflowEndpoint.objects.filter(workflow=workflow):+ connector = getattr(endpoint, "connector_instance", None)+ endpoints.append(+ {+ "id": str(endpoint.id),+ "endpoint_type": endpoint.endpoint_type,+ "connection_type": endpoint.connection_type,+ # Name and id only. The connector's metadata is credential+ # material and must not appear here.+ "connector_name": getattr(connector, "connector_name", None),+ "connector_id": str(connector.id) if connector else None,+ "is_configured": bool(endpoint.configuration),+ }+ )++ return {+ "workflow_id": str(workflow.id),+ "workflow_name": workflow.workflow_name,+ "endpoints": endpoints,+ "note": (+ "Connector credentials and endpoint configuration are deliberately "+ "not exposed. Use the Unstract UI to inspect or change them."+ ),+ }diff --git a/backend/mcp_server/tools/platform.py b/backend/mcp_server/tools/platform.pynew file mode 100644index 000000000..c5826f8bb--- /dev/null+++ b/backend/mcp_server/tools/platform.py@@ -0,0 +1,548 @@+"""Tools for the organization-scoped MCP server.++Every query goes through the model's own ``for_user`` manager rather than a raw+queryset, so the platform's sharing and visibility rules apply here exactly as+they do in the UI — this app does not get its own opinion about who may see or+touch what.++One caveat those managers impose, which the tool descriptions state plainly: a+platform key resolves to a service account, and ``for_user`` short-circuits to+"everything in the organization" for service accounts. These tools therefore+reach all org resources regardless of the role the key was created with — for+the write tools below that means they can modify anything in the organization,+not merely what one user could.++Write tools are authorized per tool by the view, using the platform key's+existing permission tier — see ``MCPTool.required_method``.+"""++import logging+from typing import Any++from api_v2.models import APIDeployment+from pipeline_v2.models import Pipeline+from prompt_studio.prompt_studio_core_v2.models import CustomTool+from workflow_manager.workflow_v2.models.workflow import Workflow++from mcp_server import spend_guard+from mcp_server.context import PlatformMCPContext+from mcp_server.exceptions import MCPToolError+from mcp_server.sanitize import (+ LIST_LIMIT,+ redact_structure,+ truncation_note,+ valid_uuid,+)++logger = logging.getLogger(__name__)+++def no_args_schema() -> dict[str, Any]:+ return {"type": "object", "properties": {}, "required": []}+++# Every write tool repeats this, because the tool description is the prompt the+# agent reads before deciding to act — and the blast radius of a write is wider+# than the caveat's usual home in whoami.+_ORG_WIDE_WARNING = (+ "This credential is a service account and can modify ANY resource in the "+ "organization, not only those shared with a particular user. Confirm you "+ "have the right target before calling."+)+++def platform_read_me_first(context: PlatformMCPContext) -> dict[str, Any]:+ """Return the orientation guide for an agent on the platform server."""+ return {+ "server": "Unstract Platform MCP Server",+ "organization": context.org_name,+ "what_this_does": (+ "Unstract runs LLM-driven extraction over unstructured documents "+ "(PDFs, scans, images) and returns structured JSON. This server is "+ "scoped to a whole organization: it discovers what exists and can "+ "change the running state of those resources."+ ),+ "discovery_tools": [+ {"name": "whoami", "purpose": "See this credential's org, tier and budget."},+ {+ "name": "listApiDeployments",+ "purpose": "Find deployed extraction endpoints.",+ },+ {"name": "listWorkflows", "purpose": "Find the workflows behind them."},+ {"name": "listPipelines", "purpose": "Find ETL and task pipelines."},+ {+ "name": "listPromptStudioProjects",+ "purpose": "Find where extraction prompts are authored.",+ },+ {+ "name": "listPromptStudioDocuments",+ "purpose": (+ "List a project's documents. The only source of the "+ "document_id every billable tool below requires."+ ),+ },+ {+ "name": "listPrompts",+ "purpose": (+ "List a project's prompts. The only source of the "+ "prompt_id fetchResponse and bulkFetchResponse require."+ ),+ },+ {+ "name": "getWorkflowEndpoints",+ "purpose": "See how a workflow is wired (shape only, no config).",+ },+ {+ "name": "listToolInstances",+ "purpose": "See the tool steps inside a workflow.",+ },+ {"name": "listTags", "purpose": "List execution tags."},+ ],+ "observability_tools": [+ {+ "name": "listExecutions",+ "purpose": "Recent runs and their status. Start here to debug.",+ },+ {+ "name": "getExecutionDetail",+ "purpose": "Per-file results and errors for one run.",+ },+ {+ "name": "getUsageSummary",+ "purpose": "Tokens and cost recorded so far.",+ },+ {+ "name": "getExecutionStatus",+ "purpose": (+ "Poll an extraction started by extractDocument. Free to "+ "call, unlike the extraction itself."+ ),+ },+ ],+ "state_change_tools": [+ {+ "name": "setApiDeploymentActive",+ "purpose": "Take a deployment offline, or bring it back.",+ },+ {+ "name": "setPipelineActive",+ "purpose": "Pause or resume a pipeline's schedule.",+ },+ ],+ "billable_tools": [+ {+ "name": "executePipeline",+ "purpose": "Trigger a real pipeline run.",+ },+ {+ "name": "indexDocument",+ "purpose": "Embed a document so prompts can run against it.",+ },+ {+ "name": "fetchResponse",+ "purpose": "Run one prompt against an indexed document.",+ },+ {+ "name": "bulkFetchResponse",+ "purpose": "Run several prompts in one pass. Prefer over looping.",+ },+ {+ "name": "singlePassExtraction",+ "purpose": "Run a project's whole prompt set. Most expensive.",+ },+ {+ "name": "extractDocument",+ "purpose": (+ "Run a deployment's extraction workflow over documents. "+ "Needs an api_name from listApiDeployments."+ ),+ },+ ],+ "before_writing": (+ "State-change tools take effect immediately for the whole "+ "organization — there is no staging or dry-run mode. Confirm the "+ "target with a discovery tool first."+ ),+ "before_spending": (+ "Billable tools cost real money each call: they drive LLM "+ "inference, embedding and vector-store writes. They are budgeted "+ "per organization — call whoami to see how much of that budget "+ "remains. Index a document once and then run prompts against it; "+ "prefer bulkFetchResponse over repeated fetchResponse calls; and "+ "never call a billable tool to 'test' whether something works."+ ),+ "not_available": (+ "Creating or rotating API keys, and deleting workflows, "+ "deployments or projects, are deliberately not exposed here."+ ),+ "to_run_an_extraction": (+ "Call listApiDeployments for an api_name, then extractDocument "+ "with that name and your document URLs. If the response is not "+ "already COMPLETED, poll getExecutionStatus with the returned "+ "execution_id.\n\n"+ "The same extraction is also available on a deployment-scoped "+ "server at /deployment/api/<org>/<api_name>/mcp, which takes that "+ "one deployment's own API key and can reach nothing else. Both "+ "run the identical workflow — pick this server for one credential "+ "across the organization, or that one for a narrower blast radius."+ ),+ "scope_warning": (+ "This credential is a service account, which reaches ALL resources "+ "in the organization regardless of the role it was created with. "+ "Do not treat these listings as one person's view, and remember "+ "the write tools have the same reach."+ ),+ }+++def whoami(context: PlatformMCPContext) -> dict[str, Any]:+ """Describe the calling credential, so an agent can reason about scope."""+ from platform_api.models import ApiKeyPermission++ key = context.platform_key+ try:+ permission = ApiKeyPermission(key.permission)+ can_write = permission.allows("POST")+ can_delete = permission.allows("DELETE")+ except ValueError:+ can_write = can_delete = False++ budget = spend_guard.peek(context.org_name)++ return {+ "organization": context.org_name,+ "key_name": key.name,+ "permission_tier": key.permission,+ "minimum_tier": "read_write",+ "tier_note": (+ "This server requires a `read_write` key at minimum — including "+ "for the read-only tools. Every JSON-RPC message is an HTTP POST "+ "and the middleware gates tiers on HTTP method, so a `read` key is "+ "refused before any tool runs. If you are reading this, your key "+ "already meets that bar."+ ),+ "is_service_account": bool(getattr(context.user, "is_service_account", False)),+ "can_use_write_tools": can_write,+ "can_use_destructive_tools": can_delete,+ "billable_budget": {+ "used": budget.used,+ "limit": budget.limit,+ "remaining": max(0, budget.limit - budget.used),+ "window_minutes": round(budget.window_seconds / 60),+ "note": (+ "Counts calls to billable tools, not tokens. Pace yourself "+ "against `remaining`; when it reaches zero, billable tools are "+ "refused until the window rolls over."+ ),+ },+ "visibility": (+ "All resources in the organization. Service-account credentials "+ "bypass per-user sharing filters, so this key can read AND modify "+ "any resource in the organization."+ ),+ }+++def list_api_deployments(context: PlatformMCPContext) -> dict[str, Any]:+ """List the organization's API deployments."""+ queryset = APIDeployment.objects.for_user(context.user).order_by("display_name")+ total = queryset.count()+ rows = queryset[:LIST_LIMIT]++ return {+ "count": total,+ "api_deployments": [+ {+ "id": str(row.id),+ "display_name": row.display_name,+ # api_name is what an agent needs to open a deployment-scoped+ # MCP session, so it is surfaced rather than left to the UI.+ "api_name": row.api_name,+ "description": row.description or None,+ "is_active": row.is_active,+ "workflow_id": str(row.workflow_id),+ }+ for row in rows+ ],+ **truncation_note(len(rows), total),+ }+++def list_workflows(context: PlatformMCPContext) -> dict[str, Any]:+ """List the organization's workflows."""+ queryset = Workflow.objects.for_user(context.user).order_by("workflow_name")+ total = queryset.count()+ rows = queryset[:LIST_LIMIT]++ return {+ "count": total,+ "workflows": [+ {+ "id": str(row.id),+ "workflow_name": row.workflow_name,+ "description": row.description or None,+ "is_active": row.is_active,+ }+ for row in rows+ ],+ **truncation_note(len(rows), total),+ }+++def list_prompt_studio_projects(context: PlatformMCPContext) -> dict[str, Any]:+ """List the organization's Prompt Studio projects."""+ queryset = CustomTool.objects.for_user(context.user).order_by("tool_name")+ total = queryset.count()+ rows = queryset[:LIST_LIMIT]++ return {+ "count": total,+ "prompt_studio_projects": [+ {+ "id": str(row.tool_id),+ "tool_name": row.tool_name,+ "description": row.description or None,+ }+ for row in rows+ ],+ **truncation_note(len(rows), total),+ }+++def list_pipelines(context: PlatformMCPContext) -> dict[str, Any]:+ """List the organization's ETL and task pipelines."""+ queryset = Pipeline.objects.for_user(context.user).order_by("pipeline_name")+ total = queryset.count()+ rows = queryset[:LIST_LIMIT]++ return {+ "count": total,+ "pipelines": [+ {+ "id": str(row.id),+ "pipeline_name": row.pipeline_name,+ "pipeline_type": row.pipeline_type,+ "active": row.active,+ "last_run_status": row.last_run_status,+ }+ for row in rows+ ],+ **truncation_note(len(rows), total),+ }+++# ============================ write tools ============================+#+# Authorization for these is not decided here. Each is registered with a+# `required_method`, and the view checks it against the platform key's tier+# using ``ApiKeyPermission.allows`` — the same method-based semantics the auth+# middleware already enforces for REST. Destructive operations therefore fall+# out as full_access-only without a second, parallel permission scheme.+++def set_api_deployment_active_schema() -> dict[str, Any]:+ return {+ "type": "object",+ "properties": {+ "api_name": {+ "type": "string",+ "description": (+ "The api_name of the deployment, as returned by listApiDeployments."+ ),+ },+ "is_active": {+ "type": "boolean",+ "description": (+ "True to activate the deployment so it accepts extraction "+ "requests, False to take it offline."+ ),+ },+ },+ "required": ["api_name", "is_active"],+ }+++def set_api_deployment_active(+ context: PlatformMCPContext, api_name: str, is_active: bool+) -> dict[str, Any]:+ """Activate or deactivate an API deployment.++ Reversible by construction — the inverse call restores the previous state —+ which is why this is the write operation exposed rather than anything that+ destroys configuration.+ """+ deployment = (+ APIDeployment.objects.for_user(context.user).filter(api_name=api_name).first()+ )+ if deployment is None:+ raise MCPToolError(+ f"No API deployment named '{api_name}' in organization "+ f"'{context.org_name}'. Call listApiDeployments to see valid names."+ )++ was_active = deployment.is_active+ if was_active == is_active:+ return {+ "api_name": api_name,+ "is_active": is_active,+ "changed": False,+ "message": f"Deployment '{api_name}' was already "+ f"{'active' if is_active else 'inactive'}.",+ }++ deployment.is_active = is_active+ deployment.save(update_fields=["is_active"])+ logger.info(+ f"Platform MCP set deployment '{api_name}' active={is_active} "+ f"(org '{context.org_name}', key '{context.platform_key.name}')"+ )++ return {+ "api_name": api_name,+ "is_active": is_active,+ "changed": True,+ "previous_state": was_active,+ }+++def execute_pipeline_schema() -> dict[str, Any]:+ return {+ "type": "object",+ "properties": {+ "pipeline_id": {+ "type": "string",+ "description": (+ "UUID of the pipeline to run, as returned by listPipelines."+ ),+ },+ },+ "required": ["pipeline_id"],+ }+++def _resolve_pipeline(context: PlatformMCPContext, pipeline_id: str) -> Pipeline:+ """Find a pipeline the caller may reach, or refuse."""+ valid_uuid(pipeline_id, "pipeline id", "Call listPipelines to see valid ids.")+ pipeline = Pipeline.objects.for_user(context.user).filter(id=pipeline_id).first()+ if pipeline is None:+ raise MCPToolError(+ f"No pipeline with id '{pipeline_id}' in organization "+ f"'{context.org_name}'. Call listPipelines to see valid ids."+ )+ return pipeline+++def preflight_pipeline(+ context: PlatformMCPContext, pipeline_id: str, **_ignored: Any+) -> None:+ """Resolve the pipeline before any budget is claimed.++ ``execute_pipeline`` resolves it again inside the handler, but that runs+ after the budget is consumed and the budget is never refunded — so a stale+ id would cost a billable slot for a run that never started.++ Liveness is checked here for the same reason, and it is the sharper case:+ ``setPipelineActive`` is registered alongside this tool, so an agent can+ pause a pipeline and then spend the whole organization's billable window+ retrying against it, with no run ever starting. The row is already fetched.+ """+ pipeline = _resolve_pipeline(context, pipeline_id)+ if not pipeline.active:+ raise MCPToolError(+ f"Pipeline '{pipeline.pipeline_name}' is paused. Resume it with "+ "setPipelineActive before triggering a run."+ )+++def execute_pipeline(context: PlatformMCPContext, pipeline_id: str) -> dict[str, Any]:+ """Trigger a run of an ETL or task pipeline.++ Delegates to ``PipelineManager``, which is what the REST endpoint calls, so+ scheduling, execution records and notifications behave identically to a run+ started from the UI.+ """+ from pipeline_v2.manager import PipelineManager++ pipeline = _resolve_pipeline(context, pipeline_id)++ if context.request is None:+ # The manager mutates request.data on the way to the workflow viewset,+ # so there is no meaningful fallback — fail loudly rather than+ # fabricate a request and produce a half-configured execution.+ raise MCPToolError(+ "Pipeline execution is unavailable in this context. "+ "Contact your Unstract administrator."+ )++ logger.info(+ f"Platform MCP executing pipeline '{pipeline_id}' "+ f"(org '{context.org_name}', key '{context.platform_key.name}')"+ )+ response = PipelineManager.execute_pipeline(+ request=context.request, pipeline_id=str(pipeline.id)+ )++ return {+ "pipeline_id": str(pipeline.id),+ "pipeline_name": pipeline.pipeline_name,+ "status": getattr(response, "status_code", None),+ # Raw manager response rather than named fields, so it gets the same+ # redaction net the error messages get.+ "result": redact_structure(getattr(response, "data", None)),+ }+++def set_pipeline_active_schema() -> dict[str, Any]:+ return {+ "type": "object",+ "properties": {+ "pipeline_id": {+ "type": "string",+ "description": ("UUID of the pipeline, as returned by listPipelines."),+ },+ "active": {+ "type": "boolean",+ "description": (+ "True to enable the pipeline's schedule, False to pause it."+ ),+ },+ },+ "required": ["pipeline_id", "active"],+ }+++def set_pipeline_active(+ context: PlatformMCPContext, pipeline_id: str, active: bool+) -> dict[str, Any]:+ """Enable or pause a pipeline's schedule."""+ valid_uuid(pipeline_id, "pipeline id", "Call listPipelines to see valid ids.")+ pipeline = Pipeline.objects.for_user(context.user).filter(id=pipeline_id).first()+ if pipeline is None:+ raise MCPToolError(+ f"No pipeline with id '{pipeline_id}' in organization "+ f"'{context.org_name}'. Call listPipelines to see valid ids."+ )++ was_active = pipeline.active+ if was_active == active:+ return {+ "pipeline_id": str(pipeline.id),+ "active": active,+ "changed": False,+ "message": f"Pipeline was already {'active' if active else 'paused'}.",+ }++ pipeline.active = active+ pipeline.save(update_fields=["active"])+ logger.info(+ f"Platform MCP set pipeline '{pipeline_id}' active={active} "+ f"(org '{context.org_name}', key '{context.platform_key.name}')"+ )++ return {+ "pipeline_id": str(pipeline.id),+ "pipeline_name": pipeline.pipeline_name,+ "active": active,+ "changed": True,+ "previous_state": was_active,+ }diff --git a/backend/mcp_server/tools/platform_execution.py b/backend/mcp_server/tools/platform_execution.pynew file mode 100644index 000000000..279cef05f--- /dev/null+++ b/backend/mcp_server/tools/platform_execution.py@@ -0,0 +1,270 @@+"""Extraction from the organization-scoped server.++The deployment server and this one now both run extractions. They are not+redundant: they are two blast radii for the same operation, and the caller+picks. A deployment key reaches exactly one deployment, so a leaked one costs+one workflow; the platform key reaches every deployment in the organization,+and buys the caller not having to obtain a second credential to extract at all.++The work itself is delegated to ``tools.execution`` rather than reimplemented,+so both servers share one code path through validation, rate limiting and the+execution helper. What differs is only how the target deployment is found:+there it arrives with the credential, here it is looked up by ``api_name``.++Two deliberate divergences from the deployment server:++* **No ``llm_profile_id``.** On the deployment server that argument is checked+ against the API key's owner (``ExecutionRequestSerializer.validate_llm_profile_id``).+ A platform caller is a service account with org-wide reach, so there is no+ meaningful owner to check it against, and borrowing some deployment key to+ satisfy the check would be asserting a principal this caller is not. The+ argument is therefore not offered here. This also keeps the serializer from+ ever needing the ``api_key`` that this context does not have — omitting the+ field means DRF never runs that validator.+* **Execution status is scoped to the organization.** See+ ``get_platform_execution_status``.+"""++import logging+from typing import Any++from api_v2.models import APIDeployment++from mcp_server.context import MCPContext, PlatformMCPContext+from mcp_server.exceptions import MCPToolError+from mcp_server.sanitize import valid_uuid+from mcp_server.tools.execution import (+ DEFAULT_TIMEOUT_SEC,+ extract_document,+ extract_document_schema,+)++logger = logging.getLogger(__name__)+++def _resolve_deployment(context: PlatformMCPContext, api_name: str) -> APIDeployment:+ """Find an API deployment the caller may reach, or refuse.++ ``for_user`` is what keeps this inside the caller's organization. The+ platform key authenticates as a service account, for which that manager+ returns every deployment in the org — which is the intended reach here —+ but it is still the boundary that stops another organization's deployment+ being named.+ """+ deployment = (+ APIDeployment.objects.for_user(context.user).filter(api_name=api_name).first()+ )+ if deployment is None:+ raise MCPToolError(+ f"No API deployment named '{api_name}' in organization "+ f"'{context.org_name}'. Call listApiDeployments for valid names."+ )+ return deployment+++def _as_deployment_context(+ context: PlatformMCPContext, deployment: APIDeployment+) -> MCPContext:+ """Adapt a platform context to the one ``tools.execution`` expects.++ ``api_key`` is deliberately ``None``. It is read in exactly one place —+ validating ``llm_profile_id`` ownership — and this tool does not accept+ that argument, so the serializer never invokes that validator. Passing a+ real deployment key instead would make the ownership check pass against+ whichever principal happens to own it, which is worse than not offering+ the feature.+ """+ return MCPContext(api=deployment, api_key=None, org_name=context.org_name)+++def platform_extract_document_schema() -> dict[str, Any]:+ """The deployment schema plus ``api_name``, minus ``llm_profile_id``.++ Derived from the deployment schema rather than restated, so the two cannot+ drift on file caps, tag rules or the timeout range.+ """+ schema = extract_document_schema()+ properties = dict(schema["properties"])+ # See the module docstring: not offerable without a deployment key to check+ # ownership against.+ properties.pop("llm_profile_id", None)++ return {+ "type": "object",+ "properties": {+ "api_name": {+ "type": "string",+ "description": (+ "Name of the API deployment to run, from "+ "listApiDeployments. This fixes the prompts and output "+ "schema — you supply documents, not instructions."+ ),+ },+ **properties,+ },+ "required": ["api_name", *schema["required"]],+ }+++def preflight_extract_document(+ context: PlatformMCPContext, api_name: str, **_ignored: Any+) -> None:+ """Resolve the named deployment before any budget is claimed.++ ``api_name`` is the one argument here an agent can plausibly get wrong: it+ is a human-chosen string rather than an id copied from a listing, so a+ guess or a typo is likely. The budget is never refunded, so without this+ check each wrong guess would cost a billable slot for a call that reached+ no LLM at all — and an agent guessing repeatedly could exhaust the window+ without ever running an extraction.++ Extra keyword arguments are accepted and ignored so that adding an argument+ to the tool does not require touching this function; the handler is what+ validates the rest.++ Liveness is checked here and not only in the handler. An inactive+ deployment is refused downstream *after* the budget has been claimed, and+ the budget is never refunded — so an agent retrying against a deployment it+ just deactivated via ``setApiDeploymentActive`` (registered alongside this+ tool) could exhaust ``MCP_BILLABLE_CALL_LIMIT`` and lock the organization+ out of every billable tool, without a single call reaching an LLM. The row+ is already fetched, so this meets the "cheap and certain" bar the+ ``preflight`` field documents.+ """+ deployment = _resolve_deployment(context, api_name)+ if not deployment.is_active:+ raise MCPToolError(+ f"API deployment '{deployment.display_name}' is not active. "+ "Activate it in Unstract before extracting."+ )+++def platform_extract_document(+ context: PlatformMCPContext,+ api_name: str,+ document_urls: list[str],+ timeout: int = DEFAULT_TIMEOUT_SEC,+ include_metadata: bool = False,+ include_metrics: bool = False,+ include_extracted_text: bool = False,+ tags: list[str] | None = None,+) -> dict[str, Any]:+ """Run a named deployment's workflow over the supplied document URLs."""+ deployment = _resolve_deployment(context, api_name)+ logger.info(+ f"MCP platform extract_document api='{api_name}' "+ f"(org '{context.org_name}', key '{context.platform_key.name}')"+ )++ result = extract_document(+ _as_deployment_context(context, deployment),+ document_urls=document_urls,+ timeout=timeout,+ include_metadata=include_metadata,+ include_metrics=include_metrics,+ include_extracted_text=include_extracted_text,+ tags=tags,+ )+ # The agent named a deployment rather than being scoped to one, so echo+ # back which one actually ran.+ return {"api_name": api_name, **result}+++def platform_execution_status_schema() -> dict[str, Any]:+ return {+ "type": "object",+ "properties": {+ "execution_id": {+ "type": "string",+ "description": (+ "The execution_id returned by a previous extractDocument call."+ ),+ },+ "include_metadata": {+ "type": "boolean",+ "default": False,+ "description": "Include extraction metadata in the result.",+ },+ "include_metrics": {+ "type": "boolean",+ "default": False,+ "description": "Include token and cost metrics in the result.",+ },+ "include_extracted_text": {+ "type": "boolean",+ "default": False,+ "description": (+ "Include the full raw text extracted from each document. "+ "This can be very large."+ ),+ },+ },+ "required": ["execution_id"],+ }+++def get_platform_execution_status(+ context: PlatformMCPContext,+ execution_id: str,+ include_metadata: bool = False,+ include_metrics: bool = False,+ include_extracted_text: bool = False,+) -> dict[str, Any]:+ """Return the status, and once ready the result, of an extraction.++ The execution id is checked against the organization *before* the status+ is fetched. ``DeploymentHelper.get_execution_status`` resolves a bare+ ``WorkflowExecution.objects.get(id=...)`` with no tenant filter, and this+ deployment runs a single shared schema rather than one per tenant, so+ without this check a caller could poll an execution id belonging to+ another organization.++ ``get_execution_status`` makes the equivalent check on the deployment+ server, anchored to that deployment's single workflow. The two differ only+ in width: a platform key reaches every workflow in the org, so the filter+ here is ``workflow_id__in=visible`` rather than one id.+ """+ from workflow_manager.workflow_v2.models.execution import WorkflowExecution++ from mcp_server.tools.execution import get_execution_status+ from mcp_server.tools.observability import _org_workflow_ids++ valid_uuid(execution_id, "execution id", "Poll only ids returned by extractDocument.")++ visible = _org_workflow_ids(context)+ execution = (+ WorkflowExecution.objects.filter(id=execution_id, workflow_id__in=visible)+ .only("id", "pipeline_id")+ .first()+ )+ if execution is None:+ raise MCPToolError(+ f"No execution with id '{execution_id}' in organization "+ f"'{context.org_name}'. Poll only ids returned by extractDocument, "+ "or call listExecutions."+ )++ # Resolve by `pipeline_id` — the deployment that actually started this run —+ # not by the workflow. `APIDeployment.workflow` has no uniqueness+ # constraint, so a workflow-based lookup returns an arbitrary one of the+ # deployments sharing it, and `get_execution_status` then scopes its own+ # check to *that* deployment and rejects an execution the caller legitimately+ # owns. Going straight to the recorded deployment also drops a hop.+ deployment = (+ APIDeployment.objects.for_user(context.user)+ .filter(id=execution.pipeline_id)+ .first()+ )+ if deployment is None:+ raise MCPToolError(+ f"Execution '{execution_id}' did not come from an API deployment. "+ "Use getExecutionDetail for pipeline and ETL runs."+ )++ return get_execution_status(+ _as_deployment_context(context, deployment),+ execution_id=execution_id,+ include_metadata=include_metadata,+ include_metrics=include_metrics,+ include_extracted_text=include_extracted_text,+ )diff --git a/backend/mcp_server/tools/prompt_studio.py b/backend/mcp_server/tools/prompt_studio.pynew file mode 100644index 000000000..917493c7a--- /dev/null+++ b/backend/mcp_server/tools/prompt_studio.py@@ -0,0 +1,488 @@+"""Billable Prompt Studio tools for the organization-scoped MCP server.++These drive real LLM inference, embedding and vector-store writes, so each is+registered ``billable=True`` and budgeted by ``mcp_server.spend_guard``.++Every tool here delegates to ``PromptStudioCoreView`` rather than+reimplementing its logic. Those actions carry a great deal that is easy to get+subtly wrong — run-id generation, profile resolution, lookup gating, indexing+race avoidance, Celery dispatch — and a parallel implementation would drift+from the UI's behaviour on the first change to either. Dispatching the real+view keeps one code path, and keeps its permission and sharing checks.+"""++import logging+from typing import Any++from django.core.exceptions import ObjectDoesNotExist+from django.http import Http404+from prompt_studio.prompt_studio_core_v2.models import CustomTool+from prompt_studio.prompt_studio_core_v2.views import PromptStudioCoreView+from rest_framework.exceptions import APIException+from rest_framework.response import Response++from mcp_server.context import PlatformMCPContext+from mcp_server.exceptions import MCPToolError+from mcp_server.sanitize import log_exception, redact_structure, valid_uuid++logger = logging.getLogger(__name__)++# Shared parameter descriptions. Repeated verbatim across tool schemas, so+# defined once — an edit here must not silently apply to only some tools.+_PROJECT_ID_DESC = "UUID of the Prompt Studio project."+_PROJECT_ID_DESC_WITH_HINT = (+ "UUID of the Prompt Studio project, from listPromptStudioProjects."+)++# Payload fields that reach a UUID-typed lookup in the delegated view, mapped to+# the label and next-step hint their refusal should carry. `id` is the view's+# name for what the tool calls `prompt_id`.+_UUID_FIELDS = {+ "document_id": ("document id", "Call listPromptStudioDocuments for valid ids."),+ "id": ("prompt id", "Call listPrompts for valid ids."),+ "profile_manager": ("profile id", "Omit it to use the project default."),+}+++def _resolve_project(context: PlatformMCPContext, project_id: str) -> CustomTool:+ """Find a Prompt Studio project the caller may reach, or refuse.++ Resolving through ``for_user`` is what keeps a key inside its own+ organization — the view's own ``get_object`` would apply the same rules,+ but failing here produces a message the agent can act on rather than a+ 404 surfaced as an unexpected error.+ """+ valid_uuid(project_id, "project id", "Call listPromptStudioProjects for valid ids.")+ project = CustomTool.objects.for_user(context.user).filter(tool_id=project_id).first()+ if project is None:+ raise MCPToolError(+ f"No Prompt Studio project with id '{project_id}' in organization "+ f"'{context.org_name}'. Call listPromptStudioProjects for valid ids."+ )+ return project+++def _dispatch(context: PlatformMCPContext, action: str, project_id: str, payload: dict):+ """Invoke a PromptStudioCoreView action with the MCP request.++ The underlying view reads its arguments from ``request.data`` and resolves+ the organization from the request, both of which the platform MCP request+ already carries — the middleware set the org, and this server's context+ holds the real DRF request.+ """+ if context.request is None:+ raise MCPToolError(+ "Prompt Studio operations are unavailable in this context. "+ "Contact your Unstract administrator."+ )++ # The sibling ids travel to the same UUID-typed lookups the project id does+ # — `DocumentManager.objects.get(pk=...)` in the view — where a malformed+ # one raises Django ValidationError. That is neither APIException nor+ # Http404, so it escapes the arm below and reaches the client as an opaque+ # failure, with the non-refundable budget already claimed. Guarding here+ # covers every tool rather than repeating it at six call sites.+ for field, (label, hint) in _UUID_FIELDS.items():+ if payload.get(field):+ valid_uuid(payload[field], label, hint)+ for prompt_id in payload.get("prompt_ids") or []:+ valid_uuid(prompt_id, "prompt id", "Call listPrompts for valid ids.")++ request = context.request+ # The view reads its inputs from request.data. Replacing it wholesale keeps+ # the JSON-RPC envelope (which carries the MCP method, not the tool's+ # arguments) from reaching the view.+ original_data = request.data+ try:+ request._full_data = payload+ # Do NOT route through as_view(): it calls initialize_request(), which+ # builds a *new* DRF Request with no _full_data, so .data re-parses+ # request.body — the JSON-RPC envelope — and the action never sees+ # document_id/prompt_id. Call the action on a bare instance holding this+ # request instead, the same pattern PipelineManager.execute_pipeline+ # uses for executePipeline.+ view_instance = PromptStudioCoreView()+ view_instance.request = request+ view_instance.args = ()+ view_instance.kwargs = {"pk": project_id}+ # get_object()/get_serializer() read self.request and self.kwargs["pk"];+ # format_kwarg is what DRF's content negotiation path expects to exist.+ view_instance.format_kwarg = None+ # `action` is normally set by ViewSetMixin.initialize_request, which+ # only as_view() invokes. Without it get_queryset() raises+ # AttributeError on `self.action == "list"` — every one of these tools+ # calls get_object(), so omitting it fails all four, after the+ # non-refundable budget has already been claimed.+ view_instance.action = action+ response = getattr(view_instance, action)(request, pk=project_id)+ except (APIException, Http404, ObjectDoesNotExist) as error:+ # dispatch() would have run these through handle_exception() and+ # returned a non-2xx Response. Bypassing it means get_object()'s Http404+ # and check_object_permissions' PermissionDenied would escape as raw+ # exceptions instead, which _result documents as being returned as data.+ #+ # ObjectDoesNotExist is here because the actions do not all resolve+ # their arguments the same way: `bulk_fetch_response` catches+ # `DocumentManager.DoesNotExist` and answers 404, but `index_document`,+ # `fetch_response` and `single_pass_extraction` call a bare+ # `.objects.get(pk=...)` (views.py:476, 581, 833). That raises+ # ObjectDoesNotExist, which is neither APIException nor Http404 — so a+ # well-formed but unknown document_id escaped as an opaque failure,+ # after the non-refundable budget was already claimed.+ response = _exception_response(error)+ finally:+ request._full_data = original_data++ return response+++def _exception_response(error: Exception):+ """Render a DRF exception the way ``dispatch`` would have.++ Not a full ``handle_exception``: authentication challenges and throttling+ headers are irrelevant here because the MCP transport already authenticated+ the caller. What matters is that the view's own refusals stay readable to+ the agent rather than becoming an opaque transport failure.++ A 5xx is logged — through ``log_exception``, so the message is redacted and+ no traceback is attached. Several Prompt Studio exceptions —+ ``IndexingAPIError``, ``AnswerFetchError``, ``ExtractionAPIError`` — are+ ``APIException`` subclasses carrying ``status_code = 500``, so without this+ a genuine server fault on a billable path would be converted to an ordinary+ tool result and leave no operator signal at all.++ A missing object is *not* logged: an agent naming an id that does not exist+ is ordinary, and logging it would make a caller's typo look like a fault.+ """+ if isinstance(error, Http404):+ return Response({"detail": "Not found."}, status=404)++ if isinstance(error, ObjectDoesNotExist):+ # `str(DocumentManager.DoesNotExist())` is empty, so pass the agent+ # something it can act on: which id to re-read and from where. A 404+ # rather than a 500 because nothing failed — the named thing is absent.+ return Response(+ {+ "detail": (+ "No such document or prompt in this project. Re-read the "+ "ids with listPromptStudioDocuments and listPrompts — an id "+ "from another project will not resolve here."+ )+ },+ status=404,+ )++ status_code = getattr(error, "status_code", 500)+ if status_code >= 500:+ # Redacted: the delegated view reaches adapters and vector stores, so+ # its exceptions can carry a provider key.+ log_exception(logger, f"Prompt Studio view failed with {status_code}", error)++ detail = getattr(error, "detail", str(error))+ return Response({"detail": detail}, status=status_code)+++def _result(response, project: CustomTool) -> dict[str, Any]:+ """Normalise a DRF response into a tool result.++ A non-2xx status is returned as data rather than raised: the view's own+ error bodies (a missing prompt id, an unindexed document) are precisely+ what the agent needs to correct its next call.+ """+ status_code = getattr(response, "status_code", None)+ data = getattr(response, "data", None)+ ok = status_code is not None and 200 <= status_code < 300++ result: dict[str, Any] = {+ "project_id": str(project.tool_id),+ "project_name": project.tool_name,+ "ok": ok,+ "status": status_code,+ # This app did not assemble `data` field by field — it is whatever the+ # delegated view returned, error bodies included — so it gets the same+ # redaction net the named error messages get.+ "result": redact_structure(data),+ }+ if not ok:+ # Distinguish "your arguments were wrong" from "the server failed".+ # These tools are billable and never refunded, so telling an agent a+ # 500 was probably a bad argument invites it to retry — and pay again —+ # against a fault no argument of its own can fix.+ if status_code is not None and status_code >= 500:+ result["note"] = (+ "The operation failed on the server, not because of your "+ "arguments. Retrying is unlikely to help and consumes budget "+ "each time. Report this to your Unstract administrator."+ )+ else:+ result["note"] = (+ "The operation was rejected. Read `result` for the reason — it "+ "is usually a missing or invalid argument, or a document that "+ "has not been indexed yet."+ )+ return result+++def index_document_schema() -> dict[str, Any]:+ return {+ "type": "object",+ "properties": {+ "project_id": {+ "type": "string",+ "description": (_PROJECT_ID_DESC_WITH_HINT),+ },+ "document_id": {+ "type": "string",+ "description": "UUID of the document within that project to index.",+ },+ },+ "required": ["project_id", "document_id"],+ }+++def index_document(+ context: PlatformMCPContext, project_id: str, document_id: str+) -> dict[str, Any]:+ """Index a document so prompts can be run against it."""+ project = _resolve_project(context, project_id)+ logger.info(+ f"MCP index_document project='{project_id}' document='{document_id}' "+ f"(org '{context.org_name}', key '{context.platform_key.name}')"+ )+ response = _dispatch(+ context, "index_document", project_id, {"document_id": document_id}+ )+ return _result(response, project)+++def fetch_response_schema() -> dict[str, Any]:+ return {+ "type": "object",+ "properties": {+ "project_id": {+ "type": "string",+ "description": _PROJECT_ID_DESC,+ },+ "document_id": {+ "type": "string",+ "description": "UUID of the document to run the prompt against.",+ },+ "prompt_id": {+ "type": "string",+ "description": "UUID of the prompt to run.",+ },+ "profile_manager_id": {+ "type": "string",+ "description": (+ "Optional UUID of an LLM profile to override the project's default."+ ),+ },+ },+ "required": ["project_id", "document_id", "prompt_id"],+ }+++def fetch_response(+ context: PlatformMCPContext,+ project_id: str,+ document_id: str,+ prompt_id: str,+ profile_manager_id: str | None = None,+) -> dict[str, Any]:+ """Run a single prompt against an indexed document."""+ project = _resolve_project(context, project_id)+ payload: dict[str, Any] = {"document_id": document_id, "id": prompt_id}+ if profile_manager_id:+ payload["profile_manager"] = profile_manager_id++ logger.info(+ f"MCP fetch_response project='{project_id}' prompt='{prompt_id}' "+ f"(org '{context.org_name}', key '{context.platform_key.name}')"+ )+ response = _dispatch(context, "fetch_response", project_id, payload)+ return _result(response, project)+++def bulk_fetch_response_schema() -> dict[str, Any]:+ return {+ "type": "object",+ "properties": {+ "project_id": {+ "type": "string",+ "description": _PROJECT_ID_DESC,+ },+ "document_id": {+ "type": "string",+ "description": "UUID of the document to run the prompts against.",+ },+ "prompt_ids": {+ "type": "array",+ "items": {"type": "string"},+ "minItems": 1,+ "description": "UUIDs of the prompts to run.",+ },+ },+ "required": ["project_id", "document_id", "prompt_ids"],+ }+++def bulk_fetch_response(+ context: PlatformMCPContext,+ project_id: str,+ document_id: str,+ prompt_ids: list[str],+) -> dict[str, Any]:+ """Run several prompts against one document in a single pass.++ Preferred over repeated ``fetchResponse`` calls: it indexes once and+ dispatches one task, which is both cheaper and avoids the+ document-being-indexed race that concurrent single calls provoke.+ """+ project = _resolve_project(context, project_id)+ logger.info(+ f"MCP bulk_fetch_response project='{project_id}' "+ f"prompts={len(prompt_ids)} (org '{context.org_name}', "+ f"key '{context.platform_key.name}')"+ )+ response = _dispatch(+ context,+ "bulk_fetch_response",+ project_id,+ {"document_id": document_id, "prompt_ids": prompt_ids},+ )+ return _result(response, project)+++def single_pass_extraction_schema() -> dict[str, Any]:+ return {+ "type": "object",+ "properties": {+ "project_id": {+ "type": "string",+ "description": _PROJECT_ID_DESC,+ },+ "document_id": {+ "type": "string",+ "description": "UUID of the document to extract from.",+ },+ },+ "required": ["project_id", "document_id"],+ }+++def single_pass_extraction(+ context: PlatformMCPContext, project_id: str, document_id: str+) -> dict[str, Any]:+ """Run the project's whole prompt set in a single LLM pass."""+ project = _resolve_project(context, project_id)+ logger.info(+ f"MCP single_pass_extraction project='{project_id}' "+ f"document='{document_id}' (org '{context.org_name}', "+ f"key '{context.platform_key.name}')"+ )+ response = _dispatch(+ context, "single_pass_extraction", project_id, {"document_id": document_id}+ )+ return _result(response, project)+++def list_prompt_studio_documents_schema() -> dict[str, Any]:+ return {+ "type": "object",+ "properties": {+ "project_id": {+ "type": "string",+ "description": (_PROJECT_ID_DESC_WITH_HINT),+ },+ },+ "required": ["project_id"],+ }+++def list_prompt_studio_documents(+ context: PlatformMCPContext, project_id: str+) -> dict[str, Any]:+ """List the documents already uploaded to a Prompt Studio project.++ This is the only producer of the ``document_id`` that every billable tool+ in this module requires, so without it those tools are unreachable.++ Documents are resolved *through* the project rather than by a direct+ ``DocumentManager`` lookup. The platform key authenticates as a service+ account, for which ``for_user`` returns everything, so the project join in+ ``_resolve_project`` is the scoping that actually holds.+ """+ project = _resolve_project(context, project_id)+ documents = project.document_managers.all().order_by("document_name")++ return {+ "project_id": str(project.tool_id),+ "project_name": project.tool_name,+ "documents": [+ {+ "document_id": str(document.document_id),+ "document_name": document.document_name,+ }+ for document in documents+ ],+ }+++def list_prompts_schema() -> dict[str, Any]:+ return {+ "type": "object",+ "properties": {+ "project_id": {+ "type": "string",+ "description": (_PROJECT_ID_DESC_WITH_HINT),+ },+ },+ "required": ["project_id"],+ }+++def list_prompts(context: PlatformMCPContext, project_id: str) -> dict[str, Any]:+ """List a project's prompts, which is where ``prompt_id`` comes from.++ The response is built field by field rather than through a serializer.+ ``ToolStudioPrompt`` carries a ``profile_manager`` FK — the LLM, embedding+ and vector-store adapters backing the prompt — and a+ ``postprocessing_webhook_url``. A serializer would carry both out to the+ agent, which is exactly what this server's README promises it does not do.+ Adding a field here is therefore a deliberate act.+ """+ project = _resolve_project(context, project_id)+ prompts = project.mapped_prompt.all().order_by("sequence_number")++ return {+ "project_id": str(project.tool_id),+ "project_name": project.tool_name,+ "prompts": [+ {+ "prompt_id": str(prompt.prompt_id),+ "prompt_key": prompt.prompt_key,+ "prompt": prompt.prompt,+ "prompt_type": prompt.prompt_type,+ "enforce_type": prompt.enforce_type,+ "sequence_number": prompt.sequence_number,+ "active": prompt.active,+ }+ for prompt in prompts+ ],+ }+++def preflight_project(+ context: PlatformMCPContext, project_id: str, **_ignored: Any+) -> None:+ """Resolve the project before any budget is claimed.++ Every billable tool in this module already calls ``_resolve_project`` — but+ inside the handler, which runs *after* the budget is consumed. A stale or+ mistyped ``project_id`` would therefore cost a slot for a call that never+ reached an LLM. Running the same resolution first makes that refusal free.++ Cheap and deterministic by design: a single indexed lookup the handler+ repeats anyway, not work that could itself fail transiently.+ """+ _resolve_project(context, project_id)diff --git a/backend/mcp_server/transport.py b/backend/mcp_server/transport.pynew file mode 100644index 000000000..555d8bd17--- /dev/null+++ b/backend/mcp_server/transport.py@@ -0,0 +1,453 @@+"""Shared JSON-RPC 2.0 transport for the hosted MCP servers.++Two MCP servers are hosted from this app — one scoped to a single API+deployment, one scoped to an organization. They differ only in how a caller is+authenticated and which tools they expose; the wire protocol, dispatch and+error mapping are identical, and live here so the two cannot drift apart on+protocol behaviour.++Subclasses supply:+ registry - the MCPToolRegistry to dispatch into+ resolve_context() - authenticate, or return None to reject+ server_instructions() - the prose an agent reads on `initialize`+"""++import json+import logging+from typing import Any++from django.http import JsonResponse+from rest_framework import status, views+from rest_framework.exceptions import ParseError+from rest_framework.request import Request++from mcp_server.constants import JSONRPC, MCPMethod, MCPServer+from mcp_server.exceptions import MCPToolError+from mcp_server.registry import MCPToolRegistry+from mcp_server.sanitize import log_exception++logger = logging.getLogger(__name__)+++def negotiate_protocol_version(requested: Any) -> str:+ """Choose the protocol revision to answer `initialize` with.++ The spec's rule is an echo, not an announcement: "If the server supports+ the requested protocol version, it MUST respond with the same version.+ Otherwise, the server MUST respond with another protocol version it+ supports" (rev 2025-06-18, Lifecycle → Version Negotiation).++ So a client asking for a revision this server speaks gets that revision+ back, and anything else — an unknown revision, a malformed value, or an+ omitted field — is answered with this server's preferred one. The client+ then decides whether it can live with that and disconnects if not, which+ is why an unsupported request is not an error response here.+ """+ if isinstance(requested, str) and requested in MCPServer.SUPPORTED_PROTOCOL_VERSIONS:+ return requested+ if requested is not None:+ logger.info(+ f"MCP client requested unsupported protocol version {requested!r}; "+ f"offering {MCPServer.PROTOCOL_VERSION}"+ )+ return MCPServer.PROTOCOL_VERSION+++def rpc_result(request_id: Any, result: Any) -> JsonResponse:+ """Build a JSON-RPC success response.++ JsonResponse rather than DRF's Response so the body is emitted verbatim:+ content negotiation must not turn a JSON-RPC envelope into the browsable+ API renderer for a client that sent a permissive Accept header.+ """+ return JsonResponse({"jsonrpc": JSONRPC.VERSION, "id": request_id, "result": result})+++def rpc_error(request_id: Any, code: int, message: str, data: Any = None) -> JsonResponse:+ """Build a JSON-RPC error response.++ Always HTTP 200: transport-level success with an application-level error is+ exactly what JSON-RPC models, and clients read the envelope, not the status+ code. The one exception is authentication, which must be a real 401 so MCP+ clients can react to it (see `auth_error`).+ """+ error: dict[str, Any] = {"code": code, "message": message}+ if data is not None:+ error["data"] = data+ return JsonResponse({"jsonrpc": JSONRPC.VERSION, "id": request_id, "error": error})+++def auth_error(+ message: str, http_status: int = status.HTTP_401_UNAUTHORIZED+) -> JsonResponse:+ """Build a 401/403 for a rejected credential.++ Unlike other failures this carries a real HTTP status, because a client+ that cannot authenticate needs to distinguish "wrong key" from "tool+ failed" before it has a usable session.+ """+ response = JsonResponse(+ {+ "jsonrpc": JSONRPC.VERSION,+ "id": None,+ "error": {"code": JSONRPC.UNAUTHORIZED, "message": message},+ },+ status=http_status,+ )+ if http_status == status.HTTP_401_UNAUTHORIZED:+ response["WWW-Authenticate"] = 'Bearer realm="unstract-mcp"'+ return response+++def tool_content(text: str, is_error: bool = False) -> dict[str, Any]:+ """Wrap tool output in the MCP `tools/call` content envelope."""+ return {"content": [{"type": "text", "text": text}], "isError": is_error}+++class BaseMCPView(views.APIView):+ """JSON-RPC transport shared by the deployment and platform MCP servers."""++ #: Tools this server exposes. Set by each subclass; None on the base class,+ #: which is never routed to directly.+ registry: MCPToolRegistry | None = None++ def get_registry(self) -> MCPToolRegistry:+ """Return the registry to dispatch into.++ Indirection exists so tests can patch a single seam rather than a+ module-level global shared by both servers.+ """+ return self.registry++ def resolve_context(self, **kwargs: Any) -> Any | None:+ """Authenticate the caller and build the per-request context.++ Return None to reject. Subclasses must answer every failure mode+ identically so the endpoint cannot be used to probe what exists.+ """+ raise NotImplementedError++ def server_instructions(self, context: Any) -> str:+ """Prose returned on `initialize`, read by the agent as a prompt."""+ raise NotImplementedError++ def auth_failure_message(self) -> str:+ """Message used for every rejected credential on this server."""+ return "Invalid or missing credentials"++ def server_info(self) -> dict[str, Any]:+ """Identity advertised on GET and on `initialize`."""+ return {+ "name": MCPServer.NAME,+ "version": MCPServer.VERSION,+ "protocolVersion": MCPServer.PROTOCOL_VERSION,+ "transport": "http",+ "authMethods": ["bearer"],+ }++ def get(self, request: Request, **kwargs: Any) -> JsonResponse:+ """Refuse the SSE stream, but say who is here.++ Under Streamable HTTP a client issues GET to open a server-to-client+ SSE stream, and a server that offers none must answer 405 (spec rev+ 2025-06-18). Nothing here pushes messages — every tool call is+ request/response — so 405 is the honest answer, and returning+ ``200 application/json`` instead would leave a conformant client+ parsing an identity document as an event stream.++ The body is kept anyway: uptime checks and humans with curl probe this+ path, and a 405 may carry one. It stays deliberately free of tenant+ detail — it reveals only that an MCP server is mounted here.++ ``JsonResponse``, not DRF's ``Response``, for the same reason ``post``+ uses it: a DRF response runs content negotiation, so a client sending+ ``Accept: text/html`` would be handed the browsable-API renderer.++ No ``Allow`` header is set here. RFC 9110 asks for one on a 405, but a+ handler cannot control it and pretending otherwise misleads a reader:+ DRF's ``finalize_response`` overwrites any handler-set value with+ ``self.allowed_methods`` (``GET, POST, HEAD, OPTIONS``, since this view+ defines both verbs), and ``RemoveAllowHeaderMiddleware`` — global in+ ``MIDDLEWARE`` — then pops the header from every response before it+ leaves the process. So a client sees no ``Allow`` at all; a test driving+ the view through ``APIRequestFactory`` bypasses that middleware and sees+ DRF's value.+ """+ return JsonResponse(self.server_info(), status=405)++ def post(self, request: Request, **kwargs: Any) -> JsonResponse:+ """Handle a single JSON-RPC request."""+ context = self.resolve_context(request=request, **kwargs)+ if context is None:+ return auth_error(self.auth_failure_message())++ try:+ body = request.data+ except ParseError:+ # DRF would answer 400 with its own error body. A JSON-RPC client+ # is parsing for an envelope, so give it -32700 as the spec+ # requires; the id is null because unparseable input has none.+ return rpc_error(+ None,+ JSONRPC.PARSE_ERROR,+ "Parse error",+ "Request body is not valid JSON",+ )++ if not isinstance(body, dict):+ # Batch requests are valid JSON-RPC 2.0 but are not used by MCP+ # clients; rejecting them explicitly beats a confusing downstream+ # AttributeError.+ return rpc_error(+ None,+ JSONRPC.INVALID_REQUEST,+ JSONRPC.MSG_INVALID_REQUEST,+ "Expected a single JSON-RPC object",+ )++ request_id = body.get("id")+ method = body.get("method")++ if body.get("jsonrpc") != JSONRPC.VERSION:+ return rpc_error(+ request_id,+ JSONRPC.INVALID_REQUEST,+ JSONRPC.MSG_INVALID_REQUEST,+ "Only JSON-RPC 2.0 is supported",+ )+ # Type-checked, not just truthiness: JSON permits any type here, and a+ # non-string reaches `method.startswith(...)` below as an+ # AttributeError — a Django 500 with no JSON-RPC envelope, which is the+ # one failure a conformant client cannot parse. Same reasoning as the+ # `params` check below.+ if not isinstance(method, str) or not method:+ return rpc_error(+ request_id,+ JSONRPC.INVALID_REQUEST,+ JSONRPC.MSG_INVALID_REQUEST,+ "Missing method" if not method else "'method' must be a string",+ )++ # `params` is optional, but when present JSON-RPC allows an array or an+ # object and MCP uses only objects. A truthy non-dict survives `or {}`+ # and reaches `params.get(...)` downstream as an AttributeError, i.e. a+ # Django 500 rather than an envelope the client can read.+ params = body.get("params") or {}+ if not isinstance(params, dict):+ return rpc_error(+ request_id,+ JSONRPC.INVALID_PARAMS,+ JSONRPC.MSG_INVALID_PARAMS,+ "'params' must be an object",+ )++ return self._dispatch(+ method=method,+ request_id=request_id,+ params=params,+ context=context,+ )++ def _dispatch(+ self, method: str, request_id: Any, params: dict[str, Any], context: Any+ ) -> JsonResponse:+ """Route a JSON-RPC method to its handler."""+ if method == MCPMethod.INITIALIZE:+ return rpc_result(+ request_id,+ {+ "protocolVersion": negotiate_protocol_version(+ params.get("protocolVersion")+ ),+ "capabilities": {"tools": {"listChanged": False}},+ "serverInfo": {+ "name": MCPServer.NAME,+ "version": MCPServer.VERSION,+ },+ "instructions": self.server_instructions(context),+ },+ )++ if method.startswith(MCPMethod.NOTIFICATION_PREFIX):+ # Notifications (e.g. notifications/initialized) carry no id and+ # must not receive a result body.+ return JsonResponse({}, status=status.HTTP_202_ACCEPTED)++ if method == MCPMethod.PING:+ return rpc_result(request_id, {})++ if method == MCPMethod.TOOLS_LIST:+ return rpc_result(request_id, {"tools": self.get_registry().list_schemas()})++ if method == MCPMethod.TOOLS_CALL:+ return self._call_tool(request_id=request_id, params=params, context=context)++ return rpc_error(+ request_id,+ JSONRPC.METHOD_NOT_FOUND,+ "Method not found",+ f"Unsupported method '{method}'",+ )++ def check_tool_allowed(self, tool: Any, context: Any) -> str | None:+ """Authorize a specific tool for this caller.++ Return None to allow, or a message explaining the refusal. The default+ allows everything the registry exposes; servers with permission tiers+ override this.+ """+ return None++ def check_spend_allowed(self, tool: Any, context: Any) -> str | None:+ """Claim budget for a billable tool.++ Return None to allow, or a message explaining that the budget is spent.+ Called once per invocation immediately before the handler runs, so+ overriding servers should treat it as consuming, not merely checking.++ The default allows everything — the deployment server's costs are+ already bounded by the API deployment rate limiter.+ """+ return None++ def _run_preflight(+ self,+ request_id: Any,+ tool: Any,+ tool_name: str,+ context: Any,+ arguments: dict[str, Any],+ ) -> JsonResponse | None:+ """Validate a billable tool's arguments before any budget is claimed.++ Returns the response to send if preflight rejects the call, or None to+ continue. Budget is consumed on invocation and never refunded, so+ without this a caller naming a deployment that does not exist would pay+ a slot for a call that spent nothing upstream — and an agent guessing at+ a name could exhaust the window without ever running an extraction.++ Split out of ``_call_tool`` to keep that method's branching readable;+ the ordering guarantee it provides (validate before charge) is the+ reason it runs where it does.+ """+ if tool.preflight is None:+ return None+ try:+ tool.preflight(context, **arguments)+ except MCPToolError as error:+ return rpc_result(request_id, tool_content(str(error), is_error=True))+ except TypeError as error:+ logger.warning(+ f"MCP tool '{tool_name}' preflight rejected arguments: {error}"+ )+ return rpc_error(+ request_id,+ JSONRPC.INVALID_PARAMS,+ JSONRPC.MSG_INVALID_PARAMS,+ str(error),+ )+ except Exception as error:+ # Same catch-all the handler path has, and for the same reason: a+ # resolver filtering a UUID column by a malformed string raises+ # Django ValidationError, which would otherwise escape post() and+ # reach the client as a bare 500 with no JSON-RPC envelope at all.+ log_exception(+ logger, f"MCP tool '{tool_name}' preflight failed unexpectedly", error+ )+ return rpc_error(+ request_id,+ JSONRPC.TOOL_EXECUTION_ERROR,+ "Tool execution failed",+ f"Tool '{tool_name}' could not validate its arguments. "+ "Contact your Unstract administrator if this persists.",+ )+ return None++ def _call_tool(+ self, request_id: Any, params: dict[str, Any], context: Any+ ) -> JsonResponse:+ """Invoke a registered tool and wrap its result in MCP content format."""+ registry = self.get_registry()+ tool_name = params.get("name")+ # Type-checked for the same reason `method` is, plus one specific to the+ # lookup: an unhashable name (a JSON object or array) raises+ # `TypeError: unhashable type` inside `registry.get()`, which is a dict+ # lookup — again a 500 with no envelope for the client to read.+ if not isinstance(tool_name, str) or not tool_name:+ return rpc_error(+ request_id,+ JSONRPC.INVALID_PARAMS,+ JSONRPC.MSG_INVALID_PARAMS,+ "Missing tool name" if not tool_name else "'name' must be a string",+ )++ tool = registry.get(tool_name)+ if tool is None:+ return rpc_error(+ request_id,+ JSONRPC.METHOD_NOT_FOUND,+ "Method not found",+ f"Tool '{tool_name}' not found. Available tools: {registry.names()}",+ )++ refusal = self.check_tool_allowed(tool, context)+ if refusal is not None:+ return rpc_error(+ request_id, JSONRPC.UNAUTHORIZED, "Permission denied", refusal+ )++ arguments = params.get("arguments") or {}+ if not isinstance(arguments, dict):+ return rpc_error(+ request_id,+ JSONRPC.INVALID_PARAMS,+ JSONRPC.MSG_INVALID_PARAMS,+ "'arguments' must be an object",+ )++ preflight_error = self._run_preflight(+ request_id, tool, tool_name, context, arguments+ )+ if preflight_error is not None:+ return preflight_error++ # Budget is checked after permission and preflight, and before the+ # handler, so neither an unauthorized call nor one that could never+ # have run consumes budget. Unlike the permission refusal above this is+ # temporal, so it comes back as an isError *result* the agent can read+ # and retry — a protocol error would read as "this tool does not work"+ # and stop it retrying later.+ over_budget = self.check_spend_allowed(tool, context)+ if over_budget is not None:+ return rpc_result(request_id, tool_content(over_budget, is_error=True))++ try:+ result = tool.handler(context, **arguments)+ except MCPToolError as error:+ # Expected, agent-actionable failure: the message is written for+ # the agent, so pass it through as an error result rather than a+ # protocol error. isError=True lets the agent see and retry it.+ return rpc_result(request_id, tool_content(str(error), is_error=True))+ except TypeError as error:+ # Almost always the agent passing arguments the tool does not+ # accept; report it as bad params rather than a server fault.+ logger.warning(f"MCP tool '{tool_name}' called with bad arguments: {error}")+ return rpc_error(+ request_id, JSONRPC.INVALID_PARAMS, JSONRPC.MSG_INVALID_PARAMS, str(error)+ )+ except Exception as error:+ # Redacted: this catches *any* tool handler, so the exception can+ # come from a connector, a provider client or the execution stack —+ # all of which report credentials in their error text.+ log_exception(logger, f"MCP tool '{tool_name}' failed", error)+ return rpc_error(+ request_id,+ JSONRPC.TOOL_EXECUTION_ERROR,+ "Tool execution failed",+ f"Tool '{tool_name}' failed unexpectedly. "+ "Contact your Unstract administrator if this persists.",+ )++ return rpc_result(+ request_id, tool_content(json.dumps(result, indent=2, default=str))+ )diff --git a/backend/mcp_server/urls.py b/backend/mcp_server/urls.pynew file mode 100644index 000000000..e78600f74--- /dev/null+++ b/backend/mcp_server/urls.py@@ -0,0 +1,40 @@+"""URLs for the deployment-scoped MCP server.++Included from ``api_v2/execution_urls.py`` so the MCP endpoint hangs directly+off the deployment's own execution URL — the same resource, reached two ways:++ POST /deployment/api/<org_name>/<api_name>/ (REST)+ POST /deployment/api/<org_name>/<api_name>/mcp (MCP)++Sitting under ``API_DEPLOYMENT_PATH_PREFIX`` means these paths are already+covered by that prefix's entry in ``WHITELISTED_PATHS``, so+``CustomAuthMiddleware`` skips them and the view authenticates the deployment+key itself — exactly as it does for the execution endpoint next door.+"""++from django.urls import re_path++from mcp_server.views import MCPServerView++mcp_server = MCPServerView.as_view()++urlpatterns = [+ re_path(+ r"^api/(?P<org_name>[\w-]+)/(?P<api_name>[\w-]+)/mcp/?$",+ mcp_server,+ name="mcp_server",+ ),+]++# There is deliberately no route carrying the API key as a path segment. An+# earlier revision offered one for "MCP clients that cannot set headers", but+# that client class does not exist for HTTP transports: the spec requires+# `Authorization` on every authenticated request ("Authorization MUST be+# included in every HTTP request from client to server"), and states that+# access tokens MUST NOT appear in the URI. A key in the path reaches nginx and+# gunicorn access logs, APM traces, and any intermediary — places a header+# never goes — and rotating a deployment key is not free.+#+# Basic auth would be spec-legal and is supported by MCP clients, but buys+# nothing here: it is the same `Authorization` header as Bearer, and base64 is+# encoding rather than encryption. Any client that can send one can send both.diff --git a/backend/mcp_server/views.py b/backend/mcp_server/views.pynew file mode 100644index 000000000..53ef1f478--- /dev/null+++ b/backend/mcp_server/views.py@@ -0,0 +1,97 @@+"""Deployment-scoped MCP server.++Mirrors the shape of the API deployment execution endpoint: the URL carries the+organization and the deployment name, and the deployment's own API key+authenticates the caller. An MCP session is therefore scoped to exactly one API+deployment, and reuses that deployment's existing key management.++This endpoint is in ``WHITELISTED_PATHS``, so ``CustomAuthMiddleware`` skips it+and the view owns its own authentication — deliberately, because a deployment+key is not a platform key and must not resolve to a user.+"""++import logging+from typing import Any++from api_v2.deployment_helper import DeploymentHelper+from rest_framework.request import Request++from mcp_server.context import MCPContext+from mcp_server.registry import DEPLOYMENT_TOOLS+from mcp_server.transport import BaseMCPView++logger = logging.getLogger(__name__)+++class MCPServerView(BaseMCPView):+ """MCP JSON-RPC endpoint for a single API deployment.++ Authentication is the deployment's API key, accepted only as a bearer+ token — never from the URL; see ``urls.py``. This is a public,+ key-authenticated endpoint like the deployment execution endpoint it sits+ beside, so session auth does not apply.+ """++ registry = DEPLOYMENT_TOOLS++ authentication_classes: list = []+ permission_classes: list = []++ def initialize_request(self, request: Request, *args: Any, **kwargs: Any) -> Request:+ """Skip CSRF, matching the public API deployment endpoint."""+ request.csrf_processing_done = True+ return super().initialize_request(request, *args, **kwargs)++ def auth_failure_message(self) -> str:+ return "Invalid API key or unknown API deployment"++ def server_instructions(self, context: MCPContext) -> str:+ return (+ "Unstract runs LLM-driven extraction over unstructured documents "+ "and returns structured JSON. This server is scoped to the API "+ f"deployment '{context.api.display_name}'. Call readMeFirst before "+ "any other tool."+ )++ def resolve_context(+ self,+ request: Request,+ org_name: str,+ api_name: str,+ **kwargs: Any,+ ) -> MCPContext | None:+ """Authenticate the key against the named deployment.++ Returns None for every failure mode — unknown deployment, wrong key,+ malformed key — so the caller answers all of them identically and the+ endpoint cannot be used to probe which deployment names exist.++ The bearer header is the only accepted credential. The key is+ deliberately not read from the URL: see ``urls.py`` for why that route+ was removed.+ """+ # The auth scheme is case-insensitive per RFC 9110, and DRF's own+ # authenticators lowercase it before comparing — so a client sending+ # `bearer <key>` is conformant and must not be rejected.+ scheme, _, credentials = request.headers.get("Authorization", "").partition(" ")+ api_key = credentials.strip() if scheme.lower() == "bearer" else ""++ if not api_key:+ return None++ # Pin the organization before touching org-scoped managers; the+ # deployment lookup below filters on it.+ DeploymentHelper.validate_parameters(+ request, api_name=api_name, org_name=org_name+ )++ api_deployment = DeploymentHelper.get_deployment_by_api_name(api_name=api_name)+ try:+ DeploymentHelper.validate_api(api_deployment=api_deployment, api_key=api_key)+ except Exception as error:+ logger.warning(+ f"MCP auth rejected for org '{org_name}', api '{api_name}': {error}"+ )+ return None++ return MCPContext(api=api_deployment, api_key=api_key, org_name=org_name)diff --git a/backend/permissions/permission.py b/backend/permissions/permission.pyindex ad29da0dc..33d4dd507 100644--- a/backend/permissions/permission.py+++ b/backend/permissions/permission.py@@ -14,15 +14,22 @@ _REQUEST_ADMIN_CACHE_ATTR = "_cached_is_organization_admin"def _is_service_account(request: Request) -> bool:- """Allow service accounts through for all non-DELETE methods.-- Two constraints are enforced upstream in the authentication middleware- before this check is ever reached:- 1. DELETE is blocked for all API keys.- 2. Write methods (POST/PUT/PATCH) are blocked for READ-only API keys.-- Therefore any service-account request that arrives here is permitted to- proceed regardless of HTTP method.+ """Allow service accounts through regardless of HTTP method.++ The authentication middleware has already checked the key's permission+ tier against the request method (``ApiKeyPermission.allows``) before this+ is reached, so the method itself needs no second look here:+ 1. ``read`` keys reach only GET/HEAD/OPTIONS.+ 2. ``read_write`` keys add POST/PUT/PATCH.+ 3. ``full_access`` keys add DELETE.++ Note the third case: an earlier version of this docstring said DELETE was+ blocked for all API keys, which stopped being true when ``full_access``+ was introduced. A ``full_access`` service-account request does reach this+ check on DELETE and is allowed through — object-level ownership is not+ consulted, because a service account is org-wide by design. That tier is+ the only thing standing between an API key and deleting any resource in+ the organization, so grant it deliberately."""return getattr(request.user, "is_service_account", False)diff --git a/backend/sample.env b/backend/sample.envindex e04445822..b57101813 100644--- a/backend/sample.env+++ b/backend/sample.env@@ -259,3 +259,14 @@ HITL_FILES_FILE_STORAGE_CREDENTIALS='{"provider": "minio", "credentials": {"endp# File active cache redis dbFILE_ACTIVE_CACHE_REDIS_DB=0++# Hosted MCP servers+# The deployment-scoped server (/deployment/api/<org>/<api_name>/mcp) is always+# mounted. The organization-scoped server (/api/v1/unstract/<org>/mcp/) is off+# by default — it exposes 23 org-wide tools and only the deployment server is+# required for the initial release. Set to "true" to enable it.+MCP_PLATFORM_SERVER_ENABLED=false+# Budget for billable MCP tool calls (LLM inference, indexing, pipeline runs),+# counted per organization over a rolling window. Counts calls, not tokens.+MCP_BILLABLE_CALL_LIMIT=50+MCP_BILLABLE_WINDOW_SECONDS=3600diff --git a/tests/critical_paths.yaml b/tests/critical_paths.yamlindex 00429280e..94aa29f00 100644--- a/tests/critical_paths.yaml+++ b/tests/critical_paths.yaml@@ -66,6 +66,16 @@ paths:covered_by: [e2e-api-deployment]proof: marker+ - id: mcp-server-auth+ description: "Unauthenticated or mis-scoped hosted-MCP calls are rejected before any tool runs."+ covered_by: [integration-backend]+ proof: marker++ - id: mcp-platform-auth+ description: "The org-scoped MCP endpoint stays behind the platform-API-key middleware; unauthenticated or mis-scoped calls reach no tool."+ covered_by: [integration-backend]+ proof: marker+- id: prompt-studio-authordescription: "Create a Prompt Studio project and add a prompt to it."covered_by: [integration-backend]