Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
refactor(pipedream_mcp): one Connect proxy port on http types (#8598)
Wolf Mermelsteine08ec62b3aa0
diff --git a/Cargo.lock b/Cargo.lockindex 57cefa5b04..f745e2fea7 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -8827,6 +8827,7 @@ dependencies = ["call","chrono","hmac 0.12.1",+ "http 1.4.0","macro_db_migrator","macro_user_id","macro_uuid",@@ -14844,6 +14845,7 @@ dependencies = ["axum","base64 0.22.1","futures",+ "http 1.4.0","macro_authorization","macro_user_id","mcp_toolset",@@ -14860,6 +14862,7 @@ dependencies = ["tracing","url","utoipa",+ "wiremock","workspace-hack",]diff --git a/crates/granola_sync/AGENTS.md b/crates/granola_sync/AGENTS.mdindex 4152bbeddf..12308e8bb7 100644--- a/crates/granola_sync/AGENTS.md+++ b/crates/granola_sync/AGENTS.md@@ -26,7 +26,7 @@ through `call::domain::imports`.## Boundaries-- Depends on: `call` import port, `pipedream_mcp` `ApiProxy` / `ConnectionStore` / `AppConnections`.+- Depends on: `call` import port, `pipedream_mcp` `ConnectedAppProxy` / `ConnectionStore` / `AppConnections`.- Used by: `services/agent_harness_service` (composition root, worker, webhook route).## Checksdiff --git a/crates/granola_sync/Cargo.toml b/crates/granola_sync/Cargo.tomlindex deac21ab32..52891e8718 100644--- a/crates/granola_sync/Cargo.toml+++ b/crates/granola_sync/Cargo.toml@@ -12,6 +12,7 @@ base64.workspace = truecall = { path = "../call", default-features = false, features = ["ports"] }chrono.workspace = truehmac.workspace = true+http.workspace = truemacro_user_id = { path = "../macro_user_id" }macro_uuid = { path = "../macro_uuid" }pipedream_mcp = { path = "../pipedream_mcp" }diff --git a/crates/granola_sync/src/outbound/api.rs b/crates/granola_sync/src/outbound/api.rsindex 6180f51708..0f38acece7 100644--- a/crates/granola_sync/src/outbound/api.rs+++ b/crates/granola_sync/src/outbound/api.rs@@ -6,10 +6,8 @@ use crate::domain::{use async_trait::async_trait;use call::domain::{imports::ImportedCall, records::*};use chrono::{DateTime, Utc};-use pipedream_mcp::domain::{- models::PipedreamConnection,- ports::{ApiProxy, ProxyMethod, ProxyResponse},-};+use http::{HeaderMap, Method};+use pipedream_mcp::domain::{models::PipedreamConnection, ports::ConnectedAppProxy};use rootcause::prelude::IntoRootcause;use serde::Deserialize;use serde_json::{Value, json};@@ -18,19 +16,26 @@ use strum::VariantArray;pub struct GranolaApi<P>(pub Arc<P>);+/// Granola's answer, whatever its status. Never log the body: registration+/// returns the webhook secret.+struct Reply {+ status: u16,+ body: Value,+}+/// Provider IDs are opaque; encode them so one can never change the path.fn segment(id: &str) -> String {url::form_urlencoded::byte_serialize(id.as_bytes()).collect()}-impl<P: ApiProxy> GranolaApi<P> {+impl<P: ConnectedAppProxy> GranolaApi<P> {async fn send(&self,connection: &Connection,- method: ProxyMethod,+ method: Method,path: &str,body: Option<Value>,- ) -> Result<ProxyResponse> {+ ) -> Result<Reply> {let account = PipedreamConnection {user_id: connection.user_id.clone(),app_slug: GRANOLA_APP.into(),@@ -38,21 +43,34 @@ impl<P: ApiProxy> GranolaApi<P> {account_id: connection.account_id.clone(),enabled: true,};- self.0+ let response = self+ .0.proxy(&account,method,&format!("https://public-api.granola.ai/v1{path}"),- body,+ &HeaderMap::new(),+ body.as_ref(),).await- .into_rootcause()+ .into_rootcause()?;+ // Preserve status for provider-specific retries and missing resources.+ let body = if response.body.trim().is_empty() {+ Value::Null+ } else {+ serde_json::from_str(&response.body)+ .map_err(|error| rootcause::report!("invalid Granola JSON response: {error}"))?+ };+ Ok(Reply {+ status: response.status.as_u16(),+ body,+ })}async fn request(&self,connection: &Connection,- method: ProxyMethod,+ method: Method,path: &str,body: Option<Value>,) -> Result<Option<Value>> {@@ -61,7 +79,7 @@ impl<P: ApiProxy> GranolaApi<P> {}}-fn checked(response: ProxyResponse) -> Result<Option<Value>> {+fn checked(response: Reply) -> Result<Option<Value>> {match response.status {200..=299 => Ok(Some(response.body)),404 => Ok(None),@@ -105,7 +123,7 @@ fn error_reason(body: &Value) -> String {}#[async_trait]-impl<P: ApiProxy> Granola for GranolaApi<P> {+impl<P: ConnectedAppProxy> Granola for GranolaApi<P> {async fn register(&self,connection: &Connection,@@ -115,7 +133,7 @@ impl<P: ApiProxy> Granola for GranolaApi<P> {let response = self.send(connection,- ProxyMethod::Post,+ Method::POST,"/webhook-endpoints",Some(json!({"url": url, "scopes": scope.scopes(),@@ -145,7 +163,7 @@ impl<P: ApiProxy> Granola for GranolaApi<P> {let id = segment(id);self.request(connection,- ProxyMethod::Delete,+ Method::DELETE,&format!("/webhook-endpoints/{id}"),None,)@@ -157,7 +175,7 @@ impl<P: ApiProxy> Granola for GranolaApi<P> {let Some(body) = self.request(connection,- ProxyMethod::Get,+ Method::GET,&format!("/notes/{}", segment(id.as_ref())),None,)@@ -184,10 +202,7 @@ impl<P: ApiProxy> Granola for GranolaApi<P> {path.push('?');path.push_str(&query);}- let Some(page) = self- .request(connection, ProxyMethod::Get, &path, None)- .await?- else {+ let Some(page) = self.request(connection, Method::GET, &path, None).await? else {if cursor.is_some() {return Err(rootcause::report!("Granola transcript disappeared during pagination"diff --git a/crates/granola_sync/src/outbound/api/test.rs b/crates/granola_sync/src/outbound/api/test.rsindex 2c235ba150..dce286e2da 100644--- a/crates/granola_sync/src/outbound/api/test.rs+++ b/crates/granola_sync/src/outbound/api/test.rs@@ -106,8 +106,9 @@ impl FakeProxy {.into_iter().map(|(status, body)| pipedream_mcp::domain::ports::ProxyResponse {- status,- body,+ status: http::StatusCode::from_u16(status).unwrap(),+ headers: http::HeaderMap::new(),+ body: body.to_string(),},).collect(),@@ -116,13 +117,14 @@ impl FakeProxy {})}}-impl ApiProxy for FakeProxy {+impl ConnectedAppProxy for FakeProxy {async fn proxy(&self,connection: &PipedreamConnection,- _method: ProxyMethod,+ _method: http::Method,url: &str,- _body: Option<Value>,+ _headers: &http::HeaderMap,+ _body: Option<&Value>,) -> anyhow::Result<pipedream_mcp::domain::ports::ProxyResponse> {assert_eq!(connection.app_slug, "granola");assert_eq!(connection.account_id, "apn_account");diff --git a/crates/pipedream_mcp/AGENTS.md b/crates/pipedream_mcp/AGENTS.mdindex 8c8efe212f..dd29944c48 100644--- a/crates/pipedream_mcp/AGENTS.md+++ b/crates/pipedream_mcp/AGENTS.md@@ -11,8 +11,10 @@ AI loop calls their tools through Pipedream's remote MCP server.- `src/lib.rs`: crate doc comment — explains why this is fully separate fromthe native `mcp_client` stack.- `src/domain/ports.rs` + `src/domain/models.rs`: connection model and ports.-- `src/outbound/api.rs`: the Pipedream HTTP API client; catalog entries preserve- Pipedream category labels for clients to group without a local taxonomy.+- `src/outbound/api.rs`: the Pipedream HTTP API client, including the+ Connect API proxy (`ConnectedAppProxy`) that calls an app's own REST API+ with the account's credentials; catalog entries preserve Pipedream category+ labels for clients to group without a local taxonomy.- `src/outbound/pg_connection_repo.rs`: the `pipedream_mcp_connections` table.- `src/inbound/toolset.rs`: how connected apps surface as agent tools.@@ -27,8 +29,7 @@ AI loop calls their tools through Pipedream's remote MCP server.- Depends on: `ai_toolset`, `mcp_toolset`, `macro_authorization`,`macro_user_id`.-- Used by: `ai_tools`, `agent_egress`, `agent_harness`, `mcp_select`,- `services/agent_harness_service`, `services/agent_harness_service`,+- Used by: `ai_tools`, `agent_egress`, `agent_harness`, `granola_sync`, `mcp_select`,`services/agent_harness_service`.## Checksdiff --git a/crates/pipedream_mcp/Cargo.toml b/crates/pipedream_mcp/Cargo.tomlindex 1278ff9e0b..f3d511c49b 100644--- a/crates/pipedream_mcp/Cargo.toml+++ b/crates/pipedream_mcp/Cargo.toml@@ -11,6 +11,7 @@ async-trait.workspace = truebase64.workspace = trueaxum.workspace = truefutures.workspace = true+http.workspace = truemacro_authorization = { path = "../macro_authorization", default-features = false, features = ["axum",] }@@ -33,3 +34,4 @@ workspace-hack = { version = "0.1", path = "../workspace-hack" }[dev-dependencies]tokio.workspace = truetower = { workspace = true }+wiremock = { workspace = true }diff --git a/crates/pipedream_mcp/src/domain/ports.rs b/crates/pipedream_mcp/src/domain/ports.rsindex 4823a3a9bd..96878af607 100644--- a/crates/pipedream_mcp/src/domain/ports.rs+++ b/crates/pipedream_mcp/src/domain/ports.rs@@ -3,38 +3,6 @@ use super::models::{PipedreamConnection,};-/// HTTP methods supported by trusted backend integrations through Connect Proxy.-#[derive(Clone, Copy)]-pub enum ProxyMethod {- /// Read a provider resource.- Get,- /// Create a provider resource, such as a webhook subscription.- Post,- /// Remove a provider resource.- Delete,-}--/// A provider response. Never log the body: registration can return secrets.-pub struct ProxyResponse {- /// Upstream HTTP status.- pub status: u16,- /// Provider JSON body.- pub body: serde_json::Value,-}--/// Authenticated provider requests using credentials held by Pipedream.-/// Callers must obtain the account from the authenticated user's connection.-pub trait ApiProxy: Send + Sync {- /// Pipedream restricts targets to the connected app's allowed domains.- fn proxy(- &self,- connection: &PipedreamConnection,- method: ProxyMethod,- url: &str,- body: Option<serde_json::Value>,- ) -> impl Future<Output = anyhow::Result<ProxyResponse>> + Send;-}-/// Cross-user lookup for background integrations that follow a connected app.pub trait AppConnections: Send + Sync + 'static {/// Every user's enabled connection to `app_slug`.@@ -170,3 +138,50 @@ pub trait ConnectorCapabilities: Send + Sync + 'static {app_slug: &str,) -> impl Future<Output = anyhow::Result<Vec<super::service::discovery::ConnectorTool>>> + Send;}++/// A connected app's answer, relayed through the proxy whatever its status.+/// Never log the body: some apps return secrets (webhook registration).+#[derive(Clone, Debug, PartialEq, Eq)]+pub struct ProxyResponse {+ /// The HTTP status.+ pub status: http::StatusCode,+ /// The app's response headers.+ pub headers: http::HeaderMap,+ /// The response body.+ pub body: String,+}++impl ProxyResponse {+ /// How long the app asked callers to wait, from a `Retry-After` header+ /// in seconds.+ pub fn retry_after(&self) -> Option<std::time::Duration> {+ self.headers+ .get(http::header::RETRY_AFTER)?+ .to_str()+ .ok()?+ .trim()+ .parse()+ .ok()+ .map(std::time::Duration::from_secs)+ }+}++/// Port for calling a connected app's own API with the account's+/// credentials, which Pipedream injects server-side.+pub trait ConnectedAppProxy: Send + Sync + 'static {+ /// Send a request to the connected app's API on behalf of `connection`'s+ /// account.+ ///+ /// `url` is absolute (e.g. `https://api.notion.com/v1/search`).+ /// `headers` are upstream headers, never `Authorization`. A non-2xx+ /// answer from the app is a response, not an error, so callers can read+ /// its status and back off.+ fn proxy(+ &self,+ connection: &PipedreamConnection,+ method: http::Method,+ url: &str,+ headers: &http::HeaderMap,+ body: Option<&serde_json::Value>,+ ) -> impl Future<Output = anyhow::Result<ProxyResponse>> + Send;+}diff --git a/crates/pipedream_mcp/src/outbound/api.rs b/crates/pipedream_mcp/src/outbound/api.rsindex e704915715..0031a7b205 100644--- a/crates/pipedream_mcp/src/outbound/api.rs+++ b/crates/pipedream_mcp/src/outbound/api.rs@@ -2,10 +2,12 @@ use crate::domain::models::{CatalogEntry, CatalogPage, ConnectToken, McpServer, PipedreamAccount, PipedreamConnection,client_info,};-use crate::domain::ports::{ApiProxy, ProxyMethod, ProxyResponse};-use crate::domain::ports::{ConnectorDirectory, McpConnection, PipedreamConnect};+use crate::domain::ports::{+ ConnectedAppProxy, ConnectorDirectory, McpConnection, PipedreamConnect, ProxyResponse,+};use anyhow::Context;use base64::Engine;+use base64::engine::general_purpose::URL_SAFE_NO_PAD;use reqwest::header::{HeaderMap, HeaderValue};use rmcp::service::ServiceExt;use rmcp::transport::StreamableHttpClientTransport;@@ -17,50 +19,6 @@ use url::Url;mod capabilities;-impl ApiProxy for PipedreamClient {- #[tracing::instrument(skip_all, err)]- async fn proxy(- &self,- connection: &PipedreamConnection,- method: ProxyMethod,- url: &str,- body: Option<serde_json::Value>,- ) -> anyhow::Result<ProxyResponse> {- let target = base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(url);- let method = match method {- ProxyMethod::Get => reqwest::Method::GET,- ProxyMethod::Post => reqwest::Method::POST,- ProxyMethod::Delete => reqwest::Method::DELETE,- };- let mut request = self- .authed(- self.http- .request(method, self.connect_api(&format!("/proxy/{target}"))),- )- .await?- .query(&[- ("external_user_id", connection.user_id.as_ref()),- ("account_id", connection.account_id.as_str()),- ]);- if let Some(body) = body {- request = request.json(&body);- }- let response = request- .send()- .await- .context("Pipedream proxy request failed")?;- let status = response.status().as_u16();- // Preserve status for provider-specific retries and missing resources.- let bytes = response.bytes().await?;- let body = if bytes.is_empty() {- serde_json::Value::Null- } else {- serde_json::from_slice(&bytes).context("invalid proxy JSON response")?- };- Ok(ProxyResponse { status, body })- }-}-/// Default base URL of the Pipedream REST API.pub const DEFAULT_API_URL: &str = "https://api.pipedream.com";@@ -474,6 +432,55 @@ impl McpConnection for PipedreamClient {}}+/// The prefix Pipedream's Connect proxy strips to find upstream headers.+const PROXY_HEADER_PREFIX: &str = "x-pd-proxy-";++impl ConnectedAppProxy for PipedreamClient {+ #[tracing::instrument(skip_all, err, fields(app = %connection.app_slug, user_id = %connection.user_id, method = ?method))]+ async fn proxy(+ &self,+ connection: &PipedreamConnection,+ method: http::Method,+ url: &str,+ headers: &http::HeaderMap,+ body: Option<&serde_json::Value>,+ ) -> anyhow::Result<ProxyResponse> {+ if headers.contains_key(http::header::AUTHORIZATION) {+ anyhow::bail!("the Connect proxy injects Authorization; callers may not set it");+ }+ let endpoint = self.connect_api(&format!("/proxy/{}", URL_SAFE_NO_PAD.encode(url)));+ let mut request = self+ .authed(self.http.request(method, endpoint))+ .await?+ .query(&[+ ("external_user_id", connection.user_id.as_ref()),+ ("account_id", connection.account_id.as_str()),+ ]);+ for (name, value) in headers {+ request = request.header(format!("{PROXY_HEADER_PREFIX}{name}"), value);+ }+ if let Some(body) = body {+ request = request.json(body);+ }++ let response = request+ .send()+ .await+ .context("calling the Pipedream Connect proxy")?;+ let status = response.status();+ let headers = response.headers().clone();+ let body = response+ .text()+ .await+ .context("reading the Pipedream Connect proxy response")?;+ Ok(ProxyResponse {+ status,+ headers,+ body,+ })+ }+}+/// The charset a Pipedream app slug may use. A segment outside it could never/// name an app, and must not be placed in a request path.fn is_app_slug(slug: &str) -> bool {diff --git a/crates/pipedream_mcp/src/outbound/api/test.rs b/crates/pipedream_mcp/src/outbound/api/test.rsindex 6a0ffd1431..ca4f2edb13 100644--- a/crates/pipedream_mcp/src/outbound/api/test.rs+++ b/crates/pipedream_mcp/src/outbound/api/test.rs@@ -1,4 +1,10 @@-use super::{AppResponse, connectable_catalog_entry};+use super::{AppResponse, PipedreamClient, PipedreamConfig, connectable_catalog_entry};+use crate::domain::models::{MacroUserIdStr, PipedreamConnection};+use crate::domain::ports::ConnectedAppProxy;+use http::{HeaderMap, HeaderValue, Method, StatusCode};+use std::time::Duration;+use wiremock::matchers::{body_json, header, method, path, query_param};+use wiremock::{Mock, MockServer, ResponseTemplate};fn app(slug: &str, auth_type: Option<&str>) -> AppResponse {AppResponse {@@ -40,96 +46,67 @@ fn app_categories_are_optional_for_older_directory_responses() {}#[tokio::test]-async fn proxy_routes_the_connected_user_and_preserves_provider_status() {- use super::{- ApiProxy, CachedToken, PipedreamClient, PipedreamConfig, PipedreamConnection, ProxyMethod,- };- use axum::{- Router,- body::Bytes,- extract::OriginalUri,- http::{HeaderMap, Method, StatusCode},- routing::any,- };- use base64::Engine;- use macro_user_id::{cowlike::CowLike, user_id::MacroUserIdStr};- use std::{- sync::{Arc, Mutex},- time::{Duration, Instant},- };- let received = Arc::new(Mutex::new(Vec::new()));- let requests = received.clone();- let router = Router::new().fallback(any(- move |method: Method, uri: OriginalUri, headers: HeaderMap, body: Bytes| {- let requests = requests.clone();- async move {- requests- .lock()- .unwrap()- .push((method, uri.0, headers, body));- (- StatusCode::TOO_MANY_REQUESTS,- axum::Json(serde_json::json!({"error":"rate limited"})),- )- }- },- ));- let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();- let address = listener.local_addr().unwrap();- let server = tokio::spawn(async move {- axum::serve(listener, router).await.unwrap();- });+async fn the_proxy_addresses_the_upstream_url_for_the_connected_account() {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(path("/v1/oauth/token"))+ .respond_with(ResponseTemplate::new(200).set_body_json(+ serde_json::json!({"access_token": "project-token", "expires_in": 3600}),+ ))+ .mount(&server)+ .await;+ // base64url (no padding) of https://api.notion.com/v1/search+ Mock::given(method("POST"))+ .and(path(+ "/v1/connect/proj_fixture/proxy/aHR0cHM6Ly9hcGkubm90aW9uLmNvbS92MS9zZWFyY2g",+ ))+ .and(query_param("external_user_id", "macro|[email protected]"))+ .and(query_param("account_id", "apn_notion"))+ .and(header("authorization", "Bearer project-token"))+ .and(header("x-pd-environment", "development"))+ .and(header("x-pd-proxy-notion-version", "2026-03-11"))+ .and(body_json(serde_json::json!({"page_size": 100})))+ .respond_with(+ ResponseTemplate::new(429)+ .insert_header("retry-after", "3")+ .set_body_string(r#"{"object":"error","code":"rate_limited"}"#),+ )+ .expect(1)+ .mount(&server)+ .await;let client = PipedreamClient::new(PipedreamConfig {- client_id: "client".into(),- client_secret: "secret".into(),- project_id: "proj_test".into(),+ client_id: "fixture".into(),+ client_secret: "fixture".into(),+ project_id: "proj_fixture".into(),environment: "development".into(),allowed_origins: vec![],- api_url: format!("http://{address}"),- mcp_url: "https://unused.test".into(),+ api_url: server.uri(),+ mcp_url: format!("{}/mcp", server.uri()),}).unwrap();- *client.access_token.lock().unwrap() = Some(CachedToken {- token: "test-token".into(),- valid_until: Instant::now() + Duration::from_secs(3600),- });- let account = PipedreamConnection {- user_id: MacroUserIdStr::parse_from_str("macro|[email protected]")- .unwrap()- .into_owned(),- app_slug: "granola".into(),- server_name: "Granola".into(),- account_id: "apn_123".into(),+ let connection = PipedreamConnection {+ user_id: MacroUserIdStr::try_from_email("[email protected]").unwrap(),+ app_slug: "notion".into(),+ server_name: "Notion".into(),+ account_id: "apn_notion".into(),enabled: true,};- let target = "https://public-api.granola.ai/v1/webhook-endpoints";- let body = serde_json::json!({"url":"https://macro.test/webhook"});- let result = client- .proxy(&account, ProxyMethod::Post, target, Some(body.clone()))++ let response = client+ .proxy(+ &connection,+ Method::POST,+ "https://api.notion.com/v1/search",+ &HeaderMap::from_iter([(+ http::HeaderName::from_static("notion-version"),+ HeaderValue::from_static("2026-03-11"),+ )]),+ Some(&serde_json::json!({"page_size": 100})),+ ).await.unwrap();- assert_eq!(result.status, 429);- let requests = received.lock().unwrap();- let (method, uri, headers, bytes) = &requests[0];- assert_eq!(*method, Method::POST);- assert_eq!(- uri.path(),- format!(- "/v1/connect/proj_test/proxy/{}",- base64::engine::general_purpose::URL_SAFE_NO_PAD.encode(target)- )- );- let query: std::collections::HashMap<_, _> =- url::form_urlencoded::parse(uri.query().unwrap().as_bytes())- .into_owned()- .collect();- assert_eq!(query["external_user_id"], "macro|[email protected]");- assert_eq!(query["account_id"], "apn_123");- assert_eq!(headers["authorization"], "Bearer test-token");- assert_eq!(headers["x-pd-environment"], "development");- assert_eq!(- serde_json::from_slice::<serde_json::Value>(bytes).unwrap(),- body- );- server.abort();++ assert_eq!(response.status, StatusCode::TOO_MANY_REQUESTS);+ assert_eq!(response.retry_after(), Some(Duration::from_secs(3)));+ assert_eq!(response.body, r#"{"object":"error","code":"rate_limited"}"#);}diff --git a/rules/ast-grep/rust-no-transport-in-domain.yml b/rules/ast-grep/rust-no-transport-in-domain.ymlindex e759859d26..8d6e49733f 100644--- a/rules/ast-grep/rust-no-transport-in-domain.yml+++ b/rules/ast-grep/rust-no-transport-in-domain.yml@@ -21,6 +21,10 @@ ignores:# not transport leakage. What this rule exists to catch — IntoResponse# impls and status-code mapping — stays in that crate's inbound adapter.- 'crates/agent_egress/src/domain/**'+ # pipedream_mcp's Connect proxy port relays an HTTP request to a connected+ # app and hands back its status and headers: that request is the port's+ # subject matter, so `http` is its domain language here too.+ - 'crates/pipedream_mcp/src/domain/**'rule:any:- kind: scoped_identifier