Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(pipedream): report connect-flow outcomes via webhook (#5924)
Eric Hayesf1bee020991d
diff --git a/Cargo.lock b/Cargo.lockindex 5eef846ddc..0a981f9bbd 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -12485,6 +12485,7 @@ dependencies = ["sqlx","thiserror 2.0.18","tokio",+ "tower 0.5.3","tracing","url","utoipa",diff --git a/crates/pipedream_mcp/Cargo.toml b/crates/pipedream_mcp/Cargo.tomlindex f453a26902..f1bbdb549d 100644--- a/crates/pipedream_mcp/Cargo.toml+++ b/crates/pipedream_mcp/Cargo.toml@@ -29,3 +29,4 @@ workspace-hack = { version = "0.1", path = "../workspace-hack" }[dev-dependencies]tokio.workspace = true+tower = { workspace = true }diff --git a/crates/pipedream_mcp/src/inbound/axum_router.rs b/crates/pipedream_mcp/src/inbound/axum_router.rsindex 044ae79bc9..05735c055a 100644--- a/crates/pipedream_mcp/src/inbound/axum_router.rs+++ b/crates/pipedream_mcp/src/inbound/axum_router.rs@@ -21,6 +21,9 @@ use serde::{Deserialize, Serialize};use std::sync::Arc;use utoipa::{IntoParams, ToSchema};+#[cfg(test)]+mod test;+/// Hook invoked after a connect flow completes and the record is saved./// Hosts use this to react to a connection the moment it exists (e.g. start/// import gather jobs); implementations must be quick or spawn.@@ -138,6 +141,86 @@ where.with_state(state)}+/// The public Pipedream webhook route.+///+/// Pipedream posts connect-flow outcomes here, including the failures that+/// happen inside its hosted Connect UI — those never reach an authenticated+/// endpoint of ours, so without this the only evidence a flow died is a+/// minted token with no matching completion. Unauthenticated by necessity+/// (Pipedream is the caller), so the configured `webhook_uri` carries a+/// shared secret that every request is checked against.+///+/// Mounted outside the authenticated router, alongside the MCP OAuth+/// callback.+pub fn pipedream_webhook_router<Global>(secret: String) -> Router<Global>+where+ Global: Send + Sync,+{+ Router::new()+ .route("/pipedream/mcp/webhook", post(pipedream_webhook))+ .with_state(PipedreamWebhookState {+ secret: Arc::from(secret),+ })+}++/// State for the public webhook route: the shared secret, nothing else.+#[derive(Clone)]+pub struct PipedreamWebhookState {+ secret: Arc<str>,+}++/// Query parameters carried by the configured `webhook_uri`.+#[derive(Debug, Deserialize)]+pub struct PipedreamWebhookQuery {+ /// Shared secret, compared against the configured one.+ secret: Option<String>,+}++/// Record a connect-flow outcome reported by Pipedream.+///+/// The body is taken as raw JSON on purpose. The payload shape is+/// Pipedream's to change, and an endpoint whose whole job is telling us why+/// connections fail must not start dropping events with a 422 the day they+/// add a field. Known keys are pulled out so they can be indexed; the rest+/// is logged verbatim.+#[tracing::instrument(skip_all)]+async fn pipedream_webhook(+ State(state): State<PipedreamWebhookState>,+ Query(query): Query<PipedreamWebhookQuery>,+ Json(payload): Json<serde_json::Value>,+) -> StatusCode {+ if query.secret.as_deref() != Some(state.secret.as_ref()) {+ // Anyone can reach this route, so the payload stays out of the log.+ tracing::warn!("rejected Pipedream webhook with a bad or missing secret");+ return StatusCode::UNAUTHORIZED;+ }++ let event = payload.get("event").and_then(serde_json::Value::as_str);+ let external_user_id = payload+ .get("external_user_id")+ .and_then(serde_json::Value::as_str);+ let failed = payload.get("error").is_some()+ || event.is_some_and(|e| e.to_ascii_uppercase().contains("ERROR"));++ if failed {+ tracing::warn!(+ event = event,+ external_user_id = external_user_id,+ payload = %payload,+ "Pipedream connect flow failed"+ );+ } else {+ tracing::info!(+ event = event,+ external_user_id = external_user_id,+ payload = %payload,+ "Pipedream connect flow event"+ );+ }++ StatusCode::NO_CONTENT+}+// -- request / response types ------------------------------------------------/// Request body for updating a connected app.diff --git a/crates/pipedream_mcp/src/inbound/axum_router/test.rs b/crates/pipedream_mcp/src/inbound/axum_router/test.rsnew file mode 100644index 0000000000..2f3d74b53a--- /dev/null+++ b/crates/pipedream_mcp/src/inbound/axum_router/test.rs@@ -0,0 +1,52 @@+use super::pipedream_webhook_router;+use axum::{+ body::Body,+ http::{Request, StatusCode},+};+use tower::ServiceExt;++const SECRET: &str = "s3cret";+const SUCCESS_BODY: &str = r#"{"event":"CONNECTION_SUCCESS"}"#;++fn router() -> axum::Router {+ pipedream_webhook_router(SECRET.to_owned())+}++async fn post_webhook(uri: &str, body: &'static str) -> StatusCode {+ router()+ .oneshot(+ Request::builder()+ .method("POST")+ .uri(uri)+ .header("content-type", "application/json")+ .body(Body::from(body))+ .unwrap(),+ )+ .await+ .unwrap()+ .status()+}++#[tokio::test]+async fn webhook_rejects_missing_secret() {+ assert_eq!(+ post_webhook("/pipedream/mcp/webhook", SUCCESS_BODY).await,+ StatusCode::UNAUTHORIZED+ );+}++#[tokio::test]+async fn webhook_rejects_wrong_secret() {+ assert_eq!(+ post_webhook("/pipedream/mcp/webhook?secret=wrong", SUCCESS_BODY).await,+ StatusCode::UNAUTHORIZED+ );+}++#[tokio::test]+async fn webhook_accepts_matching_secret() {+ assert_eq!(+ post_webhook("/pipedream/mcp/webhook?secret=s3cret", SUCCESS_BODY).await,+ StatusCode::NO_CONTENT+ );+}diff --git a/crates/pipedream_mcp/src/inbound/mod.rs b/crates/pipedream_mcp/src/inbound/mod.rsindex 16683f0b08..706dfa0d6d 100644--- a/crates/pipedream_mcp/src/inbound/mod.rs+++ b/crates/pipedream_mcp/src/inbound/mod.rs@@ -1,4 +1,4 @@/// Axum HTTP adapter for Pipedream MCP connector management.pub mod axum_router;-pub use axum_router::{PipedreamRouterState, pipedream_mcp_router};+pub use axum_router::{PipedreamRouterState, pipedream_mcp_router, pipedream_webhook_router};diff --git a/crates/pipedream_mcp/src/outbound/api.rs b/crates/pipedream_mcp/src/outbound/api.rsindex 2c3a78797f..57caf04cb3 100644--- a/crates/pipedream_mcp/src/outbound/api.rs+++ b/crates/pipedream_mcp/src/outbound/api.rs@@ -56,6 +56,9 @@ pub struct PipedreamConfig {pub struct PipedreamClient {http: reqwest::Client,config: PipedreamConfig,+ /// Absolute URL Pipedream posts connect-flow outcomes to, minted into+ /// every Connect token. Unset when the host did not configure a webhook.+ webhook_uri: Option<String>,access_token: Mutex<Option<CachedToken>>,}@@ -76,10 +79,18 @@ impl PipedreamClient {Ok(Self {http,config,+ webhook_uri: None,access_token: Mutex::new(None),})}+ /// Mint connect tokens with this webhook so Pipedream reports in-UI+ /// connect-flow outcomes. Hosts that never mint tokens can skip this.+ pub fn with_webhook_uri(mut self, webhook_uri: impl Into<String>) -> Self {+ self.webhook_uri = Some(webhook_uri.into());+ self+ }+fn api(&self, path: &str) -> String {format!("{}/v1{path}", self.config.api_url.trim_end_matches('/'))}@@ -146,6 +157,9 @@ impl PipedreamConnect for PipedreamClient {if !self.config.allowed_origins.is_empty() {body["allowed_origins"] = serde_json::json!(self.config.allowed_origins);}+ if let Some(webhook_uri) = &self.webhook_uri {+ body["webhook_uri"] = serde_json::json!(webhook_uri);+ }let response = self.authed(self.http.post(self.connect_api("/tokens")))diff --git a/services/document_cognition_service/src/api.rs b/services/document_cognition_service/src/api.rsindex aa8554deba..7d59224b73 100644--- a/services/document_cognition_service/src/api.rs+++ b/services/document_cognition_service/src/api.rs@@ -156,8 +156,18 @@ fn api_router(api_context: ApiContext) -> Router {)).with_state(api_context.clone());- Router::new()+ // Pipedream calls this one itself, so it sits outside the authenticated+ // router and is guarded by the secret baked into the configured+ // `webhook_uri`. If the secret is unset, the route is not mounted.+ let app = Router::new().nest("/{version}", internal_router.clone()).merge(internal_router)- .merge(mcp_client::inbound::mcp_oauth_callback_router(mcp_state))+ .merge(mcp_client::inbound::mcp_oauth_callback_router(mcp_state));++ match api_context.config.pipedream_webhook_secret.value() {+ Some(secret) => app.merge(pipedream_mcp::inbound::pipedream_webhook_router(+ secret.to_owned(),+ )),+ None => app,+ }}diff --git a/services/document_cognition_service/src/config.rs b/services/document_cognition_service/src/config.rsindex eceb82012a..e9d34e469a 100644--- a/services/document_cognition_service/src/config.rs+++ b/services/document_cognition_service/src/config.rs@@ -52,6 +52,15 @@ maybe_env_vars!(/// by deploy environment: the app origin (`https://macro.com` //// `https://dev.macro.com`) plus localhost outside production.pub struct PipedreamAllowedOrigins;+ /// Absolute URL Pipedream posts connect-flow outcomes to, minted into+ /// every Connect token. Must carry the shared secret as a `secret` query+ /// parameter, matching `PIPEDREAM_WEBHOOK_SECRET`. When unset (or when+ /// the secret is unset), connect tokens are minted without a webhook.+ pub struct PipedreamWebhookUri;+ /// Shared secret guarding the public Pipedream webhook route, which is+ /// unauthenticated because Pipedream is the caller. When unset, the+ /// webhook route is not mounted.+ pub struct PipedreamWebhookSecret;);/// The configuration parameters for the application.@@ -105,6 +114,10 @@ pub struct Config {pub pipedream_mcp_url: PipedreamMcpUrl,/// Browser origins allowed to embed Pipedream's hosted Connect UI.pub pipedream_allowed_origins: PipedreamAllowedOrigins,+ /// URL Pipedream posts connect-flow outcomes to.+ pub pipedream_webhook_uri: PipedreamWebhookUri,+ /// Shared secret guarding the public Pipedream webhook route.+ pub pipedream_webhook_secret: PipedreamWebhookSecret,/// The internal api keypub internal_api_key: InternalApiKey,/// AI editing worker URL@@ -177,6 +190,8 @@ impl Config {pipedream_api_url: PipedreamApiUrl::Unset,pipedream_mcp_url: PipedreamMcpUrl::Unset,pipedream_allowed_origins: PipedreamAllowedOrigins::Unset,+ pipedream_webhook_uri: PipedreamWebhookUri::Unset,+ pipedream_webhook_secret: PipedreamWebhookSecret::Unset,internal_api_key: InternalApiKey::Comptime(""),ai_editing_worker_url: AiEditingWorkerUrl::unwrap_new().to_string(),mcp_public_url: default_mcp_public_url(Environment::Local).to_string(),diff --git a/services/document_cognition_service/src/main.rs b/services/document_cognition_service/src/main.rsindex 161f07bfde..aa662f42b1 100644--- a/services/document_cognition_service/src/main.rs+++ b/services/document_cognition_service/src/main.rs@@ -461,14 +461,29 @@ async fn main() -> anyhow::Result<()> {// The Pipedream MCP stack, fully separate from the native one above// (own endpoints, own table, own toolset). Without credentials its- // endpoints answer 501 and its toolsets come up empty.+ // endpoints answer 501 and its toolsets come up empty. Connect-flow+ // webhooks need both URI and secret; a half-set pair is ignored so we+ // never mint a callback we cannot serve.let pipedream_client: ai_tools::ToolPipedreamConnection = match (config.pipedream_client_id.value(),config.pipedream_client_secret.value(),config.pipedream_project_id.value(),) {- (Some(client_id), Some(client_secret), Some(project_id)) => Some(Arc::new(- pipedream_mcp::outbound::api::PipedreamClient::new(+ (Some(client_id), Some(client_secret), Some(project_id)) => {+ let webhook_uri = match (+ config.pipedream_webhook_uri.value(),+ config.pipedream_webhook_secret.value(),+ ) {+ (Some(uri), Some(_)) => Some(uri.to_owned()),+ (None, None) => None,+ _ => {+ tracing::warn!(+ "ignoring incomplete Pipedream webhook config; set both PIPEDREAM_WEBHOOK_URI and PIPEDREAM_WEBHOOK_SECRET"+ );+ None+ }+ };+ let client = pipedream_mcp::outbound::api::PipedreamClient::new(pipedream_mcp::outbound::api::PipedreamConfig {client_id: client_id.to_owned(),client_secret: client_secret.to_owned(),@@ -508,8 +523,12 @@ async fn main() -> anyhow::Result<()> {},},)- .context("failed to build Pipedream client")?,- )),+ .context("failed to build Pipedream client")?;+ Some(Arc::new(match webhook_uri {+ Some(uri) => client.with_webhook_uri(uri),+ None => client,+ }))+ }_ => {tracing::info!("Pipedream credentials not set; Pipedream MCP connectors disabled");Nonediff --git a/tooling/xtask/crates/xtask_local/src/local/local_env.rs b/tooling/xtask/crates/xtask_local/src/local/local_env.rsindex e0b92d5f1c..9728a138da 100644--- a/tooling/xtask/crates/xtask_local/src/local/local_env.rs+++ b/tooling/xtask/crates/xtask_local/src/local/local_env.rs@@ -507,6 +507,15 @@ impl BootStubEnv {);// search_processing_service; the local cluster has the security plugin// disabled so these are accepted but ignored (same as opensearch.rs).+ // document_cognition_service mounts the Pipedream webhook when both+ // values are set. Nothing local can receive Pipedream's callbacks, so+ // these only need to be well-formed: the URI must carry the same+ // secret the route checks against.+ env.insert(+ "PIPEDREAM_WEBHOOK_URI".into(),+ "http://localhost:8080/pipedream/mcp/webhook?secret=local".into(),+ );+ env.insert("PIPEDREAM_WEBHOOK_SECRET".into(), "local".into());env.insert("OPENSEARCH_USERNAME".into(), "macrouser".into());env.insert("OPENSEARCH_PASSWORD".into(), "local".into());// document_storage_service's presigned-URL config. Locally the