Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(pipedream): report connect-flow outcomes via webhook (#5924)

Eric Hayesf1bee020991d
diff --git a/Cargo.lock b/Cargo.lock
index 5eef846ddc..0a981f9bbd 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -12485,6 +12485,7 @@ dependencies = [
"sqlx",
"thiserror 2.0.18",
"tokio",
+ "tower 0.5.3",
"tracing",
"url",
"utoipa",
diff --git a/crates/pipedream_mcp/Cargo.toml b/crates/pipedream_mcp/Cargo.toml
index f453a26902..f1bbdb549d 100644
--- a/crates/pipedream_mcp/Cargo.toml
+++ b/crates/pipedream_mcp/Cargo.toml
@@ -29,3 +29,4 @@ workspace-hack = { version = "0.1", path = "../workspace-hack" }
[dev-dependencies]
tokio.workspace = true
+tower = { workspace = true }
diff --git a/crates/pipedream_mcp/src/inbound/axum_router.rs b/crates/pipedream_mcp/src/inbound/axum_router.rs
index 044ae79bc9..05735c055a 100644
--- a/crates/pipedream_mcp/src/inbound/axum_router.rs
+++ b/crates/pipedream_mcp/src/inbound/axum_router.rs
@@ -21,6 +21,9 @@ use serde::{Deserialize, Serialize};
use std::sync::Arc;
use utoipa::{IntoParams, ToSchema};
+#[cfg(test)]
+mod test;
+
/// Hook invoked after a connect flow completes and the record is saved.
/// Hosts use this to react to a connection the moment it exists (e.g. start
/// import gather jobs); implementations must be quick or spawn.
@@ -138,6 +141,86 @@ where
.with_state(state)
}
+/// The public Pipedream webhook route.
+///
+/// Pipedream posts connect-flow outcomes here, including the failures that
+/// happen inside its hosted Connect UI — those never reach an authenticated
+/// endpoint of ours, so without this the only evidence a flow died is a
+/// minted token with no matching completion. Unauthenticated by necessity
+/// (Pipedream is the caller), so the configured `webhook_uri` carries a
+/// shared secret that every request is checked against.
+///
+/// Mounted outside the authenticated router, alongside the MCP OAuth
+/// callback.
+pub fn pipedream_webhook_router<Global>(secret: String) -> Router<Global>
+where
+ Global: Send + Sync,
+{
+ Router::new()
+ .route("/pipedream/mcp/webhook", post(pipedream_webhook))
+ .with_state(PipedreamWebhookState {
+ secret: Arc::from(secret),
+ })
+}
+
+/// State for the public webhook route: the shared secret, nothing else.
+#[derive(Clone)]
+pub struct PipedreamWebhookState {
+ secret: Arc<str>,
+}
+
+/// Query parameters carried by the configured `webhook_uri`.
+#[derive(Debug, Deserialize)]
+pub struct PipedreamWebhookQuery {
+ /// Shared secret, compared against the configured one.
+ secret: Option<String>,
+}
+
+/// Record a connect-flow outcome reported by Pipedream.
+///
+/// The body is taken as raw JSON on purpose. The payload shape is
+/// Pipedream's to change, and an endpoint whose whole job is telling us why
+/// connections fail must not start dropping events with a 422 the day they
+/// add a field. Known keys are pulled out so they can be indexed; the rest
+/// is logged verbatim.
+#[tracing::instrument(skip_all)]
+async fn pipedream_webhook(
+ State(state): State<PipedreamWebhookState>,
+ Query(query): Query<PipedreamWebhookQuery>,
+ Json(payload): Json<serde_json::Value>,
+) -> StatusCode {
+ if query.secret.as_deref() != Some(state.secret.as_ref()) {
+ // Anyone can reach this route, so the payload stays out of the log.
+ tracing::warn!("rejected Pipedream webhook with a bad or missing secret");
+ return StatusCode::UNAUTHORIZED;
+ }
+
+ let event = payload.get("event").and_then(serde_json::Value::as_str);
+ let external_user_id = payload
+ .get("external_user_id")
+ .and_then(serde_json::Value::as_str);
+ let failed = payload.get("error").is_some()
+ || event.is_some_and(|e| e.to_ascii_uppercase().contains("ERROR"));
+
+ if failed {
+ tracing::warn!(
+ event = event,
+ external_user_id = external_user_id,
+ payload = %payload,
+ "Pipedream connect flow failed"
+ );
+ } else {
+ tracing::info!(
+ event = event,
+ external_user_id = external_user_id,
+ payload = %payload,
+ "Pipedream connect flow event"
+ );
+ }
+
+ StatusCode::NO_CONTENT
+}
+
// -- request / response types ------------------------------------------------
/// Request body for updating a connected app.
diff --git a/crates/pipedream_mcp/src/inbound/axum_router/test.rs b/crates/pipedream_mcp/src/inbound/axum_router/test.rs
new file mode 100644
index 0000000000..2f3d74b53a
--- /dev/null
+++ b/crates/pipedream_mcp/src/inbound/axum_router/test.rs
@@ -0,0 +1,52 @@
+use super::pipedream_webhook_router;
+use axum::{
+ body::Body,
+ http::{Request, StatusCode},
+};
+use tower::ServiceExt;
+
+const SECRET: &str = "s3cret";
+const SUCCESS_BODY: &str = r#"{"event":"CONNECTION_SUCCESS"}"#;
+
+fn router() -> axum::Router {
+ pipedream_webhook_router(SECRET.to_owned())
+}
+
+async fn post_webhook(uri: &str, body: &'static str) -> StatusCode {
+ router()
+ .oneshot(
+ Request::builder()
+ .method("POST")
+ .uri(uri)
+ .header("content-type", "application/json")
+ .body(Body::from(body))
+ .unwrap(),
+ )
+ .await
+ .unwrap()
+ .status()
+}
+
+#[tokio::test]
+async fn webhook_rejects_missing_secret() {
+ assert_eq!(
+ post_webhook("/pipedream/mcp/webhook", SUCCESS_BODY).await,
+ StatusCode::UNAUTHORIZED
+ );
+}
+
+#[tokio::test]
+async fn webhook_rejects_wrong_secret() {
+ assert_eq!(
+ post_webhook("/pipedream/mcp/webhook?secret=wrong", SUCCESS_BODY).await,
+ StatusCode::UNAUTHORIZED
+ );
+}
+
+#[tokio::test]
+async fn webhook_accepts_matching_secret() {
+ assert_eq!(
+ post_webhook("/pipedream/mcp/webhook?secret=s3cret", SUCCESS_BODY).await,
+ StatusCode::NO_CONTENT
+ );
+}
diff --git a/crates/pipedream_mcp/src/inbound/mod.rs b/crates/pipedream_mcp/src/inbound/mod.rs
index 16683f0b08..706dfa0d6d 100644
--- a/crates/pipedream_mcp/src/inbound/mod.rs
+++ b/crates/pipedream_mcp/src/inbound/mod.rs
@@ -1,4 +1,4 @@
/// Axum HTTP adapter for Pipedream MCP connector management.
pub mod axum_router;
-pub use axum_router::{PipedreamRouterState, pipedream_mcp_router};
+pub use axum_router::{PipedreamRouterState, pipedream_mcp_router, pipedream_webhook_router};
diff --git a/crates/pipedream_mcp/src/outbound/api.rs b/crates/pipedream_mcp/src/outbound/api.rs
index 2c3a78797f..57caf04cb3 100644
--- a/crates/pipedream_mcp/src/outbound/api.rs
+++ b/crates/pipedream_mcp/src/outbound/api.rs
@@ -56,6 +56,9 @@ pub struct PipedreamConfig {
pub struct PipedreamClient {
http: reqwest::Client,
config: PipedreamConfig,
+ /// Absolute URL Pipedream posts connect-flow outcomes to, minted into
+ /// every Connect token. Unset when the host did not configure a webhook.
+ webhook_uri: Option<String>,
access_token: Mutex<Option<CachedToken>>,
}
@@ -76,10 +79,18 @@ impl PipedreamClient {
Ok(Self {
http,
config,
+ webhook_uri: None,
access_token: Mutex::new(None),
})
}
+ /// Mint connect tokens with this webhook so Pipedream reports in-UI
+ /// connect-flow outcomes. Hosts that never mint tokens can skip this.
+ pub fn with_webhook_uri(mut self, webhook_uri: impl Into<String>) -> Self {
+ self.webhook_uri = Some(webhook_uri.into());
+ self
+ }
+
fn api(&self, path: &str) -> String {
format!("{}/v1{path}", self.config.api_url.trim_end_matches('/'))
}
@@ -146,6 +157,9 @@ impl PipedreamConnect for PipedreamClient {
if !self.config.allowed_origins.is_empty() {
body["allowed_origins"] = serde_json::json!(self.config.allowed_origins);
}
+ if let Some(webhook_uri) = &self.webhook_uri {
+ body["webhook_uri"] = serde_json::json!(webhook_uri);
+ }
let response = self
.authed(self.http.post(self.connect_api("/tokens")))
diff --git a/services/document_cognition_service/src/api.rs b/services/document_cognition_service/src/api.rs
index aa8554deba..7d59224b73 100644
--- a/services/document_cognition_service/src/api.rs
+++ b/services/document_cognition_service/src/api.rs
@@ -156,8 +156,18 @@ fn api_router(api_context: ApiContext) -> Router {
))
.with_state(api_context.clone());
- Router::new()
+ // Pipedream calls this one itself, so it sits outside the authenticated
+ // router and is guarded by the secret baked into the configured
+ // `webhook_uri`. If the secret is unset, the route is not mounted.
+ let app = Router::new()
.nest("/{version}", internal_router.clone())
.merge(internal_router)
- .merge(mcp_client::inbound::mcp_oauth_callback_router(mcp_state))
+ .merge(mcp_client::inbound::mcp_oauth_callback_router(mcp_state));
+
+ match api_context.config.pipedream_webhook_secret.value() {
+ Some(secret) => app.merge(pipedream_mcp::inbound::pipedream_webhook_router(
+ secret.to_owned(),
+ )),
+ None => app,
+ }
}
diff --git a/services/document_cognition_service/src/config.rs b/services/document_cognition_service/src/config.rs
index eceb82012a..e9d34e469a 100644
--- a/services/document_cognition_service/src/config.rs
+++ b/services/document_cognition_service/src/config.rs
@@ -52,6 +52,15 @@ maybe_env_vars!(
/// by deploy environment: the app origin (`https://macro.com` /
/// `https://dev.macro.com`) plus localhost outside production.
pub struct PipedreamAllowedOrigins;
+ /// Absolute URL Pipedream posts connect-flow outcomes to, minted into
+ /// every Connect token. Must carry the shared secret as a `secret` query
+ /// parameter, matching `PIPEDREAM_WEBHOOK_SECRET`. When unset (or when
+ /// the secret is unset), connect tokens are minted without a webhook.
+ pub struct PipedreamWebhookUri;
+ /// Shared secret guarding the public Pipedream webhook route, which is
+ /// unauthenticated because Pipedream is the caller. When unset, the
+ /// webhook route is not mounted.
+ pub struct PipedreamWebhookSecret;
);
/// The configuration parameters for the application.
@@ -105,6 +114,10 @@ pub struct Config {
pub pipedream_mcp_url: PipedreamMcpUrl,
/// Browser origins allowed to embed Pipedream's hosted Connect UI.
pub pipedream_allowed_origins: PipedreamAllowedOrigins,
+ /// URL Pipedream posts connect-flow outcomes to.
+ pub pipedream_webhook_uri: PipedreamWebhookUri,
+ /// Shared secret guarding the public Pipedream webhook route.
+ pub pipedream_webhook_secret: PipedreamWebhookSecret,
/// The internal api key
pub internal_api_key: InternalApiKey,
/// AI editing worker URL
@@ -177,6 +190,8 @@ impl Config {
pipedream_api_url: PipedreamApiUrl::Unset,
pipedream_mcp_url: PipedreamMcpUrl::Unset,
pipedream_allowed_origins: PipedreamAllowedOrigins::Unset,
+ pipedream_webhook_uri: PipedreamWebhookUri::Unset,
+ pipedream_webhook_secret: PipedreamWebhookSecret::Unset,
internal_api_key: InternalApiKey::Comptime(""),
ai_editing_worker_url: AiEditingWorkerUrl::unwrap_new().to_string(),
mcp_public_url: default_mcp_public_url(Environment::Local).to_string(),
diff --git a/services/document_cognition_service/src/main.rs b/services/document_cognition_service/src/main.rs
index 161f07bfde..aa662f42b1 100644
--- a/services/document_cognition_service/src/main.rs
+++ b/services/document_cognition_service/src/main.rs
@@ -461,14 +461,29 @@ async fn main() -> anyhow::Result<()> {
// The Pipedream MCP stack, fully separate from the native one above
// (own endpoints, own table, own toolset). Without credentials its
- // endpoints answer 501 and its toolsets come up empty.
+ // endpoints answer 501 and its toolsets come up empty. Connect-flow
+ // webhooks need both URI and secret; a half-set pair is ignored so we
+ // never mint a callback we cannot serve.
let pipedream_client: ai_tools::ToolPipedreamConnection = match (
config.pipedream_client_id.value(),
config.pipedream_client_secret.value(),
config.pipedream_project_id.value(),
) {
- (Some(client_id), Some(client_secret), Some(project_id)) => Some(Arc::new(
- pipedream_mcp::outbound::api::PipedreamClient::new(
+ (Some(client_id), Some(client_secret), Some(project_id)) => {
+ let webhook_uri = match (
+ config.pipedream_webhook_uri.value(),
+ config.pipedream_webhook_secret.value(),
+ ) {
+ (Some(uri), Some(_)) => Some(uri.to_owned()),
+ (None, None) => None,
+ _ => {
+ tracing::warn!(
+ "ignoring incomplete Pipedream webhook config; set both PIPEDREAM_WEBHOOK_URI and PIPEDREAM_WEBHOOK_SECRET"
+ );
+ None
+ }
+ };
+ let client = pipedream_mcp::outbound::api::PipedreamClient::new(
pipedream_mcp::outbound::api::PipedreamConfig {
client_id: client_id.to_owned(),
client_secret: client_secret.to_owned(),
@@ -508,8 +523,12 @@ async fn main() -> anyhow::Result<()> {
},
},
)
- .context("failed to build Pipedream client")?,
- )),
+ .context("failed to build Pipedream client")?;
+ Some(Arc::new(match webhook_uri {
+ Some(uri) => client.with_webhook_uri(uri),
+ None => client,
+ }))
+ }
_ => {
tracing::info!("Pipedream credentials not set; Pipedream MCP connectors disabled");
None
diff --git a/tooling/xtask/crates/xtask_local/src/local/local_env.rs b/tooling/xtask/crates/xtask_local/src/local/local_env.rs
index e0b92d5f1c..9728a138da 100644
--- a/tooling/xtask/crates/xtask_local/src/local/local_env.rs
+++ b/tooling/xtask/crates/xtask_local/src/local/local_env.rs
@@ -507,6 +507,15 @@ impl BootStubEnv {
);
// search_processing_service; the local cluster has the security plugin
// disabled so these are accepted but ignored (same as opensearch.rs).
+ // document_cognition_service mounts the Pipedream webhook when both
+ // values are set. Nothing local can receive Pipedream's callbacks, so
+ // these only need to be well-formed: the URI must carry the same
+ // secret the route checks against.
+ env.insert(
+ "PIPEDREAM_WEBHOOK_URI".into(),
+ "http://localhost:8080/pipedream/mcp/webhook?secret=local".into(),
+ );
+ env.insert("PIPEDREAM_WEBHOOK_SECRET".into(), "local".into());
env.insert("OPENSEARCH_USERNAME".into(), "macrouser".into());
env.insert("OPENSEARCH_PASSWORD".into(), "local".into());
// document_storage_service's presigned-URL config. Locally the