Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(import): try improve slack import (#5228)

teocd7e0bf94c4b
diff --git a/crates/import/src/domain/service.rs b/crates/import/src/domain/service.rs
index 67d2588378..8e7a516286 100644
--- a/crates/import/src/domain/service.rs
+++ b/crates/import/src/domain/service.rs
@@ -62,6 +62,13 @@ const NOTION_PAGE_IMPORT_TIMEOUT: Duration = Duration::from_secs(120);
/// enough to stay polite to Notion's MCP.
const NOTION_IMPORT_CONCURRENCY: usize = 4;
+/// How many channels the deterministic Slack gather stages, mirroring the
+/// "8-15 strong candidates" the agent prompt asks for.
+const SLACK_GATHER_MAX_CHANNELS: usize = 15;
+/// How many channel-search pages the deterministic Slack gather follows
+/// before staging what it has.
+const SLACK_GATHER_MAX_PAGES: usize = 5;
+
/// How often a running import batch touches its rows' `updated_at`. The
/// heartbeat covers every row the batch owns — queued AND in-flight — so a
/// fresh `updated_at` means "a live process is still responsible for this
@@ -443,6 +450,20 @@ where
source: ImportSource,
) -> anyhow::Result<()> {
let mcp_tools = self.connector_tools(user, source).await?;
+
+ // Slack discovery is a listing problem, not a language problem:
+ // enumerate channels through the connector directly and stage the
+ // strongest. The agent session only runs as a fallback, when the
+ // connector's tool surface changed under us.
+ if source == ImportSource::Slack {
+ match self.gather_slack_direct(user, &mcp_tools).await {
+ Ok(_) => return Ok(()),
+ Err(e) => {
+ tracing::warn!(error = ?e, "direct slack gather failed; trying the agent");
+ }
+ }
+ }
+
let native = gather_toolset::<Self>();
let toolset = NativePlusMcp::new(native, mcp_tools);
let context = ImportToolContext {
@@ -523,6 +544,116 @@ where
Ok(())
}
+ /// Call one connector tool and surface both dispatch and tool errors as
+ /// plain errors.
+ async fn connector_tool_call(
+ &self,
+ user: &MacroUserIdStr<'static>,
+ mcp_tools: &McpToolSet,
+ tool_name: &str,
+ arguments: &serde_json::Value,
+ ) -> anyhow::Result<serde_json::Value> {
+ ToolSet::<()>::try_tool_call(
+ mcp_tools,
+ (),
+ RequestContext::new(user.clone()),
+ tool_name,
+ arguments,
+ )
+ .await
+ .map_err(|e| anyhow::anyhow!("{tool_name} dispatch failed: {e}"))?
+ .map_err(|e| anyhow::anyhow!("{tool_name} failed: {}", e.description))
+ }
+
+ /// Stage the user's Slack channels WITHOUT a model: call the connector's
+ /// channel-search tool directly (an empty query lists every channel the
+ /// connected user can see), follow pagination, and stage the strongest
+ /// candidates. The agent-driven gather routinely finished "successfully"
+ /// having staged nothing; a direct call either produces channels or a
+ /// real error. Staged rows carry no participants — inviting teammates
+ /// stays best-effort and must never block discovery.
+ #[tracing::instrument(skip(self, user, mcp_tools), err)]
+ async fn gather_slack_direct(
+ &self,
+ user: &MacroUserIdStr<'static>,
+ mcp_tools: &McpToolSet,
+ ) -> anyhow::Result<usize> {
+ let search_tool = slack_channel_search_tool_name(mcp_tools)
+ .ok_or_else(|| anyhow::anyhow!("connector exposes no channel-search tool"))?;
+
+ let mut channels: Vec<SlackChannelCandidate> = Vec::new();
+ let mut cursor: Option<String> = None;
+ for page in 0..SLACK_GATHER_MAX_PAGES {
+ let mut arguments = serde_json::json!({ "query": "" });
+ if let Some(cursor) = &cursor {
+ arguments["cursor"] = serde_json::Value::String(cursor.clone());
+ }
+ let result = match self
+ .connector_tool_call(user, mcp_tools, &search_tool, &arguments)
+ .await
+ {
+ Ok(result) => Ok(result),
+ // Some servers reject an explicit empty query; try the first
+ // page again without one before giving up on the direct path.
+ Err(empty_query_error) if page == 0 => self
+ .connector_tool_call(user, mcp_tools, &search_tool, &serde_json::json!({}))
+ .await
+ .map_err(|no_query_error| {
+ anyhow::anyhow!(
+ "channel search failed with an empty query ({empty_query_error}) \
+ and without one ({no_query_error})"
+ )
+ }),
+ Err(e) => {
+ // Later pages are best-effort: keep what earlier pages
+ // already produced.
+ tracing::warn!(page, error = ?e, "channel search page failed");
+ break;
+ }
+ }?;
+
+ let parsed = parse_slack_channel_page(result);
+ channels.extend(parsed.channels);
+ cursor = parsed.next_cursor;
+ if cursor.is_none() {
+ break;
+ }
+ }
+ anyhow::ensure!(!channels.is_empty(), "channel search returned no channels");
+
+ let enumerated = channels.len();
+ let mut staged = 0usize;
+ for channel in select_slack_candidates(channels, SLACK_GATHER_MAX_CHANNELS) {
+ let foreign_id = channel.id.clone().unwrap_or_else(|| channel.name.clone());
+ let metadata = serde_json::json!({
+ "name": channel.name,
+ "channel_id": channel.id,
+ "purpose": channel.purpose,
+ "participants": [],
+ });
+ match self
+ .stage(
+ user,
+ Initiator::Onboarding,
+ ImportSource::Slack,
+ &foreign_id,
+ metadata,
+ )
+ .await
+ {
+ Ok(StageOutcome::Staged(_)) => staged += 1,
+ // Already imported (by the user or a teammate), declined, or
+ // mid-import — the ledger already covers it.
+ Ok(_) => {}
+ Err(e) => {
+ tracing::warn!(channel = %channel.name, error = ?e, "failed to stage slack channel");
+ }
+ }
+ }
+ tracing::info!(enumerated, staged, "direct slack gather finished");
+ Ok(staged)
+ }
+
/// Fallback: run a single-page Haiku session over the shared connector
/// tools. Rows the agent fails to finalize are handled by the caller.
#[tracing::instrument(skip(self, user, mcp_tools, rows), fields(pages = rows.len()), err)]
@@ -1363,6 +1494,185 @@ fn is_notion_fetch_tool_name(name: &str) -> bool {
)
}
+/// The mangled name of the connector's channel-search tool. Slack's hosted
+/// MCP has shipped several tool-name spellings, so this matches the shape —
+/// a search/list over channels — rather than one literal name.
+fn slack_channel_search_tool_name(mcp_tools: &McpToolSet) -> Option<String> {
+ ToolSet::<()>::request_schemas(mcp_tools)?
+ .into_iter()
+ .map(|schema| schema.name)
+ .find(|name| is_slack_channel_search_tool_name(name))
+}
+
+fn is_slack_channel_search_tool_name(name: &str) -> bool {
+ let Some((_, tool)) = name.rsplit_once("__") else {
+ return false;
+ };
+ let tool = tool.to_ascii_lowercase().replace('-', "_");
+ let channel_noun = tool.contains("channel") || tool.contains("conversation");
+ let listing_verb = tool.contains("search") || tool.contains("list");
+ let other_surface = [
+ "member", "history", "message", "canvas", "create", "user", "emoji", "file",
+ ]
+ .iter()
+ .any(|word| tool.contains(word));
+ channel_noun && listing_verb && !other_surface
+}
+
+/// One channel parsed out of a Slack channel-search result.
+#[derive(Debug, Clone, PartialEq)]
+struct SlackChannelCandidate {
+ id: Option<String>,
+ name: String,
+ purpose: Option<String>,
+ member_count: Option<u64>,
+ archived: bool,
+}
+
+/// One page of channel-search results.
+#[derive(Debug, Default, PartialEq)]
+struct SlackChannelPage {
+ channels: Vec<SlackChannelCandidate>,
+ next_cursor: Option<String>,
+}
+
+/// Split a channel-search result into channels plus a pagination cursor.
+/// Handles structured MCP output, JSON re-encoded as text, a bare array of
+/// channels, and the channel list hiding under common wrapper keys.
+fn parse_slack_channel_page(result: serde_json::Value) -> SlackChannelPage {
+ match result {
+ serde_json::Value::String(text) => match serde_json::from_str::<serde_json::Value>(&text) {
+ Ok(structured @ (serde_json::Value::Object(_) | serde_json::Value::Array(_))) => {
+ parse_slack_channel_page(structured)
+ }
+ _ => SlackChannelPage::default(),
+ },
+ serde_json::Value::Array(items) => SlackChannelPage {
+ channels: items.iter().filter_map(parse_slack_channel).collect(),
+ next_cursor: None,
+ },
+ serde_json::Value::Object(map) => {
+ let next_cursor = slack_next_cursor(&map);
+ for key in ["channels", "results", "items", "matches", "data"] {
+ if let Some(value) = map.get(key) {
+ let inner = parse_slack_channel_page(value.clone());
+ if !inner.channels.is_empty() {
+ return SlackChannelPage {
+ channels: inner.channels,
+ next_cursor: inner.next_cursor.or(next_cursor),
+ };
+ }
+ }
+ }
+ // A single channel object at the top level.
+ let channels = parse_slack_channel(&serde_json::Value::Object(map))
+ .into_iter()
+ .collect();
+ SlackChannelPage {
+ channels,
+ next_cursor,
+ }
+ }
+ _ => SlackChannelPage::default(),
+ }
+}
+
+fn slack_next_cursor(map: &serde_json::Map<String, serde_json::Value>) -> Option<String> {
+ map.get("next_cursor")
+ .or_else(|| map.get("nextCursor"))
+ .or_else(|| map.get("cursor"))
+ .or_else(|| {
+ map.get("response_metadata")
+ .and_then(|value| value.as_object())
+ .and_then(|metadata| metadata.get("next_cursor"))
+ })
+ .and_then(|value| value.as_str())
+ .map(str::trim)
+ .filter(|cursor| !cursor.is_empty())
+ .map(str::to_string)
+}
+
+fn parse_slack_channel(value: &serde_json::Value) -> Option<SlackChannelCandidate> {
+ let map = value.as_object()?;
+ // DMs and group DMs are not importable channels.
+ if ["is_im", "is_mpim"]
+ .iter()
+ .any(|key| map.get(*key).and_then(serde_json::Value::as_bool) == Some(true))
+ {
+ return None;
+ }
+ let name = map
+ .get("name")
+ .or_else(|| map.get("channel_name"))
+ .and_then(|value| value.as_str())
+ .map(|name| name.trim().trim_start_matches('#'))
+ .filter(|name| !name.is_empty())?
+ .to_string();
+ let id = map
+ .get("id")
+ .or_else(|| map.get("channel_id"))
+ .and_then(|value| value.as_str())
+ .map(str::trim)
+ .filter(|id| !id.is_empty())
+ .map(str::to_string);
+ let purpose = slack_channel_text(map, "purpose")
+ .or_else(|| slack_channel_text(map, "topic"))
+ .or_else(|| slack_channel_text(map, "description"));
+ let member_count = map
+ .get("member_count")
+ .or_else(|| map.get("num_members"))
+ .and_then(serde_json::Value::as_u64);
+ let archived = map.get("is_archived").and_then(serde_json::Value::as_bool) == Some(true);
+ Some(SlackChannelCandidate {
+ id,
+ name,
+ purpose,
+ member_count,
+ archived,
+ })
+}
+
+/// Slack renders purpose/topic either as a plain string or as the classic
+/// API's `{ "value": "…" }` wrapper.
+fn slack_channel_text(
+ map: &serde_json::Map<String, serde_json::Value>,
+ key: &str,
+) -> Option<String> {
+ let value = map.get(key)?;
+ value
+ .as_str()
+ .or_else(|| value.get("value").and_then(|nested| nested.as_str()))
+ .map(str::trim)
+ .filter(|text| !text.is_empty())
+ .map(str::to_string)
+}
+
+/// Rank enumerated channels and keep the strongest `cap`: drop archived
+/// channels, dedupe by id (falling back to the name), prefer larger channels
+/// (member count is the best activity proxy a listing offers), and keep the
+/// listing order among ties.
+fn select_slack_candidates(
+ channels: Vec<SlackChannelCandidate>,
+ cap: usize,
+) -> Vec<SlackChannelCandidate> {
+ let mut seen = HashSet::new();
+ let mut candidates: Vec<SlackChannelCandidate> = channels
+ .into_iter()
+ .filter(|channel| !channel.archived)
+ .filter(|channel| {
+ seen.insert(
+ channel
+ .id
+ .clone()
+ .unwrap_or_else(|| format!("#{}", channel.name.to_ascii_lowercase())),
+ )
+ })
+ .collect();
+ candidates.sort_by_key(|channel| std::cmp::Reverse(channel.member_count.unwrap_or(0)));
+ candidates.truncate(cap);
+ candidates
+}
+
/// Split a Notion fetch result into its title, Markdown body, and properties.
/// Structured MCP output and JSON text may use either Notion's `markdown`
/// field or the generic MCP fetch `text` field.
diff --git a/crates/import/src/domain/service/test.rs b/crates/import/src/domain/service/test.rs
index ed8f625f61..0f8da61cdf 100644
--- a/crates/import/src/domain/service/test.rs
+++ b/crates/import/src/domain/service/test.rs
@@ -124,6 +124,175 @@ fn slack_gather_lists_channels_with_an_explicit_empty_query() {
assert!(prompt.contains("Participant details are optional"));
}
+#[test]
+fn slack_channel_search_tool_names_match_by_shape() {
+ assert!(is_slack_channel_search_tool_name(
+ "mcp__Slack__search_channels"
+ ));
+ assert!(is_slack_channel_search_tool_name(
+ "mcp__Slack__slack_search_channels"
+ ));
+ assert!(is_slack_channel_search_tool_name(
+ "mcp__Slack__search-channels"
+ ));
+ assert!(is_slack_channel_search_tool_name(
+ "mcp__Slack__list_channels"
+ ));
+ assert!(is_slack_channel_search_tool_name(
+ "mcp__Slack__conversations_list"
+ ));
+
+ // Other channel-adjacent surfaces must not be mistaken for the listing.
+ assert!(!is_slack_channel_search_tool_name(
+ "mcp__Slack__list_channel_members"
+ ));
+ assert!(!is_slack_channel_search_tool_name(
+ "mcp__Slack__search_messages_and_files"
+ ));
+ assert!(!is_slack_channel_search_tool_name(
+ "mcp__Slack__get_channel_history"
+ ));
+ assert!(!is_slack_channel_search_tool_name(
+ "mcp__Slack__create_channel"
+ ));
+ assert!(!is_slack_channel_search_tool_name(
+ "mcp__Slack__search_users"
+ ));
+ assert!(!is_slack_channel_search_tool_name("search_channels")); // unmangled
+}
+
+#[test]
+fn slack_channel_page_parses_structured_results_with_cursors() {
+ let page = parse_slack_channel_page(serde_json::json!({
+ "channels": [
+ {
+ "id": "C0123456789",
+ "name": "engineering",
+ "purpose": { "value": "Build the thing" },
+ "member_count": 42,
+ "is_archived": false,
+ },
+ {
+ "id": "C0000000001",
+ "name": "#design",
+ "topic": "Make it pretty",
+ "num_members": 7,
+ },
+ { "id": "D0123456789", "name": "dm", "is_im": true },
+ { "id": "C0000000002", "name": "old-stuff", "is_archived": true },
+ ],
+ "response_metadata": { "next_cursor": "cursor-2" },
+ }));
+
+ assert_eq!(page.next_cursor.as_deref(), Some("cursor-2"));
+ assert_eq!(
+ page.channels,
+ vec![
+ SlackChannelCandidate {
+ id: Some("C0123456789".into()),
+ name: "engineering".into(),
+ purpose: Some("Build the thing".into()),
+ member_count: Some(42),
+ archived: false,
+ },
+ SlackChannelCandidate {
+ id: Some("C0000000001".into()),
+ name: "design".into(),
+ purpose: Some("Make it pretty".into()),
+ member_count: Some(7),
+ archived: false,
+ },
+ SlackChannelCandidate {
+ id: Some("C0000000002".into()),
+ name: "old-stuff".into(),
+ purpose: None,
+ member_count: None,
+ archived: true,
+ },
+ ]
+ );
+}
+
+#[test]
+fn slack_channel_page_parses_json_text_and_bare_arrays() {
+ // Results without structured content arrive as JSON re-encoded as text.
+ let page = parse_slack_channel_page(serde_json::Value::String(
+ r#"{"results": [{"id": "C1", "name": "general"}], "next_cursor": "abc"}"#.to_string(),
+ ));
+ assert_eq!(page.channels.len(), 1);
+ assert_eq!(page.channels[0].name, "general");
+ assert_eq!(page.next_cursor.as_deref(), Some("abc"));
+
+ let page = parse_slack_channel_page(serde_json::json!([
+ { "id": "C1", "name": "general" },
+ { "id": "C2", "name": "random" },
+ ]));
+ assert_eq!(page.channels.len(), 2);
+ assert_eq!(page.next_cursor, None);
+
+ // Plain prose is not a channel list.
+ let page = parse_slack_channel_page(serde_json::Value::String("No channels found.".into()));
+ assert_eq!(page, SlackChannelPage::default());
+
+ // An empty cursor string means "no more pages", not a page named "".
+ let page = parse_slack_channel_page(serde_json::json!({
+ "channels": [{ "id": "C1", "name": "general" }],
+ "response_metadata": { "next_cursor": "" },
+ }));
+ assert_eq!(page.next_cursor, None);
+}
+
+#[test]
+fn slack_candidates_rank_by_size_dedupe_and_cap() {
+ let channel = |id: &str, name: &str, members: Option<u64>| SlackChannelCandidate {
+ id: Some(id.to_string()),
+ name: name.to_string(),
+ purpose: None,
+ member_count: members,
+ archived: false,
+ };
+
+ let mut channels = vec![
+ channel("C1", "tiny", Some(2)),
+ channel("C2", "big", Some(50)),
+ channel("C2", "big", Some(50)), // duplicate id collapses
+ channel("C3", "medium", Some(10)),
+ SlackChannelCandidate {
+ archived: true,
+ ..channel("C4", "archived", Some(99))
+ },
+ // No id: deduped by name instead.
+ SlackChannelCandidate {
+ id: None,
+ ..channel("", "no-id", None)
+ },
+ SlackChannelCandidate {
+ id: None,
+ ..channel("", "no-id", None)
+ },
+ ];
+ channels
+ .extend((0..20).map(|i| channel(&format!("C{}", 100 + i), &format!("filler-{i}"), None)));
+
+ let selected = select_slack_candidates(channels, SLACK_GATHER_MAX_CHANNELS);
+ assert_eq!(selected.len(), SLACK_GATHER_MAX_CHANNELS);
+ // Largest first; archived and duplicates gone.
+ assert_eq!(selected[0].name, "big");
+ assert_eq!(selected[1].name, "medium");
+ assert_eq!(selected[2].name, "tiny");
+ assert!(selected.iter().all(|channel| !channel.archived));
+ assert_eq!(
+ selected
+ .iter()
+ .filter(|channel| channel.name == "no-id")
+ .count(),
+ 1
+ );
+ // Ties keep listing order.
+ assert_eq!(selected[3].name, "no-id");
+ assert_eq!(selected[4].name, "filler-0");
+}
+
#[test]
fn notion_fetch_text_parses_the_fetch_document_shape() {
let parsed = parse_notion_fetch_result(serde_json::Value::String(