Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(search_processing_service): channel message kafka (#5282)

Will Hutchinson32cecd2af50d
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.json
index d675257146..898d4abc37 100644
--- a/.github/workspace-dep-closures.json
+++ b/.github/workspace-dep-closures.json
@@ -496,7 +496,6 @@
"crates/ai_toolset",
"crates/ai_usage",
"crates/attachment",
- "crates/authentication_service_client",
"crates/bot_id",
"crates/bots",
"crates/channel_sender",
@@ -505,14 +504,9 @@
"crates/connection_gateway_models",
"crates/contacts",
"crates/cowlike",
- "crates/crm",
"crates/document_sub_type",
- "crates/email",
- "crates/email_db_client",
- "crates/email_utils",
"crates/entity_access",
"crates/entity_access_db_utils",
- "crates/entity_access_management",
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
@@ -536,25 +530,17 @@
"crates/model_file_type",
"crates/model_notifications",
"crates/model_user",
- "crates/models_ai_projection",
"crates/models_bulk_upload",
- "crates/models_email",
"crates/models_entity_access_management",
"crates/models_pagination",
"crates/models_permissions",
- "crates/models_properties",
"crates/non_empty",
"crates/notification",
"crates/prompt",
- "crates/properties",
"crates/rate_limit",
"crates/remote_env_var",
- "crates/s3_key",
"crates/share_permission_db_utils",
- "crates/sqs_client",
"crates/sqs_worker",
- "crates/system_properties",
- "crates/unfurl",
"crates/workspace-hack"
],
"broadcast": [
@@ -592,7 +578,6 @@
"crates/ai_toolset",
"crates/ai_usage",
"crates/attachment",
- "crates/authentication_service_client",
"crates/bot_id",
"crates/call",
"crates/channel_sender",
@@ -602,14 +587,9 @@
"crates/connection_gateway_models",
"crates/contacts",
"crates/cowlike",
- "crates/crm",
"crates/document_sub_type",
- "crates/email",
- "crates/email_db_client",
- "crates/email_utils",
"crates/entity_access",
"crates/entity_access_db_utils",
- "crates/entity_access_management",
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
@@ -634,25 +614,17 @@
"crates/model_file_type",
"crates/model_notifications",
"crates/model_user",
- "crates/models_ai_projection",
"crates/models_bulk_upload",
- "crates/models_email",
"crates/models_entity_access_management",
"crates/models_pagination",
"crates/models_permissions",
- "crates/models_properties",
"crates/non_empty",
"crates/notification",
"crates/prompt",
- "crates/properties",
"crates/rate_limit",
"crates/remote_env_var",
- "crates/s3_key",
"crates/share_permission_db_utils",
- "crates/sqs_client",
"crates/sqs_worker",
- "crates/system_properties",
- "crates/unfurl",
"crates/workspace-hack"
],
"call_recording_preview_handler": [
@@ -780,7 +752,6 @@
"crates/ai_toolset",
"crates/ai_usage",
"crates/attachment",
- "crates/authentication_service_client",
"crates/bot_id",
"crates/channel_sender",
"crates/channels",
@@ -788,14 +759,9 @@
"crates/connection_gateway_models",
"crates/contacts",
"crates/cowlike",
- "crates/crm",
"crates/document_sub_type",
- "crates/email",
- "crates/email_db_client",
- "crates/email_utils",
"crates/entity_access",
"crates/entity_access_db_utils",
- "crates/entity_access_management",
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
@@ -819,25 +785,17 @@
"crates/model_file_type",
"crates/model_notifications",
"crates/model_user",
- "crates/models_ai_projection",
"crates/models_bulk_upload",
- "crates/models_email",
"crates/models_entity_access_management",
"crates/models_pagination",
"crates/models_permissions",
- "crates/models_properties",
"crates/non_empty",
"crates/notification",
"crates/prompt",
- "crates/properties",
"crates/rate_limit",
"crates/remote_env_var",
- "crates/s3_key",
"crates/share_permission_db_utils",
- "crates/sqs_client",
"crates/sqs_worker",
- "crates/system_properties",
- "crates/unfurl",
"crates/workspace-hack"
],
"chat": [
@@ -882,7 +840,6 @@
"crates/ai_toolset",
"crates/ai_usage",
"crates/attachment",
- "crates/authentication_service_client",
"crates/bot_id",
"crates/channel_sender",
"crates/channels",
@@ -891,14 +848,9 @@
"crates/connection_gateway_models",
"crates/contacts",
"crates/cowlike",
- "crates/crm",
"crates/document_sub_type",
- "crates/email",
- "crates/email_db_client",
- "crates/email_utils",
"crates/entity_access",
"crates/entity_access_db_utils",
- "crates/entity_access_management",
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
@@ -922,25 +874,17 @@
"crates/model_file_type",
"crates/model_notifications",
"crates/model_user",
- "crates/models_ai_projection",
"crates/models_bulk_upload",
- "crates/models_email",
"crates/models_entity_access_management",
"crates/models_pagination",
"crates/models_permissions",
- "crates/models_properties",
"crates/non_empty",
"crates/notification",
"crates/prompt",
- "crates/properties",
"crates/rate_limit",
"crates/remote_env_var",
- "crates/s3_key",
"crates/share_permission_db_utils",
- "crates/sqs_client",
"crates/sqs_worker",
- "crates/system_properties",
- "crates/unfurl",
"crates/workspace-hack"
],
"complete_graph": [
@@ -1792,7 +1736,6 @@
"crates/model_file_type",
"crates/model_notifications",
"crates/model_user",
- "crates/models_ai_projection",
"crates/models_bulk_upload",
"crates/models_email",
"crates/models_entity_access_management",
@@ -1814,7 +1757,6 @@
"crates/s3_key",
"crates/secretsmanager_client",
"crates/share_permission_db_utils",
- "crates/sqs_client",
"crates/sqs_worker",
"crates/sync_service",
"crates/sync_service_client",
@@ -1877,7 +1819,6 @@
"crates/model_file_type",
"crates/model_notifications",
"crates/model_user",
- "crates/models_ai_projection",
"crates/models_bulk_upload",
"crates/models_email",
"crates/models_entity_access_management",
@@ -1899,7 +1840,6 @@
"crates/s3_key",
"crates/secretsmanager_client",
"crates/share_permission_db_utils",
- "crates/sqs_client",
"crates/sqs_worker",
"crates/sync_service",
"crates/sync_service_client",
@@ -2682,7 +2622,6 @@
"crates/model_file_type",
"crates/model_notifications",
"crates/model_user",
- "crates/models_ai_projection",
"crates/models_bulk_upload",
"crates/models_email",
"crates/models_entity_access_management",
@@ -2704,7 +2643,6 @@
"crates/s3_key",
"crates/secretsmanager_client",
"crates/share_permission_db_utils",
- "crates/sqs_client",
"crates/sqs_worker",
"crates/sync_service",
"crates/sync_service_client",
@@ -3333,7 +3271,6 @@
"crates/model_file_type",
"crates/model_notifications",
"crates/model_user",
- "crates/models_ai_projection",
"crates/models_bulk_upload",
"crates/models_email",
"crates/models_entity_access_management",
@@ -3352,9 +3289,7 @@
"crates/properties",
"crates/rate_limit",
"crates/remote_env_var",
- "crates/s3_key",
"crates/share_permission_db_utils",
- "crates/sqs_client",
"crates/sqs_worker",
"crates/system_properties",
"crates/unfurl",
@@ -3409,7 +3344,6 @@
"crates/model_file_type",
"crates/model_notifications",
"crates/model_user",
- "crates/models_ai_projection",
"crates/models_bulk_upload",
"crates/models_email",
"crates/models_entity_access_management",
@@ -3428,9 +3362,7 @@
"crates/properties",
"crates/rate_limit",
"crates/remote_env_var",
- "crates/s3_key",
"crates/share_permission_db_utils",
- "crates/sqs_client",
"crates/sqs_worker",
"crates/system_properties",
"crates/unfurl",
@@ -4107,7 +4039,6 @@
"crates/model_file_type",
"crates/model_notifications",
"crates/model_user",
- "crates/models_ai_projection",
"crates/models_bulk_upload",
"crates/models_email",
"crates/models_entity_access_management",
@@ -4126,9 +4057,7 @@
"crates/properties",
"crates/rate_limit",
"crates/remote_env_var",
- "crates/s3_key",
"crates/share_permission_db_utils",
- "crates/sqs_client",
"crates/sqs_worker",
"crates/system_properties",
"crates/unfurl",
@@ -4189,7 +4118,6 @@
"crates/model_file_type",
"crates/model_notifications",
"crates/model_user",
- "crates/models_ai_projection",
"crates/models_bulk_upload",
"crates/models_email",
"crates/models_entity_access_management",
@@ -4203,9 +4131,7 @@
"crates/properties",
"crates/rate_limit",
"crates/remote_env_var",
- "crates/s3_key",
"crates/share_permission_db_utils",
- "crates/sqs_client",
"crates/sqs_worker",
"crates/system_properties",
"crates/unfurl",
@@ -4220,7 +4146,6 @@
"crates/ai_toolset",
"crates/ai_usage",
"crates/attachment",
- "crates/authentication_service_client",
"crates/bot_id",
"crates/channel_sender",
"crates/channels",
@@ -4228,14 +4153,9 @@
"crates/connection_gateway_models",
"crates/contacts",
"crates/cowlike",
- "crates/crm",
"crates/document_sub_type",
- "crates/email",
- "crates/email_db_client",
- "crates/email_utils",
"crates/entity_access",
"crates/entity_access_db_utils",
- "crates/entity_access_management",
"crates/entity_mutation",
"crates/filter_ast",
"crates/frecency",
@@ -4259,28 +4179,20 @@
"crates/model_file_type",
"crates/model_notifications",
"crates/model_user",
- "crates/models_ai_projection",
"crates/models_bulk_upload",
- "crates/models_email",
"crates/models_entity_access_management",
"crates/models_opensearch",
"crates/models_pagination",
"crates/models_permissions",
- "crates/models_properties",
"crates/models_search_cursor",
"crates/name_search",
"crates/non_empty",
"crates/notification",
"crates/prompt",
- "crates/properties",
"crates/rate_limit",
"crates/remote_env_var",
- "crates/s3_key",
"crates/share_permission_db_utils",
- "crates/sqs_client",
"crates/sqs_worker",
- "crates/system_properties",
- "crates/unfurl",
"crates/workspace-hack"
],
"native_app_server": [
@@ -5082,7 +4994,6 @@
"crates/model_file_type",
"crates/model_notifications",
"crates/model_user",
- "crates/models_ai_projection",
"crates/models_bulk_upload",
"crates/models_dcs",
"crates/models_email",
@@ -5105,11 +5016,9 @@
"crates/rate_limit",
"crates/readonly_pool",
"crates/remote_env_var",
- "crates/s3_key",
"crates/search_service",
"crates/secretsmanager_client",
"crates/share_permission_db_utils",
- "crates/sqs_client",
"crates/sqs_worker",
"crates/system_properties",
"crates/unfurl",
@@ -5163,7 +5072,6 @@
"crates/model_file_type",
"crates/model_notifications",
"crates/model_user",
- "crates/models_ai_projection",
"crates/models_bulk_upload",
"crates/models_email",
"crates/models_entity_access_management",
@@ -5182,10 +5090,8 @@
"crates/properties",
"crates/rate_limit",
"crates/remote_env_var",
- "crates/s3_key",
"crates/search_service_client",
"crates/share_permission_db_utils",
- "crates/sqs_client",
"crates/sqs_worker",
"crates/system_properties",
"crates/unfurl",
@@ -5315,7 +5221,6 @@
"crates/model_file_type",
"crates/model_notifications",
"crates/model_user",
- "crates/models_ai_projection",
"crates/models_bulk_upload",
"crates/models_dcs",
"crates/models_email",
@@ -5337,9 +5242,7 @@
"crates/rate_limit",
"crates/remote_env_var",
"crates/roles_and_permissions",
- "crates/s3_key",
"crates/share_permission_db_utils",
- "crates/sqs_client",
"crates/sqs_worker",
"crates/sync_service",
"crates/sync_service_client",
@@ -5430,7 +5333,6 @@
"crates/model_file_type",
"crates/model_notifications",
"crates/model_user",
- "crates/models_ai_projection",
"crates/models_bulk_upload",
"crates/models_email",
"crates/models_entity_access_management",
@@ -5446,10 +5348,8 @@
"crates/rate_limit",
"crates/readonly_pool",
"crates/remote_env_var",
- "crates/s3_key",
"crates/share_permission_db_utils",
"crates/soup",
- "crates/sqs_client",
"crates/sqs_worker",
"crates/system_properties",
"crates/unfurl",
@@ -5778,7 +5678,6 @@
"crates/model_file_type",
"crates/model_notifications",
"crates/model_user",
- "crates/models_ai_projection",
"crates/models_bulk_upload",
"crates/models_email",
"crates/models_entity_access_management",
@@ -5797,10 +5696,8 @@
"crates/properties",
"crates/rate_limit",
"crates/remote_env_var",
- "crates/s3_key",
"crates/secretsmanager_client",
"crates/share_permission_db_utils",
- "crates/sqs_client",
"crates/sqs_worker",
"crates/system_properties",
"crates/task_dedup",
diff --git a/Cargo.lock b/Cargo.lock
index 4bf965dece..7ea064e6ed 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -2778,7 +2778,6 @@ dependencies = [
"serde_json",
"share_permission_db_utils",
"sqlx",
- "sqs_client",
"thiserror 2.0.18",
"tokio",
"tower 0.5.3",
@@ -12821,6 +12820,7 @@ dependencies = [
"aws-sdk-sqs",
"axum",
"call",
+ "channels",
"chrono",
"comms_db_client",
"document_sub_type",
diff --git a/crates/ai_tools/Cargo.toml b/crates/ai_tools/Cargo.toml
index 916de28414..8e840e7fba 100644
--- a/crates/ai_tools/Cargo.toml
+++ b/crates/ai_tools/Cargo.toml
@@ -73,7 +73,7 @@ serde.workspace = true
serde_json.workspace = true
properties = { path = "../properties", features = ["ai_tools"] }
soup = { path = "../soup", features = ["ai_tools", "outbound"] }
-sqs_client = { path = "../sqs_client", default-features = false, features = ["email", "gmail", "search"] }
+sqs_client = { path = "../sqs_client", default-features = false, features = ["email", "gmail"] }
teams = { path = "../teams", default-features = false, features = ["ai_tools", "outbound"] }
sqlx.workspace = true
strum.workspace = true
diff --git a/crates/ai_tools/src/build_context.rs b/crates/ai_tools/src/build_context.rs
index 8662891fca..83ac152928 100644
--- a/crates/ai_tools/src/build_context.rs
+++ b/crates/ai_tools/src/build_context.rs
@@ -263,9 +263,10 @@ pub async fn build_tool_service_context_from_env(
event_task_tracker,
);
// Channel messages sent by AI tools dispatch the same side effects as the
- // document-storage channel API (realtime, notifications, search indexing,
- // contact sync, broker events), so agent-sent messages notify mentioned
- // users and stream to connected clients instead of landing silently.
+ // document-storage channel API (realtime, notifications, contact sync, and
+ // broker events that drive live search indexing), so agent-sent messages
+ // notify mentioned users and stream to connected clients instead of landing
+ // silently.
let channel_tool_context = crate::tool_context::build_channel_tool_context_with_side_effects(
pool.clone(),
Arc::new(lexical_client.clone()),
diff --git a/crates/ai_tools/src/tool_context.rs b/crates/ai_tools/src/tool_context.rs
index 143467c9e7..bc967bca90 100644
--- a/crates/ai_tools/src/tool_context.rs
+++ b/crates/ai_tools/src/tool_context.rs
@@ -15,7 +15,6 @@ use channels::outbound::{
connection_gateway_realtime::ConnectionGatewayChannelRealtimePublisher,
contacts_dispatcher::ContactsChannelDispatcher, notification_sender::NotificationChannelSender,
pg_channels_repo::PgChannelsRepo, pg_side_effect_context::PgChannelSideEffectContext,
- sqs_search_indexer::SqsChannelSearchIndexer,
};
use chat::domain::service::ChatServiceImpl;
use chat::inbound::toolset::ChatToolContext;
@@ -124,11 +123,11 @@ pub type ToolChannelMessagesService = ChannelServiceImpl<
pub type ToolChannelToolContext =
ChannelToolContext<ToolChannelMessagesService, ToolEntityAccessService>;
-/// Build the channel AI tool context from a Postgres pool, with no side
-/// effects (no notifications/realtime/search indexing) — messages sent
-/// through it are persisted but never notify anyone. Only for tests and
-/// hosts that genuinely lack the side-effect clients; production hosts
-/// should use [`build_channel_tool_context_with_side_effects`].
+/// Build the channel AI tool context from a Postgres pool with no side
+/// effects. Messages sent through it are persisted, but never notify connected
+/// clients or publish the channel macro events that drive live search indexing.
+/// Only for tests and hosts that genuinely lack the side-effect clients;
+/// production hosts should use [`build_channel_tool_context_with_side_effects`].
/// `lexical_client` derives the mention list for messages the agent sends,
/// since bot-authored content arrives without the editor-tracked mentions.
pub fn build_channel_tool_context_without_side_effects(
@@ -143,14 +142,13 @@ pub fn build_channel_tool_context_without_side_effects(
}
/// Clients a host provides to wire the real channel side effects for AI
-/// tools. Queue names (notification ingress, contacts, search events) are
-/// resolved through `macro_queues`, so hosts only supply the shared clients.
+/// tools. Notification-ingress and contacts queue names are resolved through
+/// `macro_queues`, so hosts only supply the shared clients.
pub struct ChannelSideEffectClients {
/// Connection gateway client used to fan realtime updates out to
/// connected clients.
pub connection_gateway: Arc<ConnectionGatewayClient>,
- /// SQS client used for the notification-ingress, contacts, and
- /// search-event queues.
+ /// SQS client used for the notification-ingress and contacts queues.
pub sqs: aws_sdk_sqs::Client,
/// Broker publishing channel events to the `macro.channels` topic.
pub macro_event_broker: ToolEventBroker,
@@ -158,10 +156,10 @@ pub struct ChannelSideEffectClients {
/// Build the channel AI tool context dispatching the same side effects as the
/// document-storage channel API: realtime updates via the connection gateway,
-/// notifications via the notification-ingress queue, search indexing via the
-/// search-event queue, contact sync via the contacts queue, and channel
-/// events on the macro event broker. Hosts that let the agent send channel
-/// messages need this so mentions and replies notify their recipients.
+/// notifications via the notification-ingress queue, contact sync via the
+/// contacts queue, and channel events on the macro event broker. Those channel
+/// macro events drive live search indexing. Hosts that let the agent send
+/// channel messages need this so mentions and replies notify their recipients.
pub fn build_channel_tool_context_with_side_effects(
pool: sqlx::PgPool,
lexical_client: Arc<lexical_client::LexicalClient>,
@@ -179,14 +177,10 @@ pub fn build_channel_tool_context_with_side_effects(
macro_queues::ContactsQueue::new().to_string(),
),
});
- let search_event_queue = macro_queues::SearchEventQueue::new();
- let search_sqs =
- Arc::new(sqs_client::SQS::new(clients.sqs).search_event_queue(&search_event_queue));
let side_effects = ChannelSideEffectService::new(
PgChannelSideEffectContext::new(pool.clone()),
ConnectionGatewayChannelRealtimePublisher::new(clients.connection_gateway),
NotificationChannelSender::new(notification_ingress),
- SqsChannelSearchIndexer::new(search_sqs),
ContactsChannelDispatcher::new(contacts_ingress),
)
.with_macro_event_broker(clients.macro_event_broker);
@@ -198,8 +192,8 @@ pub fn build_channel_tool_context_with_side_effects(
}
/// Build the channel AI tool context wired to `dispatcher`, so messages sent by
-/// agent tools fire the host's channel side effects (notifications, realtime,
-/// search indexing).
+/// agent tools fire the host's notification, realtime, and macro-event side
+/// effects. Channel macro events drive live search indexing.
pub fn build_channel_tool_context_with_dispatcher(
pool: sqlx::PgPool,
dispatcher: ToolChannelEventDispatcher,
diff --git a/crates/channels/Cargo.toml b/crates/channels/Cargo.toml
index 7259b9cbfa..e779424305 100644
--- a/crates/channels/Cargo.toml
+++ b/crates/channels/Cargo.toml
@@ -58,7 +58,6 @@ outbound = [
"dep:serde_json",
"dep:share_permission_db_utils",
"dep:sqlx",
- "dep:sqs_client",
"entity_access/ports",
"list",
"macro_user_id/sqlx",
@@ -114,9 +113,6 @@ serde = { workspace = true }
serde_json = { workspace = true, optional = true }
share_permission_db_utils = { path = "../share_permission_db_utils", optional = true }
sqlx = { workspace = true, optional = true }
-sqs_client = { path = "../sqs_client", default-features = false, features = [
- "search",
-], optional = true }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["macros", "sync"], optional = true }
tracing = { workspace = true }
diff --git a/crates/channels/src/domain/ports.rs b/crates/channels/src/domain/ports.rs
index 7c2a9acd8a..64c47fec6d 100644
--- a/crates/channels/src/domain/ports.rs
+++ b/crates/channels/src/domain/ports.rs
@@ -1096,19 +1096,6 @@ pub trait ChannelContactsDispatcher: Send + Sync + 'static {
) -> impl Future<Output = Result<(), Self::Err>> + Send;
}
-/// Indexer for channel search updates.
-pub trait ChannelSearchIndexer: Send + Sync + 'static {
- /// Enqueue a message upsert.
- fn index_message(&self, channel_id: Uuid, message_id: Uuid) -> impl Future<Output = ()> + Send;
-
- /// Enqueue a message or channel removal.
- fn remove_message(
- &self,
- channel_id: Uuid,
- message_id: Option<Uuid>,
- ) -> impl Future<Output = ()> + Send;
-}
-
/// Share-permission updater for items referenced by channel messages.
pub trait ChannelReferenceSharePermissions: Send + Sync + 'static {
/// Error type for reference share-permission operations.
diff --git a/crates/channels/src/domain/side_effects.rs b/crates/channels/src/domain/side_effects.rs
index 576f6d5834..28e428c4ee 100644
--- a/crates/channels/src/domain/side_effects.rs
+++ b/crates/channels/src/domain/side_effects.rs
@@ -16,8 +16,7 @@ use crate::domain::{
},
ports::{
ChannelContactsDispatcher, ChannelEventDispatcher, ChannelEventHandler,
- ChannelNotificationSender, ChannelRealtimePublisher, ChannelSearchIndexer,
- ChannelSideEffectContext,
+ ChannelNotificationSender, ChannelRealtimePublisher, ChannelSideEffectContext,
},
};
use bot_id::BotIdStr;
@@ -295,11 +294,10 @@ pub enum ChannelNotificationEffect {
/// Domain service that derives and dispatches side effects for channel events.
#[derive(Clone)]
-pub struct ChannelSideEffectService<C, R, N, S, K, B = NoopMacroEventBroker> {
+pub struct ChannelSideEffectService<C, R, N, K, B = NoopMacroEventBroker> {
context: C,
realtime: R,
notifications: N,
- search: S,
contacts: K,
bot_triggers: Option<ChannelBotTriggerSender>,
macro_event_broker: B,
@@ -327,17 +325,16 @@ struct InviteNotificationRequest {
metadata: ChannelMetadata,
}
-impl<C, R, N, S, K> ChannelSideEffectService<C, R, N, S, K> {
+impl<C, R, N, K> ChannelSideEffectService<C, R, N, K> {
/// Create a channel side-effect service that drops broker events.
///
/// Use [`Self::with_macro_event_broker`] to publish channel events to the
/// macro event broker.
- pub fn new(context: C, realtime: R, notifications: N, search: S, contacts: K) -> Self {
+ pub fn new(context: C, realtime: R, notifications: N, contacts: K) -> Self {
Self {
context,
realtime,
notifications,
- search,
contacts,
bot_triggers: None,
macro_event_broker: NoopMacroEventBroker,
@@ -345,7 +342,7 @@ impl<C, R, N, S, K> ChannelSideEffectService<C, R, N, S, K> {
}
}
-impl<C, R, N, S, K, B> ChannelSideEffectService<C, R, N, S, K, B> {
+impl<C, R, N, K, B> ChannelSideEffectService<C, R, N, K, B> {
/// Configure a sender for bot triggers derived from channel messages.
pub fn with_bot_trigger_sender(mut self, bot_triggers: ChannelBotTriggerSender) -> Self {
self.bot_triggers = Some(bot_triggers);
@@ -356,12 +353,11 @@ impl<C, R, N, S, K, B> ChannelSideEffectService<C, R, N, S, K, B> {
pub fn with_macro_event_broker<B2: MacroEventBroker>(
self,
macro_event_broker: B2,
- ) -> ChannelSideEffectService<C, R, N, S, K, B2> {
+ ) -> ChannelSideEffectService<C, R, N, K, B2> {
ChannelSideEffectService {
context: self.context,
realtime: self.realtime,
notifications: self.notifications,
- search: self.search,
contacts: self.contacts,
bot_triggers: self.bot_triggers,
macro_event_broker,
@@ -428,12 +424,11 @@ where
}
}
-impl<C, R, N, S, K, B> ChannelEventHandler for ChannelSideEffectService<C, R, N, S, K, B>
+impl<C, R, N, K, B> ChannelEventHandler for ChannelSideEffectService<C, R, N, K, B>
where
C: ChannelSideEffectContext + Clone,
R: ChannelRealtimePublisher + Clone,
N: ChannelNotificationSender + Clone,
- S: ChannelSearchIndexer + Clone,
K: ChannelContactsDispatcher + Clone,
B: MacroEventBroker + Clone,
{
@@ -448,9 +443,7 @@ where
ChannelEvent::ParticipantsRemoved { .. } => {}
ChannelEvent::EntityMentionCreated { .. } => {}
ChannelEvent::EntityMentionDeleted { .. } => {}
- ChannelEvent::ChannelDeleted { channel_id, .. } => {
- self.search.remove_message(channel_id, None).await;
- }
+ ChannelEvent::ChannelDeleted { .. } => {}
ChannelEvent::MessagePosted {
channel_id,
metadata,
@@ -491,7 +484,6 @@ where
nonce,
})
.await;
- self.search.index_message(channel_id, message_id).await;
}
ChannelEvent::MessageChanged {
channel_id,
@@ -501,7 +493,6 @@ where
posted_notification,
..
} => {
- let message_id = message.id;
let bot_profile = self.bot_profile_for_message(&message).await;
self.publish_realtime(ChannelRealtimeEffect::Message {
recipients,
@@ -510,7 +501,6 @@ where
nonce,
})
.await;
- self.search.index_message(channel_id, message_id).await;
if let Some(notification) = posted_notification {
self.send_message_posted_notifications(PostedMessageNotificationInputs {
@@ -527,13 +517,11 @@ where
}
}
ChannelEvent::MessageDeleted {
- channel_id,
message,
recipients,
nonce,
..
} => {
- let message_id = message.id;
let bot_profile = self.bot_profile_for_message(&message).await;
self.publish_realtime(ChannelRealtimeEffect::Message {
recipients,
@@ -542,9 +530,6 @@ where
nonce,
})
.await;
- self.search
- .remove_message(channel_id, Some(message_id))
- .await;
}
ChannelEvent::ReactionChanged {
channel_id,
@@ -632,12 +617,11 @@ where
}
}
-impl<C, R, N, S, K, B> ChannelSideEffectService<C, R, N, S, K, B>
+impl<C, R, N, K, B> ChannelSideEffectService<C, R, N, K, B>
where
C: ChannelSideEffectContext + Clone,
R: ChannelRealtimePublisher + Clone,
N: ChannelNotificationSender + Clone,
- S: ChannelSearchIndexer + Clone,
K: ChannelContactsDispatcher + Clone,
{
async fn handle_message_posted(&self, event: MessagePostedSideEffects) {
@@ -673,7 +657,6 @@ where
.await;
}
- self.search.index_message(channel_id, message.id).await;
self.dispatch_bot_triggers(channel_id, &message, &mentions);
if notification_policy != PostMessageNotificationPolicy::Silent {
self.send_message_posted_notifications(PostedMessageNotificationInputs {
diff --git a/crates/channels/src/domain/side_effects/test.rs b/crates/channels/src/domain/side_effects/test.rs
index 8547300b01..022319b366 100644
--- a/crates/channels/src/domain/side_effects/test.rs
+++ b/crates/channels/src/domain/side_effects/test.rs
@@ -4,15 +4,12 @@ use crate::domain::{
models::{BotId, EntityMention, ParticipantRole, Sender},
ports::{
ChannelEventHandler, ChannelNotificationSender, ChannelRealtimePublisher,
- ChannelSearchIndexer, ChannelSideEffectContext,
+ ChannelSideEffectContext,
},
};
use chrono::Utc;
use std::sync::{Arc, Mutex};
-type IndexedMessages = Arc<Mutex<Vec<(Uuid, Uuid)>>>;
-type RemovedMessages = Arc<Mutex<Vec<(Uuid, Option<Uuid>)>>>;
-
#[derive(Clone)]
struct FakeContext {
message_count: i64,
@@ -114,22 +111,6 @@ impl ChannelNotificationSender for FakeNotifications {
}
}
-#[derive(Clone, Default)]
-struct FakeSearch {
- indexed: IndexedMessages,
- removed: RemovedMessages,
-}
-
-impl ChannelSearchIndexer for FakeSearch {
- async fn index_message(&self, channel_id: Uuid, message_id: Uuid) {
- self.indexed.lock().unwrap().push((channel_id, message_id));
- }
-
- async fn remove_message(&self, channel_id: Uuid, message_id: Option<Uuid>) {
- self.removed.lock().unwrap().push((channel_id, message_id));
- }
-}
-
#[derive(Clone, Default)]
struct FakeContacts {
users: Arc<Mutex<Vec<HashSet<MacroUserIdStr<'static>>>>>,
@@ -166,7 +147,6 @@ async fn macro_ai_bot_profile_is_builtin_without_context_lookup() {
},
FakeRealtime::default(),
FakeNotifications::default(),
- FakeSearch::default(),
FakeContacts::default(),
);
let now = Utc::now();
@@ -203,7 +183,6 @@ async fn non_macro_bot_profile_uses_context_lookup() {
},
FakeRealtime::default(),
FakeNotifications::default(),
- FakeSearch::default(),
FakeContacts::default(),
);
let now = Utc::now();
@@ -230,20 +209,18 @@ async fn non_macro_bot_profile_uses_context_lookup() {
}
#[tokio::test]
-async fn message_posted_derives_realtime_search_and_notification_effects() {
+async fn message_posted_derives_realtime_and_notification_effects() {
let channel_id = Uuid::new_v4();
let message_id = Uuid::new_v4();
let sender = user("[email protected]");
let recipient = user("[email protected]");
let realtime = FakeRealtime::default();
let notifications = FakeNotifications::default();
- let search = FakeSearch::default();
let contacts = FakeContacts::default();
let service = ChannelSideEffectService::new(
FakeContext::default(),
realtime.clone(),
notifications.clone(),
- search.clone(),
contacts,
);
let now = Utc::now();
@@ -307,11 +284,6 @@ async fn message_posted_derives_realtime_search_and_notification_effects() {
assert_eq!(recipients.len(), 2);
drop(realtime_effects);
- assert_eq!(
- *search.indexed.lock().unwrap(),
- vec![(channel_id, message_id)]
- );
-
let notification_effects = notifications.effects.lock().unwrap();
assert_eq!(notification_effects.len(), 1);
let ChannelNotificationEffect::ChannelMessage {
@@ -383,12 +355,10 @@ async fn silent_message_posted_skips_notifications_only() {
let recipient = user("[email protected]");
let realtime = FakeRealtime::default();
let notifications = FakeNotifications::default();
- let search = FakeSearch::default();
let service = ChannelSideEffectService::new(
FakeContext::default(),
realtime.clone(),
notifications.clone(),
- search.clone(),
FakeContacts::default(),
);
let now = Utc::now();
@@ -437,10 +407,6 @@ async fn silent_message_posted_skips_notifications_only() {
.await;
assert_eq!(realtime.effects.lock().unwrap().len(), 1);
- assert_eq!(
- *search.indexed.lock().unwrap(),
- vec![(channel_id, message_id)]
- );
assert!(notifications.effects.lock().unwrap().is_empty());
}
@@ -461,7 +427,6 @@ async fn mentions_only_skips_failing_invite_lookup_and_sends_mention() {
},
FakeRealtime::default(),
notifications.clone(),
- FakeSearch::default(),
FakeContacts::default(),
);
let now = Utc::now();
@@ -531,7 +496,6 @@ async fn message_changed_with_posted_notification_context_sends_notification() {
},
FakeRealtime::default(),
notifications.clone(),
- FakeSearch::default(),
FakeContacts::default(),
);
let now = Utc::now();
@@ -595,12 +559,10 @@ async fn bot_message_posted_sends_channel_message_notification() {
let recipient = user("[email protected]");
let realtime = FakeRealtime::default();
let notifications = FakeNotifications::default();
- let search = FakeSearch::default();
let service = ChannelSideEffectService::new(
FakeContext::default(),
realtime.clone(),
notifications.clone(),
- search.clone(),
FakeContacts::default(),
);
@@ -614,11 +576,6 @@ async fn bot_message_posted_sends_channel_message_notification() {
.await;
assert_eq!(realtime.effects.lock().unwrap().len(), 1);
- assert_eq!(
- *search.indexed.lock().unwrap(),
- vec![(channel_id, message_id)]
- );
-
let notification_effects = notifications.effects.lock().unwrap();
assert_eq!(notification_effects.len(), 1);
let ChannelNotificationEffect::ChannelMessage {
@@ -651,7 +608,6 @@ async fn bot_message_without_profile_skips_notifications() {
let recipient = user("[email protected]");
let realtime = FakeRealtime::default();
let notifications = FakeNotifications::default();
- let search = FakeSearch::default();
let service = ChannelSideEffectService::new(
FakeContext {
bot_profile: None,
@@ -659,7 +615,6 @@ async fn bot_message_without_profile_skips_notifications() {
},
realtime.clone(),
notifications.clone(),
- search.clone(),
FakeContacts::default(),
);
@@ -673,10 +628,6 @@ async fn bot_message_without_profile_skips_notifications() {
.await;
assert_eq!(realtime.effects.lock().unwrap().len(), 1);
- assert_eq!(
- *search.indexed.lock().unwrap(),
- vec![(channel_id, message_id)]
- );
assert!(notifications.effects.lock().unwrap().is_empty());
}
@@ -692,7 +643,6 @@ async fn bot_first_message_sends_channel_message_not_invite() {
},
FakeRealtime::default(),
notifications.clone(),
- FakeSearch::default(),
FakeContacts::default(),
);
@@ -729,7 +679,6 @@ async fn bot_thread_reply_sends_reply_notification() {
},
FakeRealtime::default(),
notifications.clone(),
- FakeSearch::default(),
FakeContacts::default(),
);
@@ -776,7 +725,6 @@ async fn bot_participant_is_never_a_notification_recipient() {
FakeContext::default(),
FakeRealtime::default(),
notifications.clone(),
- FakeSearch::default(),
FakeContacts::default(),
);
@@ -810,7 +758,6 @@ async fn user_message_with_bot_mention_enqueues_bot_trigger() {
FakeContext::default(),
FakeRealtime::default(),
FakeNotifications::default(),
- FakeSearch::default(),
FakeContacts::default(),
)
.with_bot_trigger_sender(bot_trigger_sender);
@@ -891,7 +838,6 @@ async fn document_mentions_notify_participants_except_sender() {
},
FakeRealtime::default(),
notifications.clone(),
- FakeSearch::default(),
FakeContacts::default(),
);
let now = Utc::now();
@@ -1137,7 +1083,9 @@ impl MacroEventBroker for TestEventBroker {
}
#[derive(Clone, Default)]
-struct FailingEventBroker;
+struct FailingEventBroker {
+ attempts: Arc<Mutex<usize>>,
+}
impl MacroEventBroker for FailingEventBroker {
fn send_event<E: macro_event_broker::MacroEvent + ?Sized>(
@@ -1147,6 +1095,7 @@ impl MacroEventBroker for FailingEventBroker {
tokio::task::JoinHandle<Result<(), macro_event_broker::EventBrokerError>>,
macro_event_broker::EventBrokerError,
> {
+ *self.attempts.lock().unwrap() += 1;
Err(macro_event_broker::EventBrokerError::Publish(
"broker unavailable".to_string(),
))
@@ -1159,7 +1108,6 @@ fn broker_service(
FakeContext,
FakeRealtime,
FakeNotifications,
- FakeSearch,
FakeContacts,
TestEventBroker,
> {
@@ -1167,7 +1115,6 @@ fn broker_service(
FakeContext::default(),
FakeRealtime::default(),
FakeNotifications::default(),
- FakeSearch::default(),
FakeContacts::default(),
)
.with_macro_event_broker(broker)
@@ -1186,6 +1133,22 @@ fn attachment(channel_id: Uuid, message_id: Uuid) -> MutatedAttachment {
}
}
+fn channel_message(channel_id: Uuid, message_id: Uuid) -> MutatedMessage {
+ let now = Utc::now();
+ MutatedMessage {
+ id: message_id,
+ channel_id,
+ thread_id: None,
+ sender_id: Sender::new_from_user(user("[email protected]")),
+ triggered_by: None,
+ content: "updated message".to_string(),
+ created_at: now,
+ updated_at: now,
+ edited_at: Some(now),
+ deleted_at: None,
+ }
+}
+
#[tokio::test]
async fn handle_publishes_channel_created_event() {
let broker = TestEventBroker::default();
@@ -1316,6 +1279,65 @@ async fn handle_publishes_attachment_deltas() {
);
}
+#[tokio::test]
+async fn handle_publishes_message_patch_and_delete_events() {
+ let broker = TestEventBroker::default();
+ let service = broker_service(broker.clone());
+ let channel_id = Uuid::new_v4();
+ let message_id = Uuid::new_v4();
+ let actor = Sender::new_from_user(user("[email protected]"));
+ let patched_message = channel_message(channel_id, message_id);
+ let mut deleted_message = patched_message.clone();
+ deleted_message.deleted_at = Some(Utc::now());
+
+ service
+ .handle(ChannelEvent::MessageChanged {
+ channel_id,
+ actor: actor.clone(),
+ message: patched_message,
+ recipients: Vec::new(),
+ nonce: None,
+ posted_notification: None,
+ })
+ .await;
+ service
+ .handle(ChannelEvent::MessageDeleted {
+ channel_id,
+ actor: actor.clone(),
+ message: deleted_message,
+ recipients: Vec::new(),
+ nonce: None,
+ })
+ .await;
+ service
+ .handle(ChannelEvent::ChannelDeleted { channel_id, actor })
+ .await;
+
+ let published = broker.published.lock().unwrap();
+ assert_eq!(published.len(), 3);
+ assert_eq!(
+ published[0].envelope["event_type"],
+ "channel.message_patched"
+ );
+ assert_eq!(
+ published[1].envelope["event_type"],
+ "channel.message_deleted"
+ );
+ assert_eq!(published[2].envelope["event_type"], "channel.deleted");
+ assert_eq!(
+ published[0].envelope["metadata"]["message_id"],
+ message_id.to_string()
+ );
+ assert_eq!(
+ published[1].envelope["metadata"]["message_id"],
+ message_id.to_string()
+ );
+ for event in published.iter() {
+ assert_eq!(event.topic, "macro.channels");
+ assert_eq!(event.key, channel_id.to_string());
+ }
+}
+
#[tokio::test]
async fn handle_publishes_participant_events() {
let broker = TestEventBroker::default();
@@ -1368,28 +1390,36 @@ async fn handle_publishes_nothing_for_typing() {
#[tokio::test]
async fn publish_failure_does_not_break_other_side_effects() {
- let search = FakeSearch::default();
+ let realtime = FakeRealtime::default();
+ let broker = FailingEventBroker::default();
let service = ChannelSideEffectService::new(
FakeContext::default(),
- FakeRealtime::default(),
+ realtime.clone(),
FakeNotifications::default(),
- search.clone(),
FakeContacts::default(),
)
- .with_macro_event_broker(FailingEventBroker);
+ .with_macro_event_broker(broker.clone());
let channel_id = Uuid::new_v4();
+ let message_id = Uuid::new_v4();
service
- .handle(ChannelEvent::ChannelDeleted {
+ .handle(ChannelEvent::MessageChanged {
channel_id,
actor: Sender::new_from_user(user("[email protected]")),
+ message: channel_message(channel_id, message_id),
+ recipients: users(&["[email protected]"]),
+ nonce: Some("nonce-1".to_string()),
+ posted_notification: None,
})
.await;
- assert_eq!(
- search.removed.lock().unwrap().as_slice(),
- &[(channel_id, None)]
- );
+ assert_eq!(*broker.attempts.lock().unwrap(), 1);
+ let realtime_effects = realtime.effects.lock().unwrap();
+ let ChannelRealtimeEffect::Message { message, nonce, .. } = &realtime_effects[0] else {
+ panic!("expected message realtime effect");
+ };
+ assert_eq!(message.id, message_id);
+ assert_eq!(nonce.as_deref(), Some("nonce-1"));
}
#[test]
diff --git a/crates/channels/src/outbound/mod.rs b/crates/channels/src/outbound/mod.rs
index 1864dddfb2..5c50321dc5 100644
--- a/crates/channels/src/outbound/mod.rs
+++ b/crates/channels/src/outbound/mod.rs
@@ -12,5 +12,3 @@ pub mod pg_channel_reference_share_permissions;
pub mod pg_channels_repo;
/// Postgres context adapter for channel side-effect policy.
pub mod pg_side_effect_context;
-/// SQS search-index adapter for channel side effects.
-pub mod sqs_search_indexer;
diff --git a/crates/channels/src/outbound/sqs_search_indexer.rs b/crates/channels/src/outbound/sqs_search_indexer.rs
deleted file mode 100644
index 2c1d8e184a..0000000000
--- a/crates/channels/src/outbound/sqs_search_indexer.rs
+++ /dev/null
@@ -1,60 +0,0 @@
-//! SQS search-index adapter for channel side effects.
-
-use crate::domain::ports::ChannelSearchIndexer;
-use std::sync::Arc;
-use uuid::Uuid;
-
-/// SQS-backed search index adapter.
-#[derive(Clone)]
-pub struct SqsChannelSearchIndexer {
- sqs: Arc<sqs_client::SQS>,
-}
-
-impl SqsChannelSearchIndexer {
- /// Create a search index adapter.
- pub fn new(sqs: Arc<sqs_client::SQS>) -> Self {
- Self { sqs }
- }
-}
-
-impl ChannelSearchIndexer for SqsChannelSearchIndexer {
- async fn index_message(&self, channel_id: Uuid, message_id: Uuid) {
- let sqs = self.sqs.clone();
- tokio::spawn(async move {
- sqs.send_message_to_search_event_queue(
- sqs_client::search::SearchQueueMessage::ChannelMessageUpdate(
- sqs_client::search::channel::ChannelMessageUpdate {
- channel_id: channel_id.to_string(),
- message_id: message_id.to_string(),
- index_override: None,
- },
- ),
- )
- .await
- .inspect_err(|e| {
- tracing::error!(error=?e, "SEARCH_QUEUE unable to enqueue message");
- })
- .ok();
- });
- }
-
- async fn remove_message(&self, channel_id: Uuid, message_id: Option<Uuid>) {
- let sqs = self.sqs.clone();
- tokio::spawn(async move {
- sqs.send_message_to_search_event_queue(
- sqs_client::search::SearchQueueMessage::RemoveChannelMessage(
- sqs_client::search::channel::RemoveChannelMessage {
- channel_id: channel_id.to_string(),
- message_id: message_id.map(|id| id.to_string()),
- index_override: None,
- },
- ),
- )
- .await
- .inspect_err(|e| {
- tracing::error!(error=?e, "SEARCH_QUEUE unable to enqueue remove message");
- })
- .ok();
- });
- }
-}
diff --git a/crates/sqs_client/src/search/channel.rs b/crates/sqs_client/src/search/channel.rs
index 61a6a4add2..627051a473 100644
--- a/crates/sqs_client/src/search/channel.rs
+++ b/crates/sqs_client/src/search/channel.rs
@@ -8,14 +8,3 @@ pub struct ChannelMessageUpdate {
#[serde(skip_serializing_if = "Option::is_none", default)]
pub index_override: Option<String>,
}
-
-#[derive(serde::Serialize, serde::Deserialize, PartialEq, Eq, Debug)]
-pub struct RemoveChannelMessage {
- /// The channel id
- pub channel_id: String,
- /// The message id
- pub message_id: Option<String>,
- /// Optional override for the target OpenSearch index
- #[serde(skip_serializing_if = "Option::is_none", default)]
- pub index_override: Option<String>,
-}
diff --git a/crates/sqs_client/src/search/mod.rs b/crates/sqs_client/src/search/mod.rs
index ada427e0da..0f0f0c82a4 100644
--- a/crates/sqs_client/src/search/mod.rs
+++ b/crates/sqs_client/src/search/mod.rs
@@ -2,7 +2,7 @@ use crate::{
SQS,
search::{
call::{CallRecordMessage, RemoveCallRecord},
- channel::{ChannelMessageUpdate, RemoveChannelMessage},
+ channel::ChannelMessageUpdate,
chat::{ChatMessage, RemoveChatMessage},
document::{DocumentId, DocumentPropertiesUpdate, SearchExtractorMessage},
email::{EmailLinkMessage, EmailMessage, EmailThreadBatchMessage, EmailThreadMessage},
@@ -85,7 +85,6 @@ pub enum SearchQueueMessage {
ExtractEmailThreadBatch(EmailThreadBatchMessage),
// Channel
ChannelMessageUpdate(ChannelMessageUpdate),
- RemoveChannelMessage(RemoveChannelMessage),
// Call
CallRecord(CallRecordMessage),
RemoveCallRecord(RemoveCallRecord),
@@ -116,13 +115,6 @@ impl PrimaryId for SearchQueueMessage {
}
SearchQueueMessage::RemoveEmailLink(message) => message.link_id.clone(),
SearchQueueMessage::ChannelMessageUpdate(message) => message.message_id.clone(),
- SearchQueueMessage::RemoveChannelMessage(message) => {
- format!(
- "{}{}",
- message.channel_id,
- message.message_id.clone().unwrap_or_default()
- )
- }
SearchQueueMessage::CallRecord(message) => message.call_id.clone(),
SearchQueueMessage::RemoveCallRecord(message) => format!(
"{}{}",
@@ -157,7 +149,6 @@ impl SearchQueueMessage {
SearchQueueMessage::RemoveEmailLink(_) => Operation::Remove,
// Channels
SearchQueueMessage::ChannelMessageUpdate(_) => Operation::ExtractText,
- SearchQueueMessage::RemoveChannelMessage(_) => Operation::Remove,
// Calls
SearchQueueMessage::CallRecord(_) => Operation::ExtractText,
SearchQueueMessage::RemoveCallRecord(_) => Operation::Remove,
diff --git a/services/authentication_service/src/api/context.rs b/services/authentication_service/src/api/context.rs
index dce6734514..432ec34a67 100644
--- a/services/authentication_service/src/api/context.rs
+++ b/services/authentication_service/src/api/context.rs
@@ -13,7 +13,6 @@ use channels::{
notification_sender::NotificationChannelSender,
pg_channel_reference_share_permissions::PgChannelReferenceSharePermissions,
pg_channels_repo::PgChannelsRepo, pg_side_effect_context::PgChannelSideEffectContext,
- sqs_search_indexer::SqsChannelSearchIndexer,
},
};
use contacts::{domain::service::SqsContactsIngress, outbound::ingress::SqsContactsQueue};
@@ -62,7 +61,6 @@ pub(crate) type ChannelServiceType = ChannelServiceImpl<
PgChannelSideEffectContext,
ConnectionGatewayChannelRealtimePublisher,
NotificationChannelSender<NotificationIngressType>,
- SqsChannelSearchIndexer,
ContactsChannelDispatcher<SqsContactsIngress<SqsContactsQueue>>,
AuthenticationEventBroker,
>,
diff --git a/services/authentication_service/src/main.rs b/services/authentication_service/src/main.rs
index 9f7209966f..5cbab0b67e 100644
--- a/services/authentication_service/src/main.rs
+++ b/services/authentication_service/src/main.rs
@@ -14,7 +14,6 @@ use channels::{
notification_sender::NotificationChannelSender,
pg_channel_reference_share_permissions::PgChannelReferenceSharePermissions,
pg_channels_repo::PgChannelsRepo, pg_side_effect_context::PgChannelSideEffectContext,
- sqs_search_indexer::SqsChannelSearchIndexer,
},
};
use config::{Config, Environment};
@@ -337,13 +336,13 @@ async fn main() -> anyhow::Result<()> {
ConnectionGatewayUrl::new()?.to_string(),
));
// Authentication creates channels and posts support welcome messages in-process, so its
- // channel service must dispatch the same realtime, notification, search, contact, and broker
- // side effects as the document-storage channel API.
+ // channel service must dispatch the same realtime, notification, contact, and broker side
+ // effects as the document-storage channel API. Channel broker events drive live search
+ // indexing.
let channel_side_effects = ChannelSideEffectService::new(
PgChannelSideEffectContext::new(db.clone()),
ConnectionGatewayChannelRealtimePublisher::new(connection_gateway_client),
NotificationChannelSender::new(notification_ingress_service.clone()),
- SqsChannelSearchIndexer::new(sqs_client.clone()),
ContactsChannelDispatcher::new(contacts_ingress),
)
.with_macro_event_broker(macro_event_broker.clone());
diff --git a/services/document_storage_service/src/api/context.rs b/services/document_storage_service/src/api/context.rs
index d0992fa593..8c72c91d0c 100644
--- a/services/document_storage_service/src/api/context.rs
+++ b/services/document_storage_service/src/api/context.rs
@@ -30,7 +30,6 @@ use channels::{
notification_sender::NotificationChannelSender,
pg_channel_reference_share_permissions::PgChannelReferenceSharePermissions,
pg_channels_repo::PgChannelsRepo, pg_side_effect_context::PgChannelSideEffectContext,
- sqs_search_indexer::SqsChannelSearchIndexer,
},
};
use connection::{
@@ -309,7 +308,6 @@ pub(crate) type DssChannelService = ChannelServiceImpl<
PgChannelSideEffectContext,
ConnectionGatewayChannelRealtimePublisher,
NotificationChannelSender<NotificationIngressType>,
- SqsChannelSearchIndexer,
ContactsChannelDispatcher<SqsContactsIngress<SqsContactsQueue>>,
DssEventBroker,
>,
diff --git a/services/document_storage_service/src/main.rs b/services/document_storage_service/src/main.rs
index 8cd04deef2..49f7abd9cc 100644
--- a/services/document_storage_service/src/main.rs
+++ b/services/document_storage_service/src/main.rs
@@ -44,7 +44,6 @@ use channels::{
notification_sender::NotificationChannelSender,
pg_channel_reference_share_permissions::PgChannelReferenceSharePermissions,
pg_channels_repo::PgChannelsRepo, pg_side_effect_context::PgChannelSideEffectContext,
- sqs_search_indexer::SqsChannelSearchIndexer,
},
};
use config::{Config, Environment};
@@ -834,7 +833,6 @@ async fn main() -> anyhow::Result<()> {
PgChannelSideEffectContext::new(db.clone()),
ConnectionGatewayChannelRealtimePublisher::new(conn_gateway_client.clone()),
NotificationChannelSender::new(notification_ingress_service.clone()),
- SqsChannelSearchIndexer::new(sqs_client.clone()),
ContactsChannelDispatcher::new(contacts_ingress.clone()),
)
.with_bot_trigger_sender(bot_trigger_sender)
diff --git a/services/search_processing_service/Cargo.toml b/services/search_processing_service/Cargo.toml
index 3dc0413aa2..23cb6e3773 100644
--- a/services/search_processing_service/Cargo.toml
+++ b/services/search_processing_service/Cargo.toml
@@ -9,6 +9,7 @@ version = "0.1.0"
default = ["processing", "service", "pdf"]
processing = [
"dep:call",
+ "dep:channels",
"dep:kafka_util",
"dep:macro_event_broker",
"dep:macro_event_topics",
@@ -35,6 +36,7 @@ aws-sdk-s3 = { workspace = true }
aws-sdk-sqs = { workspace = true }
axum = { workspace = true }
call = { path = "../../crates/call", default-features = false, optional = true }
+channels = { path = "../../crates/channels", default-features = false, optional = true }
chrono = { workspace = true }
comms_db_client = { path = "../../crates/comms_db_client" }
email_db_client = { path = "../../crates/email_db_client" }
diff --git a/services/search_processing_service/src/inbound/kafka_consumer.rs b/services/search_processing_service/src/inbound/kafka_consumer.rs
index 9d99295319..391070dee2 100644
--- a/services/search_processing_service/src/inbound/kafka_consumer.rs
+++ b/services/search_processing_service/src/inbound/kafka_consumer.rs
@@ -12,6 +12,7 @@ mod test;
use std::{future::Future, time::Duration};
use call::domain::events::{CallMacroEvent, CallTopicEvent};
+use channels::domain::broker_events::{ChannelMacroEvent, ChannelTopicEvent};
use kafka_util::{GroupName, KafkaEventConsumer};
use macro_event_broker::{
KafkaConsumerAdapter, MacroEvent as _, MacroEventCollection as _, MacroEventConsumerService,
@@ -27,7 +28,10 @@ use tokio::sync::mpsc;
use tokio_retry::{Retry, strategy::ExponentialBackoff};
use uuid::Uuid;
-use crate::process::call::{process_call_record, process_remove_call_record};
+use crate::process::{
+ call::{process_call_record, process_remove_call_record},
+ channel::{process_channel_message_update, process_remove_channel_message},
+};
/// Consumer group used for live search-index event offsets.
pub(crate) struct SearchProcessingConsumerGroup;
@@ -41,7 +45,9 @@ type SearchProcessingKafkaAdapter =
type SearchProcessingKafkaConsumer =
MacroEventConsumerService<SearchProcessingBrokerEvent, SearchProcessingKafkaAdapter>;
-macro_event_broker::declare_topics!(SearchProcessingBrokerEvent: CallMacroEvent);
+macro_event_broker::declare_topics!(
+ SearchProcessingBrokerEvent: CallMacroEvent, ChannelMacroEvent
+);
/// Maximum number of decoded events waiting for the sequential worker.
const CHANNEL_CAPACITY: usize = 128;
@@ -73,6 +79,21 @@ struct CallEventDescription {
event_type: &'static str,
}
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+enum ChannelIndexAction {
+ UpsertMessage { channel_id: Uuid, message_id: Uuid },
+ RemoveMessage { channel_id: Uuid, message_id: Uuid },
+ RemoveChannel { channel_id: Uuid },
+ Ignore,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+struct ChannelEventDescription {
+ action: ChannelIndexAction,
+ channel_id: Uuid,
+ event_type: &'static str,
+}
+
/// Outcome after the worker handles one decoded broker event.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum EventOutcome {
@@ -140,6 +161,78 @@ fn describe_call_event(event: &CallTopicEvent) -> CallEventDescription {
}
}
+fn describe_channel_event(event: &ChannelTopicEvent) -> ChannelEventDescription {
+ match event {
+ ChannelTopicEvent::Created(metadata) => ChannelEventDescription {
+ action: ChannelIndexAction::Ignore,
+ channel_id: metadata.channel_id,
+ event_type: "channel.created",
+ },
+ ChannelTopicEvent::Updated(metadata) => ChannelEventDescription {
+ action: ChannelIndexAction::Ignore,
+ channel_id: metadata.channel_id,
+ event_type: "channel.updated",
+ },
+ ChannelTopicEvent::Deleted(metadata) => ChannelEventDescription {
+ action: ChannelIndexAction::RemoveChannel {
+ channel_id: metadata.channel_id,
+ },
+ channel_id: metadata.channel_id,
+ event_type: "channel.deleted",
+ },
+ ChannelTopicEvent::MessagePosted(metadata) => ChannelEventDescription {
+ action: ChannelIndexAction::UpsertMessage {
+ channel_id: metadata.channel_id,
+ message_id: metadata.message_id,
+ },
+ channel_id: metadata.channel_id,
+ event_type: "channel.message_posted",
+ },
+ ChannelTopicEvent::MessagePatched(metadata) => ChannelEventDescription {
+ action: ChannelIndexAction::UpsertMessage {
+ channel_id: metadata.channel_id,
+ message_id: metadata.message_id,
+ },
+ channel_id: metadata.channel_id,
+ event_type: "channel.message_patched",
+ },
+ ChannelTopicEvent::MessageDeleted(metadata) => ChannelEventDescription {
+ action: ChannelIndexAction::RemoveMessage {
+ channel_id: metadata.channel_id,
+ message_id: metadata.message_id,
+ },
+ channel_id: metadata.channel_id,
+ event_type: "channel.message_deleted",
+ },
+ ChannelTopicEvent::MessageAttachmentCreated(metadata) => ChannelEventDescription {
+ action: ChannelIndexAction::UpsertMessage {
+ channel_id: metadata.channel_id,
+ message_id: metadata.message_id,
+ },
+ channel_id: metadata.channel_id,
+ event_type: "channel.message_attachment_created",
+ },
+ ChannelTopicEvent::MessageAttachmentRemoved(metadata) => ChannelEventDescription {
+ action: ChannelIndexAction::UpsertMessage {
+ channel_id: metadata.channel_id,
+ message_id: metadata.message_id,
+ },
+ channel_id: metadata.channel_id,
+ event_type: "channel.message_attachment_removed",
+ },
+ ChannelTopicEvent::ParticipantAdded(metadata) => ChannelEventDescription {
+ action: ChannelIndexAction::Ignore,
+ channel_id: metadata.channel_id,
+ event_type: "channel.participant_added",
+ },
+ ChannelTopicEvent::ParticipantRemoved(metadata) => ChannelEventDescription {
+ action: ChannelIndexAction::Ignore,
+ channel_id: metadata.channel_id,
+ event_type: "channel.participant_removed",
+ },
+ }
+}
+
fn processing_retry_strategy() -> impl Iterator<Item = Duration> {
ExponentialBackoff::from_millis(2)
.factor(500)
@@ -215,6 +308,33 @@ async fn process_call_index_action(
}
}
+async fn process_channel_index_action(
+ db: &PgPool,
+ opensearch_client: &OpensearchClient,
+ action: ChannelIndexAction,
+) -> anyhow::Result<()> {
+ match action {
+ ChannelIndexAction::UpsertMessage {
+ channel_id,
+ message_id,
+ } => {
+ process_channel_message_update(opensearch_client, db, channel_id, message_id, None)
+ .await
+ }
+ ChannelIndexAction::RemoveMessage {
+ channel_id,
+ message_id,
+ } => {
+ process_remove_channel_message(opensearch_client, channel_id, Some(message_id), None)
+ .await
+ }
+ ChannelIndexAction::RemoveChannel { channel_id } => {
+ process_remove_channel_message(opensearch_client, channel_id, None, None).await
+ }
+ ChannelIndexAction::Ignore => Ok(()),
+ }
+}
+
async fn process_call_event(
db: &PgPool,
opensearch_client: &OpensearchClient,
@@ -281,6 +401,72 @@ async fn process_call_event(
}
}
+async fn process_channel_event(
+ db: &PgPool,
+ opensearch_client: &OpensearchClient,
+ event: &ChannelMacroEvent,
+ partition: i32,
+ offset: i64,
+) -> EventOutcome {
+ let description = describe_channel_event(&event.event().event);
+ if description.action == ChannelIndexAction::Ignore {
+ tracing::trace!(
+ channel_id = %description.channel_id,
+ event_type = description.event_type,
+ partition,
+ offset,
+ "ignoring channel event without a search-index action"
+ );
+ return EventOutcome::Ignored;
+ }
+
+ let result = retry_processing(|attempt| async move {
+ tracing::trace!(
+ channel_id = %description.channel_id,
+ event_type = description.event_type,
+ partition,
+ offset,
+ attempt,
+ "processing channel search-index event"
+ );
+ process_channel_index_action(db, opensearch_client, description.action)
+ .await
+ .inspect_err(|error| {
+ if attempt < MAX_PROCESSING_ATTEMPTS {
+ let retry_delay =
+ PROCESSING_RETRY_BASE_DELAY * 2u32.pow(attempt.saturating_sub(1));
+ tracing::warn!(
+ error = ?error,
+ channel_id = %description.channel_id,
+ event_type = description.event_type,
+ partition,
+ offset,
+ attempt,
+ delay_secs = retry_delay.as_secs(),
+ "channel search-index processing failed, retrying"
+ );
+ }
+ })
+ })
+ .await;
+
+ match result {
+ Ok(()) => EventOutcome::Indexed,
+ Err(error) => {
+ tracing::error!(
+ error = ?error,
+ channel_id = %description.channel_id,
+ event_type = description.event_type,
+ partition,
+ offset,
+ attempts = MAX_PROCESSING_ATTEMPTS,
+ "dropping channel event after processing retries were exhausted"
+ );
+ EventOutcome::Dropped
+ }
+ }
+}
+
#[tracing::instrument(skip(db, opensearch_client, event), fields(partition, offset))]
async fn process_event(
db: &PgPool,
@@ -293,6 +479,9 @@ async fn process_event(
SearchProcessingBrokerEvent::CallMacroEvent(event) => {
process_call_event(db, opensearch_client, event, partition, offset).await
}
+ SearchProcessingBrokerEvent::ChannelMacroEvent(event) => {
+ process_channel_event(db, opensearch_client, event, partition, offset).await
+ }
}
}
diff --git a/services/search_processing_service/src/inbound/kafka_consumer/test.rs b/services/search_processing_service/src/inbound/kafka_consumer/test.rs
index ebad2884ba..421e5a0dae 100644
--- a/services/search_processing_service/src/inbound/kafka_consumer/test.rs
+++ b/services/search_processing_service/src/inbound/kafka_consumer/test.rs
@@ -6,17 +6,29 @@ use call::domain::events::{
CallRecordSummarizedMetadata, CallRecordUpdatedMetadata, CallRecordingReadyMetadata,
CallStartedMetadata,
};
+use channels::domain::{
+ broker_events::{
+ ChannelCreatedMetadata, ChannelDeletedMetadata, ChannelMessageAttachmentCreatedMetadata,
+ ChannelMessageAttachmentRemovedMetadata, ChannelMessageDeletedMetadata,
+ ChannelMessagePatchedMetadata, ChannelMessagePostedMetadata,
+ ChannelParticipantAddedMetadata, ChannelParticipantRemovedMetadata, ChannelTopicEvent,
+ ChannelUpdatedMetadata,
+ },
+ models::{ChannelSender, ChannelType},
+};
use chrono::Utc;
use macro_event_broker::{Event, EventBrokerError, MacroEvent as _, MessageParts};
-use macro_event_topics::{MacroCallsTopic, Topic as _};
+use macro_event_topics::{MacroCallsTopic, MacroChannelsTopic, Topic as _};
use macro_user_id::user_id::MacroUserIdStr;
use super::*;
const CALL_ID: Uuid = Uuid::from_u128(1);
const CHANNEL_ID: Uuid = Uuid::from_u128(2);
+const MESSAGE_ID: Uuid = Uuid::from_u128(3);
struct TestMessage {
+ topic: &'static str,
key: Option<String>,
payload: Option<Vec<u8>>,
}
@@ -31,10 +43,14 @@ impl MessageParts for TestMessage {
}
fn topic(&self) -> &str {
- MacroCallsTopic::TOPIC_STR
+ self.topic
}
}
+fn channel_sender() -> ChannelSender<'static> {
+ ChannelSender::try_from("macro|[email protected]".to_string()).expect("valid channel sender")
+}
+
fn user_id() -> MacroUserIdStr<'static> {
MacroUserIdStr::try_from("macro|[email protected]".to_string()).expect("valid user id")
}
@@ -96,23 +112,190 @@ fn recording_ready_event() -> CallTopicEvent {
})
}
-fn encoded_message(event: Event<CallTopicEvent>) -> TestMessage {
+fn encoded_message<E: serde::Serialize>(
+ topic: &'static str,
+ key: Uuid,
+ event: Event<E>,
+) -> TestMessage {
TestMessage {
- key: Some(CALL_ID.to_string()),
- payload: Some(serde_json::to_vec(&event).expect("serializable call event")),
+ topic,
+ key: Some(key.to_string()),
+ payload: Some(serde_json::to_vec(&event).expect("serializable broker event")),
}
}
+fn channel_event_cases() -> Vec<(ChannelTopicEvent, ChannelEventDescription)> {
+ let sender = channel_sender();
+
+ vec![
+ (
+ ChannelTopicEvent::Created(ChannelCreatedMetadata {
+ channel_id: CHANNEL_ID,
+ actor: sender.clone(),
+ channel_type: ChannelType::Private,
+ channel_name: Some("general".to_string()),
+ participant_user_ids: vec![user_id()],
+ }),
+ ChannelEventDescription {
+ action: ChannelIndexAction::Ignore,
+ channel_id: CHANNEL_ID,
+ event_type: "channel.created",
+ },
+ ),
+ (
+ ChannelTopicEvent::Updated(ChannelUpdatedMetadata {
+ channel_id: CHANNEL_ID,
+ actor: user_id(),
+ previous_name: Some("general".to_string()),
+ channel_name: Some("renamed".to_string()),
+ }),
+ ChannelEventDescription {
+ action: ChannelIndexAction::Ignore,
+ channel_id: CHANNEL_ID,
+ event_type: "channel.updated",
+ },
+ ),
+ (
+ ChannelTopicEvent::Deleted(ChannelDeletedMetadata {
+ channel_id: CHANNEL_ID,
+ actor: sender.clone(),
+ }),
+ ChannelEventDescription {
+ action: ChannelIndexAction::RemoveChannel {
+ channel_id: CHANNEL_ID,
+ },
+ channel_id: CHANNEL_ID,
+ event_type: "channel.deleted",
+ },
+ ),
+ (
+ ChannelTopicEvent::MessagePosted(ChannelMessagePostedMetadata {
+ channel_id: CHANNEL_ID,
+ message_id: MESSAGE_ID,
+ thread_id: None,
+ sender: sender.clone(),
+ triggered_by: None,
+ channel_type: ChannelType::Private,
+ content: "hello".to_string(),
+ mentions: vec![],
+ attachments: vec![],
+ created_at: Utc::now(),
+ }),
+ ChannelEventDescription {
+ action: ChannelIndexAction::UpsertMessage {
+ channel_id: CHANNEL_ID,
+ message_id: MESSAGE_ID,
+ },
+ channel_id: CHANNEL_ID,
+ event_type: "channel.message_posted",
+ },
+ ),
+ (
+ ChannelTopicEvent::MessagePatched(ChannelMessagePatchedMetadata {
+ channel_id: CHANNEL_ID,
+ message_id: MESSAGE_ID,
+ thread_id: None,
+ actor: sender.clone(),
+ content: "edited".to_string(),
+ edited_at: Some(Utc::now()),
+ updated_at: Utc::now(),
+ }),
+ ChannelEventDescription {
+ action: ChannelIndexAction::UpsertMessage {
+ channel_id: CHANNEL_ID,
+ message_id: MESSAGE_ID,
+ },
+ channel_id: CHANNEL_ID,
+ event_type: "channel.message_patched",
+ },
+ ),
+ (
+ ChannelTopicEvent::MessageDeleted(ChannelMessageDeletedMetadata {
+ channel_id: CHANNEL_ID,
+ message_id: MESSAGE_ID,
+ thread_id: None,
+ actor: sender.clone(),
+ deleted_at: Some(Utc::now()),
+ }),
+ ChannelEventDescription {
+ action: ChannelIndexAction::RemoveMessage {
+ channel_id: CHANNEL_ID,
+ message_id: MESSAGE_ID,
+ },
+ channel_id: CHANNEL_ID,
+ event_type: "channel.message_deleted",
+ },
+ ),
+ (
+ ChannelTopicEvent::MessageAttachmentCreated(ChannelMessageAttachmentCreatedMetadata {
+ channel_id: CHANNEL_ID,
+ message_id: MESSAGE_ID,
+ actor: sender.clone(),
+ attachments: vec![],
+ }),
+ ChannelEventDescription {
+ action: ChannelIndexAction::UpsertMessage {
+ channel_id: CHANNEL_ID,
+ message_id: MESSAGE_ID,
+ },
+ channel_id: CHANNEL_ID,
+ event_type: "channel.message_attachment_created",
+ },
+ ),
+ (
+ ChannelTopicEvent::MessageAttachmentRemoved(ChannelMessageAttachmentRemovedMetadata {
+ channel_id: CHANNEL_ID,
+ message_id: MESSAGE_ID,
+ actor: sender.clone(),
+ attachments: vec![],
+ }),
+ ChannelEventDescription {
+ action: ChannelIndexAction::UpsertMessage {
+ channel_id: CHANNEL_ID,
+ message_id: MESSAGE_ID,
+ },
+ channel_id: CHANNEL_ID,
+ event_type: "channel.message_attachment_removed",
+ },
+ ),
+ (
+ ChannelTopicEvent::ParticipantAdded(ChannelParticipantAddedMetadata {
+ channel_id: CHANNEL_ID,
+ channel_type: ChannelType::Private,
+ added_by: sender,
+ added_user_ids: vec![user_id()],
+ }),
+ ChannelEventDescription {
+ action: ChannelIndexAction::Ignore,
+ channel_id: CHANNEL_ID,
+ event_type: "channel.participant_added",
+ },
+ ),
+ (
+ ChannelTopicEvent::ParticipantRemoved(ChannelParticipantRemovedMetadata {
+ channel_id: CHANNEL_ID,
+ channel_type: ChannelType::Private,
+ removed_by: user_id(),
+ removed_user_ids: vec![user_id()],
+ }),
+ ChannelEventDescription {
+ action: ChannelIndexAction::Ignore,
+ channel_id: CHANNEL_ID,
+ event_type: "channel.participant_removed",
+ },
+ ),
+ ]
+}
+
#[test]
fn subscribes_to_declared_search_processing_topics_with_durable_group() {
assert_eq!(
SearchProcessingConsumerGroup::GROUP_NAME,
"search-processing-service"
);
- assert_eq!(
- SearchProcessingBrokerEvent::topics(),
- [MacroCallsTopic::TOPIC_STR]
- );
+ let topics = SearchProcessingBrokerEvent::topics();
+ assert!(topics.contains(&MacroCallsTopic::TOPIC_STR));
+ assert!(topics.contains(&MacroChannelsTopic::TOPIC_STR));
}
#[test]
@@ -170,9 +353,74 @@ fn maps_all_call_lifecycle_events_to_index_actions() {
);
}
+#[test]
+fn maps_all_channel_lifecycle_events_to_index_actions() {
+ let cases = channel_event_cases();
+ assert_eq!(cases.len(), 10);
+
+ for (event, expected) in cases {
+ let serialized = serde_json::to_value(&event).expect("serializable channel event");
+ assert_eq!(serialized["event_type"], expected.event_type);
+ assert_eq!(describe_channel_event(&event), expected);
+ }
+}
+
+#[test]
+fn channel_envelope_decodes_round_trip() {
+ let event = ChannelTopicEvent::Deleted(ChannelDeletedMetadata {
+ channel_id: CHANNEL_ID,
+ actor: channel_sender(),
+ });
+ let message = encoded_message(
+ MacroChannelsTopic::TOPIC_STR,
+ CHANNEL_ID,
+ Event::new(event.clone()),
+ );
+
+ let decoded = SearchProcessingBrokerEvent::decode(&message).expect("decodable channel event");
+ let SearchProcessingBrokerEvent::ChannelMacroEvent(decoded_event) = decoded else {
+ panic!("expected channel event");
+ };
+ assert_eq!(decoded_event.key(), CHANNEL_ID.to_string());
+ assert_eq!(decoded_event.event().event, event);
+}
+
+#[tokio::test]
+async fn unsupported_channel_schema_message_is_commit_safe() {
+ let event = ChannelTopicEvent::Deleted(ChannelDeletedMetadata {
+ channel_id: CHANNEL_ID,
+ actor: channel_sender(),
+ });
+ let message = encoded_message(
+ MacroChannelsTopic::TOPIC_STR,
+ CHANNEL_ID,
+ Event::with_schema_version(event, 2),
+ );
+ let decoded = attach_event_coordinates(SearchProcessingBrokerEvent::decode(&message), 2, 20);
+ let (sender, mut receiver) = mpsc::channel(1);
+
+ match handoff_decoded(&sender, decoded).await {
+ HandoffOutcome::MalformedRecord(EventBrokerError::UnsupportedSchemaVersion {
+ topic,
+ expected,
+ actual,
+ }) => {
+ assert_eq!(topic, MacroChannelsTopic::TOPIC_STR);
+ assert_eq!(expected, 1);
+ assert_eq!(actual, 2);
+ }
+ outcome => panic!("expected malformed channel record, got {outcome:?}"),
+ }
+ assert!(matches!(
+ receiver.try_recv(),
+ Err(mpsc::error::TryRecvError::Empty)
+ ));
+}
+
#[tokio::test]
async fn malformed_missing_key_and_unsupported_schema_messages_are_commit_safe() {
let malformed = TestMessage {
+ topic: MacroCallsTopic::TOPIC_STR,
key: Some(CALL_ID.to_string()),
payload: Some(b"not json".to_vec()),
};
@@ -181,8 +429,14 @@ async fn malformed_missing_key_and_unsupported_schema_messages_are_commit_safe()
assert!(matches!(malformed, Err(EventBrokerError::Serialization(_))));
let missing_key = TestMessage {
+ topic: MacroCallsTopic::TOPIC_STR,
key: None,
- payload: encoded_message(Event::new(archived_event())).payload,
+ payload: encoded_message(
+ MacroCallsTopic::TOPIC_STR,
+ CALL_ID,
+ Event::new(archived_event()),
+ )
+ .payload,
};
let missing_key =
attach_event_coordinates(SearchProcessingBrokerEvent::decode(&missing_key), 1, 11);
@@ -191,7 +445,11 @@ async fn malformed_missing_key_and_unsupported_schema_messages_are_commit_safe()
Err(EventBrokerError::MissingMessageKey)
));
- let unsupported = encoded_message(Event::with_schema_version(archived_event(), 2));
+ let unsupported = encoded_message(
+ MacroCallsTopic::TOPIC_STR,
+ CALL_ID,
+ Event::with_schema_version(archived_event(), 2),
+ );
let unsupported =
attach_event_coordinates(SearchProcessingBrokerEvent::decode(&unsupported), 1, 12);
assert!(matches!(
@@ -219,7 +477,11 @@ async fn malformed_missing_key_and_unsupported_schema_messages_are_commit_safe()
#[tokio::test]
async fn successful_handoff_carries_event_partition_and_offset() {
let event = archived_event();
- let message = encoded_message(Event::new(event.clone()));
+ let message = encoded_message(
+ MacroCallsTopic::TOPIC_STR,
+ CALL_ID,
+ Event::new(event.clone()),
+ );
let decoded = attach_event_coordinates(SearchProcessingBrokerEvent::decode(&message), 3, 42);
let (sender, mut receiver) = mpsc::channel(1);
@@ -231,13 +493,19 @@ async fn successful_handoff_carries_event_partition_and_offset() {
let received = receiver.recv().await.expect("handed-off event");
assert_eq!(received.partition, 3);
assert_eq!(received.offset, 42);
- let SearchProcessingBrokerEvent::CallMacroEvent(received_event) = received.event;
+ let SearchProcessingBrokerEvent::CallMacroEvent(received_event) = received.event else {
+ panic!("expected call event");
+ };
assert_eq!(received_event.event().event, event);
}
#[tokio::test]
async fn closed_worker_channel_leaves_the_current_message_uncommitted() {
- let message = encoded_message(Event::new(archived_event()));
+ let message = encoded_message(
+ MacroCallsTopic::TOPIC_STR,
+ CALL_ID,
+ Event::new(archived_event()),
+ );
let decoded = attach_event_coordinates(SearchProcessingBrokerEvent::decode(&message), 3, 42);
let (sender, receiver) = mpsc::channel(1);
drop(receiver);
diff --git a/services/search_processing_service/src/process/channel.rs b/services/search_processing_service/src/process/channel.rs
index 6470dac707..e30eb4d3f9 100644
--- a/services/search_processing_service/src/process/channel.rs
+++ b/services/search_processing_service/src/process/channel.rs
@@ -4,17 +4,15 @@ use opensearch_client::{
OpensearchClient, date_format::EpochMillis, upsert::channel_message::UpsertChannelMessageArgs,
};
use sqlx::{Pool, Postgres};
-use sqs_client::search::channel::{ChannelMessageUpdate, RemoveChannelMessage};
use uuid::Uuid;
pub async fn process_channel_message_update(
opensearch_client: &OpensearchClient,
db: &Pool<Postgres>,
- message: &ChannelMessageUpdate,
+ channel_id: Uuid,
+ message_id: Uuid,
+ index_override: Option<&str>,
) -> anyhow::Result<()> {
- let channel_id = Uuid::parse_str(&message.channel_id).context("invalid channel_id uuid")?;
- let message_id = Uuid::parse_str(&message.message_id).context("invalid message_id uuid")?;
-
let channel_message_info =
comms_db_client::messages::get_channel_message::get_channel_message_by_id(
db,
@@ -24,11 +22,12 @@ pub async fn process_channel_message_update(
.await
.context("unable to get channel message")?;
- let index_override = message.index_override.as_deref();
if channel_message_info.channel_message.deleted_at.is_some() {
tracing::trace!("channel message is deleted, removing from search index");
+ let channel_id = channel_id.to_string();
+ let message_id = message_id.to_string();
opensearch_client
- .delete_channel_message(&message.channel_id, &message.message_id, index_override)
+ .delete_channel_message(&channel_id, &message_id, index_override)
.await?;
return Ok(());
}
@@ -78,17 +77,20 @@ pub async fn process_channel_message_update(
pub async fn process_remove_channel_message(
opensearch_client: &OpensearchClient,
- message: &RemoveChannelMessage,
+ channel_id: Uuid,
+ message_id: Option<Uuid>,
+ index_override: Option<&str>,
) -> anyhow::Result<()> {
- let index_override = message.index_override.as_deref();
- if let Some(message_id) = &message.message_id {
+ let channel_id = channel_id.to_string();
+ if let Some(message_id) = message_id {
+ let message_id = message_id.to_string();
opensearch_client
- .delete_channel_message(&message.channel_id, message_id, index_override)
+ .delete_channel_message(&channel_id, &message_id, index_override)
.await?;
} else {
tracing::trace!("message id is empty, deleting channel");
opensearch_client
- .delete_channel(&message.channel_id, index_override)
+ .delete_channel(&channel_id, index_override)
.await?;
}
diff --git a/services/search_processing_service/src/process/mod.rs b/services/search_processing_service/src/process/mod.rs
index 954539ecbe..6702c24158 100644
--- a/services/search_processing_service/src/process/mod.rs
+++ b/services/search_processing_service/src/process/mod.rs
@@ -1,5 +1,5 @@
pub(crate) mod call;
-mod channel;
+pub(crate) mod channel;
mod chat;
pub mod context;
mod document;
@@ -40,11 +40,22 @@ pub async fn process_message(
user::remove_user_profile(&ctx.opensearch_client, &user_profile_id).await?;
}
SearchQueueMessage::ChannelMessageUpdate(message) => {
- channel::process_channel_message_update(&ctx.opensearch_client, &ctx.db, &message)
- .await?;
- }
- SearchQueueMessage::RemoveChannelMessage(message) => {
- channel::process_remove_channel_message(&ctx.opensearch_client, &message).await?;
+ let channel_id = message
+ .channel_id
+ .parse::<Uuid>()
+ .context("failed to parse channel_id as UUID")?;
+ let message_id = message
+ .message_id
+ .parse::<Uuid>()
+ .context("failed to parse message_id as UUID")?;
+ channel::process_channel_message_update(
+ &ctx.opensearch_client,
+ &ctx.db,
+ channel_id,
+ message_id,
+ message.index_override.as_deref(),
+ )
+ .await?;
}
SearchQueueMessage::RemoveEmailLink(message) => {
email::remove::process_remove_messages_by_link_id(&ctx.opensearch_client, &message)