Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(search_processing_service): chats kafka (#5304)
Will Hutchinson7183fca6f74b
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.jsonindex 09e2587f83..da5f74eea2 100644--- a/.github/workspace-dep-closures.json+++ b/.github/workspace-dep-closures.json@@ -1304,6 +1304,7 @@"crates/authentication_service_client","crates/bot_id","crates/channel_sender",+ "crates/chat","crates/cowlike","crates/crm","crates/document_sub_type",@@ -1359,6 +1360,7 @@"crates/properties","crates/rate_limit","crates/remote_env_var",+ "crates/roles_and_permissions","crates/s3_key","crates/sqs_client","crates/system_properties",@@ -4924,6 +4926,7 @@"crates/call","crates/channel_sender","crates/channels",+ "crates/chat","crates/comms_db_client","crates/connection","crates/connection_gateway_client",@@ -4996,6 +4999,7 @@"crates/properties","crates/rate_limit","crates/remote_env_var",+ "crates/roles_and_permissions","crates/s3_client","crates/s3_key","crates/share_permission_db_utils",diff --git a/.sqlx/query-92e2916a16d165600f457c778ac3db9ae11c6625c948798e19ca19da4a30f3c1.json b/.sqlx/query-eb360ba6aad69fe2c501baa8b61b4243051bd3bde6e517014cd5768334fd7757.jsonsimilarity index 51%rename from .sqlx/query-92e2916a16d165600f457c778ac3db9ae11c6625c948798e19ca19da4a30f3c1.jsonrename to .sqlx/query-eb360ba6aad69fe2c501baa8b61b4243051bd3bde6e517014cd5768334fd7757.jsonindex f2a02380a7..d6a8a3072f 100644--- a/.sqlx/query-92e2916a16d165600f457c778ac3db9ae11c6625c948798e19ca19da4a30f3c1.json+++ b/.sqlx/query-eb360ba6aad69fe2c501baa8b61b4243051bd3bde6e517014cd5768334fd7757.json@@ -1,6 +1,6 @@{"db_name": "PostgreSQL",- "query": "\n SELECT\n m.content as \"content\",\n c.name as \"name\",\n m.role as \"role\",\n c.\"deletedAt\"::timestamptz as \"deleted_at\"\n FROM\n \"ChatMessage\" m\n JOIN\n \"Chat\" c on c.\"id\" = m.\"chatId\"\n WHERE\n m.id = $1 AND m.\"chatId\" = $2\n ",+ "query": "\n SELECT\n m.content as \"content\",\n c.name as \"name\",\n m.role as \"role\",\n c.\"deletedAt\"::timestamptz as \"deleted_at\",\n c.\"userId\" as \"owner_user_id\",\n m.\"createdAt\" as \"created_at\",\n m.\"updatedAt\" as \"updated_at\"\n FROM\n \"ChatMessage\" m\n JOIN\n \"Chat\" c on c.\"id\" = m.\"chatId\"\n WHERE\n m.id = $1 AND m.\"chatId\" = $2\n ","describe": {"columns": [{@@ -22,6 +22,21 @@"ordinal": 3,"name": "deleted_at","type_info": "Timestamptz"+ },+ {+ "ordinal": 4,+ "name": "owner_user_id",+ "type_info": "Text"+ },+ {+ "ordinal": 5,+ "name": "created_at",+ "type_info": "Timestamp"+ },+ {+ "ordinal": 6,+ "name": "updated_at",+ "type_info": "Timestamp"}],"parameters": {@@ -34,8 +49,11 @@false,false,false,- null+ null,+ false,+ false,+ false]},- "hash": "92e2916a16d165600f457c778ac3db9ae11c6625c948798e19ca19da4a30f3c1"+ "hash": "eb360ba6aad69fe2c501baa8b61b4243051bd3bde6e517014cd5768334fd7757"}diff --git a/Cargo.lock b/Cargo.lockindex a33935b223..f5d94018ca 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -3995,6 +3995,7 @@ dependencies = ["aws-sdk-sqs","aws-smithy-types","aws_lambda_events",+ "chat","chrono","futures","lambda_runtime",@@ -12888,6 +12889,7 @@ dependencies = ["axum","call","channels",+ "chat","chrono","comms_db_client","document_sub_type",diff --git a/crates/macro_db_client/fixtures/chat_message_info.sql b/crates/macro_db_client/fixtures/chat_message_info.sqlindex 02bc75042c..0bf201b794 100644--- a/crates/macro_db_client/fixtures/chat_message_info.sql+++ b/crates/macro_db_client/fixtures/chat_message_info.sql@@ -4,12 +4,14 @@ VALUES ('a1111111-1111-1111-1111-111111111111', 'user', '[email protected]', 'stripeINSERT INTO public."User" ("id","email","stripeCustomerId","macro_user_id")VALUES ('macro|[email protected]', '[email protected]','stripe_id','a1111111-1111-1111-1111-111111111111');-INSERT INTO public."Chat" ("id","name","userId","model","createdAt","updatedAt","isPersistent")+INSERT INTO public."Chat" ("id","name","userId","model","createdAt","updatedAt","deletedAt","isPersistent")VALUES- ('chat-persistent', 'persistent chat', 'macro|[email protected]', 'gpt-4o', '2024-01-01 00:00:00', '2024-01-01 00:00:00', true),- ('chat-ephemeral', 'ephemeral chat', 'macro|[email protected]', 'gpt-4o', '2024-01-01 00:00:00', '2024-01-01 00:00:00', false);+ ('chat-persistent', 'persistent chat', 'macro|[email protected]', 'gpt-4o', '2024-01-01 00:00:00', '2024-01-01 00:00:00', NULL, true),+ ('chat-ephemeral', 'ephemeral chat', 'macro|[email protected]', 'gpt-4o', '2024-02-01 00:00:00', '2024-02-01 00:00:00', NULL, false),+ ('chat-deleted', 'deleted chat', 'macro|[email protected]', 'gpt-4o', '2024-03-01 00:00:00', '2024-03-01 00:00:00', '2024-03-04 05:06:07.890', true);-INSERT INTO public."ChatMessage" ("id","content","role","chatId")+INSERT INTO public."ChatMessage" ("id","content","role","chatId","createdAt","updatedAt")VALUES- ('msg-persistent', '"codebase brighter"', 'user', 'chat-persistent'),- ('msg-ephemeral', '"another message"', 'user', 'chat-ephemeral');+ ('msg-persistent', '"codebase brighter"', 'user', 'chat-persistent', '2024-01-02 03:04:05.123', '2024-01-03 04:05:06.789'),+ ('msg-ephemeral', '"another message"', 'assistant', 'chat-ephemeral', '2024-02-02 03:04:05.123', '2024-02-03 04:05:06.789'),+ ('msg-deleted', '"remove from search"', 'assistant', 'chat-deleted', '2024-03-02 03:04:05.123', '2024-03-03 04:05:06.789');diff --git a/crates/macro_db_client/src/chat/get.rs b/crates/macro_db_client/src/chat/get.rsindex 02cc071baf..cdda6dfe2a 100644--- a/crates/macro_db_client/src/chat/get.rs+++ b/crates/macro_db_client/src/chat/get.rs@@ -258,6 +258,9 @@ pub struct ChatMessageInfo {pub name: String,pub content: String,pub role: String,+ pub owner_user_id: String,+ pub created_at: DateTime<Utc>,+ pub updated_at: DateTime<Utc>,pub deleted_at: Option<DateTime<Utc>>,}@@ -277,7 +280,10 @@ pub async fn get_chat_message_info(m.content as "content",c.name as "name",m.role as "role",- c."deletedAt"::timestamptz as "deleted_at"+ c."deletedAt"::timestamptz as "deleted_at",+ c."userId" as "owner_user_id",+ m."createdAt" as "created_at",+ m."updatedAt" as "updated_at"FROM"ChatMessage" mJOIN@@ -300,6 +306,9 @@ pub async fn get_chat_message_info(name: row.name,content,role: row.role,+ owner_user_id: row.owner_user_id,+ created_at: DateTime::<Utc>::from_naive_utc_and_offset(row.created_at, Utc),+ updated_at: DateTime::<Utc>::from_naive_utc_and_offset(row.updated_at, Utc),deleted_at: row.deleted_at,}))}diff --git a/crates/macro_db_client/src/chat/get/test.rs b/crates/macro_db_client/src/chat/get/test.rsindex 4533d82ec7..633eab9cc6 100644--- a/crates/macro_db_client/src/chat/get/test.rs+++ b/crates/macro_db_client/src/chat/get/test.rs@@ -2,20 +2,86 @@ use super::*;use sqlx::{Pool, Postgres};#[sqlx::test(fixtures(path = "../../../fixtures", scripts("chat_message_info")))]-async fn persistent_chat_messages_are_indexed_for_search(+async fn returns_persistent_and_ephemeral_chat_message_metadata(pool: Pool<Postgres>,) -> anyhow::Result<()> {let persistent = get_chat_message_info(&pool, "chat-persistent", "msg-persistent").await?.expect("persistent chat message should be returned for indexing");+ assert_eq!(persistent.name, "persistent chat");assert_eq!(persistent.content, "codebase brighter");+ assert_eq!(persistent.role, "user");+ assert_eq!(persistent.owner_user_id, "macro|[email protected]");+ assert_eq!(+ persistent.created_at,+ "2024-01-02T03:04:05.123Z".parse::<DateTime<Utc>>()?+ );+ assert_eq!(+ persistent.updated_at,+ "2024-01-03T04:05:06.789Z".parse::<DateTime<Utc>>()?+ );assert!(persistent.deleted_at.is_none());let ephemeral = get_chat_message_info(&pool, "chat-ephemeral", "msg-ephemeral").await?.expect("ephemeral chat message should be returned for indexing");+ assert_eq!(ephemeral.name, "ephemeral chat");assert_eq!(ephemeral.content, "another message");+ assert_eq!(ephemeral.role, "assistant");+ assert_eq!(ephemeral.owner_user_id, "macro|[email protected]");+ assert_eq!(+ ephemeral.created_at,+ "2024-02-02T03:04:05.123Z".parse::<DateTime<Utc>>()?+ );+ assert_eq!(+ ephemeral.updated_at,+ "2024-02-03T04:05:06.789Z".parse::<DateTime<Utc>>()?+ );assert!(ephemeral.deleted_at.is_none());Ok(())}++#[sqlx::test(fixtures(path = "../../../fixtures", scripts("chat_message_info")))]+async fn returns_soft_deleted_chat_message_metadata(pool: Pool<Postgres>) -> anyhow::Result<()> {+ let message = get_chat_message_info(&pool, "chat-deleted", "msg-deleted")+ .await?+ .expect("soft-deleted chat message should be returned for removal");++ assert_eq!(message.name, "deleted chat");+ assert_eq!(message.content, "remove from search");+ assert_eq!(message.role, "assistant");+ assert_eq!(message.owner_user_id, "macro|[email protected]");+ assert_eq!(+ message.created_at,+ "2024-03-02T03:04:05.123Z".parse::<DateTime<Utc>>()?+ );+ assert_eq!(+ message.updated_at,+ "2024-03-03T04:05:06.789Z".parse::<DateTime<Utc>>()?+ );+ assert_eq!(+ message.deleted_at,+ Some("2024-03-04T05:06:07.890Z".parse::<DateTime<Utc>>()?)+ );++ Ok(())+}++#[sqlx::test(fixtures(path = "../../../fixtures", scripts("chat_message_info")))]+async fn returns_none_for_missing_or_mismatched_message_id(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ assert!(+ get_chat_message_info(&pool, "chat-persistent", "msg-missing")+ .await?+ .is_none()+ );+ assert!(+ get_chat_message_info(&pool, "chat-ephemeral", "msg-persistent")+ .await?+ .is_none()+ );++ Ok(())+}diff --git a/crates/projects/src/domain/ports.rs b/crates/projects/src/domain/ports.rsindex 5ec4e2e3cb..6031b00a16 100644--- a/crates/projects/src/domain/ports.rs+++ b/crates/projects/src/domain/ports.rs@@ -206,15 +206,9 @@ pub trait ShaCounterPort: Send + Sync + 'static {) -> impl Future<Output = anyhow::Result<()>> + Send;}-/// Port for publishing cross-family search and document-deletion work.-// TODO: Remove this port and its SQS adapter after the chats and documents migrations.+/// Port for publishing document search-removal and deletion work.+// TODO: Remove this port and its SQS adapter after the documents migration.pub trait ProjectSearchIndexer: Send + Sync + 'static {- /// Remove chats from the search index.- fn remove_chats(- &self,- chat_ids: Vec<String>,- ) -> impl Future<Output = anyhow::Result<()>> + Send;-/// Remove documents from the search index.fn remove_documents(&self,diff --git a/crates/projects/src/domain/service.rs b/crates/projects/src/domain/service.rsindex e63707d81f..2016320a17 100644--- a/crates/projects/src/domain/service.rs+++ b/crates/projects/src/domain/service.rs@@ -77,7 +77,7 @@ wherepub sha_counter: Sha,/// Entity-access inheritance manager.pub entity_access_management_service: Eam,- /// Search and deletion queue publisher.+ /// Document search and deletion queue publisher.pub search_indexer: Idx,/// Project lifecycle event broker.pub macro_event_broker: B,@@ -534,13 +534,6 @@ where.map(|(document_id, _)| document_id.clone()).collect::<Vec<_>>();- if !purged.chat_ids.is_empty() {- let _ = self- .search_indexer- .remove_chats(purged.chat_ids)- .await- .inspect_err(|error| tracing::error!(error = ?error, "unable to enqueue purged chats for search"));- }if !purged.documents.is_empty() {let _ = self.search_indexerdiff --git a/crates/projects/src/domain/service/tests.rs b/crates/projects/src/domain/service/tests.rsindex b986ddb57c..5d2d5a7a82 100644--- a/crates/projects/src/domain/service/tests.rs+++ b/crates/projects/src/domain/service/tests.rs@@ -153,10 +153,6 @@ impl EntityAccessManagementService for NullPort {}impl ProjectSearchIndexer for NullPort {- async fn remove_chats(&self, _chat_ids: Vec<String>) -> anyhow::Result<()> {- unreachable!()- }-async fn remove_documents(&self, _document_ids: Vec<String>) -> anyhow::Result<()> {unreachable!()}@@ -224,10 +220,6 @@ struct RecordingIndexer {}impl ProjectSearchIndexer for RecordingIndexer {- async fn remove_chats(&self, _chat_ids: Vec<String>) -> anyhow::Result<()> {- unreachable!()- }-async fn remove_documents(&self, _document_ids: Vec<String>) -> anyhow::Result<()> {unreachable!()}@@ -1574,10 +1566,6 @@ impl OrderedIndexer {}impl ProjectSearchIndexer for OrderedIndexer {- async fn remove_chats(&self, _chat_ids: Vec<String>) -> anyhow::Result<()> {- self.record("chats")- }-async fn remove_documents(&self, _document_ids: Vec<String>) -> anyhow::Result<()> {self.record("documents")}@@ -1740,7 +1728,7 @@ async fn permanent_delete_runs_external_work_after_committed_purge() {assert_eq!(*events.lock().unwrap(),- vec!["repo", "sha", "chats", "documents", "document_deletes"]+ vec!["repo", "sha", "documents", "document_deletes"]);let published = published.lock().unwrap();assert_eq!(published.len(), 1);diff --git a/crates/projects/src/outbound/sqs_search_indexer.rs b/crates/projects/src/outbound/sqs_search_indexer.rsindex 447429bc2c..4455625854 100644--- a/crates/projects/src/outbound/sqs_search_indexer.rs+++ b/crates/projects/src/outbound/sqs_search_indexer.rs@@ -1,12 +1,12 @@-//! SQS adapter for cross-family search cleanup and document deletion.+//! SQS adapter for document search cleanup and deletion.use std::sync::Arc;-use sqs_client::search::{SearchQueueMessage, chat::RemoveChatMessage, document::DocumentId};+use sqs_client::search::{SearchQueueMessage, document::DocumentId};use crate::domain::ports::ProjectSearchIndexer;-/// SQS-backed cross-family search cleanup and document-deletion adapter.+/// SQS-backed document search cleanup and deletion adapter.#[derive(Clone)]pub struct SqsProjectSearchIndexer {sqs: Arc<sqs_client::SQS>,@@ -20,24 +20,6 @@ impl SqsProjectSearchIndexer {}impl ProjectSearchIndexer for SqsProjectSearchIndexer {- #[tracing::instrument(skip(self), err)]- async fn remove_chats(&self, chat_ids: Vec<String>) -> anyhow::Result<()> {- let messages = chat_ids- .into_iter()- .map(|chat_id| {- SearchQueueMessage::RemoveChatMessage(RemoveChatMessage {- chat_id,- message_id: None,- index_override: None,- })- })- .collect();-- self.sqs- .bulk_send_message_to_search_event_queue(messages)- .await- }-#[tracing::instrument(skip(self), err)]async fn remove_documents(&self, document_ids: Vec<String>) -> anyhow::Result<()> {let messages = document_idsdiff --git a/crates/sqs_client/src/search/chat.rs b/crates/sqs_client/src/search/chat.rsindex 1180d5db73..f6f0b8dae3 100644--- a/crates/sqs_client/src/search/chat.rs+++ b/crates/sqs_client/src/search/chat.rs@@ -1,5 +1,13 @@use chrono::{DateTime, Utc};+#[cfg(test)]+mod test;++/// SQS backfill work-queue contract for reconciling a chat message.+///+/// Live chat updates use Kafka. This payload remains available so search+/// backfills can target an alternate OpenSearch index with+/// [`Self::index_override`].#[derive(serde::Serialize, serde::Deserialize, Debug, PartialEq, Eq)]pub struct ChatMessage {/// The chat id@@ -14,14 +22,3 @@ pub struct ChatMessage {#[serde(skip_serializing_if = "Option::is_none", default)]pub index_override: Option<String>,}--#[derive(serde::Serialize, serde::Deserialize, Debug, PartialEq, Eq)]-pub struct RemoveChatMessage {- /// The chat id to remove- pub chat_id: String,- /// The message id to remove, if None then all messages for the chat will be removed- pub message_id: Option<String>,- /// Optional override for the target OpenSearch index- #[serde(skip_serializing_if = "Option::is_none", default)]- pub index_override: Option<String>,-}diff --git a/crates/sqs_client/src/search/chat/test.rs b/crates/sqs_client/src/search/chat/test.rsnew file mode 100644index 0000000000..d7d850bf41--- /dev/null+++ b/crates/sqs_client/src/search/chat/test.rs@@ -0,0 +1,51 @@+use super::*;+use crate::{+ PrimaryId,+ search::{Operation, SearchQueueMessage},+};++#[test]+fn chat_message_with_index_override_round_trips() {+ let message = SearchQueueMessage::ChatMessage(ChatMessage {+ chat_id: "chat-1".to_string(),+ message_id: "message-1".to_string(),+ user_id: "user-1".to_string(),+ created_at: "2026-01-02T03:04:05Z".parse().unwrap(),+ updated_at: "2026-01-02T04:05:06Z".parse().unwrap(),+ index_override: Some("chats-backfill-v2".to_string()),+ });++ let serialized = serde_json::to_string(&message).unwrap();+ let deserialized: SearchQueueMessage = serde_json::from_str(&serialized).unwrap();++ assert_eq!(deserialized, message);+ assert_eq!(deserialized.id(), "message-1");+ assert!(matches!(deserialized.operation(), Operation::ExtractText));+}++#[test]+fn legacy_chat_backfill_without_index_override_deserializes() {+ let payload = r#"{+ "ChatMessage": {+ "chat_id": "chat-1",+ "message_id": "message-1",+ "user_id": "user-1",+ "created_at": "2026-01-02T03:04:05Z",+ "updated_at": "2026-01-02T04:05:06Z"+ }+ }"#;++ let message: SearchQueueMessage = serde_json::from_str(payload).unwrap();++ assert_eq!(+ message,+ SearchQueueMessage::ChatMessage(ChatMessage {+ chat_id: "chat-1".to_string(),+ message_id: "message-1".to_string(),+ user_id: "user-1".to_string(),+ created_at: "2026-01-02T03:04:05Z".parse().unwrap(),+ updated_at: "2026-01-02T04:05:06Z".parse().unwrap(),+ index_override: None,+ })+ );+}diff --git a/crates/sqs_client/src/search/mod.rs b/crates/sqs_client/src/search/mod.rsindex e1bd1f8bfc..44fd2d6b78 100644--- a/crates/sqs_client/src/search/mod.rs+++ b/crates/sqs_client/src/search/mod.rs@@ -3,7 +3,7 @@ use crate::{search::{call::{CallRecordMessage, RemoveCallRecord},channel::ChannelMessageUpdate,- chat::{ChatMessage, RemoveChatMessage},+ chat::ChatMessage,document::{DocumentId, DocumentPropertiesUpdate, SearchExtractorMessage},email::{EmailLinkMessage, EmailMessage, EmailThreadBatchMessage, EmailThreadMessage},project::UpsertProject,@@ -75,8 +75,8 @@ pub enum SearchQueueMessage {UpdateDocumentProperties(DocumentPropertiesUpdate),UpdateDocumentName(DocumentId),// Chat+ /// SQS backfill work-queue contract for reconciling a chat message.ChatMessage(ChatMessage),- RemoveChatMessage(RemoveChatMessage),ExtractEmailMessage(EmailMessage),RemoveEmailMessage(EmailMessage),@@ -103,9 +103,8 @@ impl PrimaryId for SearchQueueMessage {SearchQueueMessage::ExtractSync(message) => message.document_id.clone(),SearchQueueMessage::UpdateDocumentProperties(message) => message.document_id.clone(),SearchQueueMessage::UpdateDocumentName(message) => message.document_id.clone(),- SearchQueueMessage::ChatMessage(message) => message.message_id.clone(), // needs- // to be the message id to ensure it's unique for batch- SearchQueueMessage::RemoveChatMessage(message) => message.chat_id.clone(),+ // The message id keeps entries unique within an SQS batch.+ SearchQueueMessage::ChatMessage(message) => message.message_id.clone(),SearchQueueMessage::ExtractEmailMessage(message)| SearchQueueMessage::RemoveEmailMessage(message) => message.message_id.clone(),SearchQueueMessage::ExtractEmailThreadMessage(message) => message.thread_id.clone(),@@ -138,7 +137,6 @@ impl SearchQueueMessage {SearchQueueMessage::UpdateDocumentName(_) => Operation::UpdateMetadata,// ChatSearchQueueMessage::ChatMessage(_) => Operation::ExtractText,- SearchQueueMessage::RemoveChatMessage(_) => Operation::Remove,SearchQueueMessage::ExtractEmailMessage(_) => Operation::ExtractText,SearchQueueMessage::RemoveEmailMessage(_) => Operation::Remove,diff --git a/services/deleted_item_poller/Cargo.toml b/services/deleted_item_poller/Cargo.tomlindex eba4186fcf..1280e61cb1 100644--- a/services/deleted_item_poller/Cargo.toml+++ b/services/deleted_item_poller/Cargo.toml@@ -10,6 +10,7 @@ aws-sdk-sqs = { workspace = true }macro_aws_config = { path = "../../crates/macro_aws_config" }aws-smithy-types = { workspace = true }aws_lambda_events = { workspace = true, features = ["eventbridge"] }+chat = { path = "../../crates/chat", default-features = false }chrono = { workspace = true }futures = { workspace = true }lambda_runtime = { workspace = true }diff --git a/services/deleted_item_poller/src/config.rs b/services/deleted_item_poller/src/config.rsindex 5780c8e957..ee3db73e7b 100644--- a/services/deleted_item_poller/src/config.rs+++ b/services/deleted_item_poller/src/config.rs@@ -13,7 +13,7 @@ pub struct Config {/// The connection URL for the Postgres database this application should use.pub database_url: String,- /// Comma-separated Kafka bootstrap servers for project lifecycle events.+ /// Comma-separated Kafka bootstrap servers for project and chat lifecycle events.pub kafka_brokers: KafkaBrokers,/// The environment we are indiff --git a/services/deleted_item_poller/src/handler.rs b/services/deleted_item_poller/src/handler.rsindex 4599839b54..5fe629c4cf 100644--- a/services/deleted_item_poller/src/handler.rs+++ b/services/deleted_item_poller/src/handler.rs@@ -4,6 +4,7 @@ mod test;use crate::context::{self};use anyhow::Context;use aws_lambda_events::eventbridge::EventBridgeEvent;+use chat::domain::events::{ChatMacroEvent, ChatPermanentlyDeletedMetadata};use futures::future::join_all;use lambda_runtime::{Error, LambdaEvent,@@ -13,7 +14,7 @@ use macro_db_client::projects::ProjectToDelete;use macro_event_broker::MacroEventBroker;use macro_user_id::user_id::MacroUserIdStr;use projects::domain::events::{ProjectMacroEvent, ProjectPermanentlyDeletedMetadata};-use sqs_client::search::{SearchQueueMessage, chat::RemoveChatMessage, document::DocumentId};+use sqs_client::search::{SearchQueueMessage, document::DocumentId};#[tracing::instrument(skip(ctx, _event), err)]pub async fn handler(@@ -76,6 +77,42 @@ async fn publish_project_purge_events<B: MacroEventBroker>(Ok(())}+#[tracing::instrument(skip(event_broker, chat_ids), err)]+async fn publish_chat_purge_events<B: MacroEventBroker>(+ event_broker: &B,+ chat_ids: &[String],+) -> anyhow::Result<()> {+ let events = chat_ids+ .iter()+ .map(|chat_id| {+ ChatMacroEvent::permanently_deleted(ChatPermanentlyDeletedMetadata {+ chat_id: chat_id.clone(),+ actor_user_id: None,+ project_id: None,+ })+ })+ .collect::<Vec<_>>();++ let publications = events+ .iter()+ .map(|event| event_broker.send_event(event))+ .collect::<Vec<_>>();+ let publication_results = join_all(publications.into_iter().map(|publication| async move {+ let handle = publication.context("failed to enqueue chat purge event")?;+ handle+ .await+ .context("chat purge event publication task failed")?+ .context("failed to publish chat purge event")+ }))+ .await;++ for result in publication_results {+ result?;+ }++ Ok(())+}+#[tracing::instrument(skip(ctx), err)]async fn handle_projects(ctx: &context::Context) -> anyhow::Result<()> {let date = chrono::Utc::now().naive_utc() - chrono::Duration::days(30);@@ -121,20 +158,9 @@ async fn handle_chats(ctx: &context::Context) -> anyhow::Result<()> {tracing::debug!(chats_to_delete=?chats_to_delete, "chats to delete");- ctx.sqs_client- .bulk_send_message_to_search_event_queue(- chats_to_delete- .iter()- .map(|id| {- SearchQueueMessage::RemoveChatMessage(RemoveChatMessage {- chat_id: id.to_string(),- message_id: None,- index_override: None,- })- })- .collect(),- )- .await?;+ publish_chat_purge_events(&ctx.macro_event_broker, &chats_to_delete)+ .await+ .context("unable to publish chat purge events")?;ctx.sqs_client.bulk_enqueue_chat_delete(chats_to_delete)diff --git a/services/deleted_item_poller/src/handler/test.rs b/services/deleted_item_poller/src/handler/test.rsindex c342399143..10f77529c0 100644--- a/services/deleted_item_poller/src/handler/test.rs+++ b/services/deleted_item_poller/src/handler/test.rs@@ -2,6 +2,7 @@ use std::sync::{Arc, Mutex};use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker};use serde_json::{Value, json};+use tokio::sync::Semaphore;use super::*;@@ -12,16 +13,47 @@ struct PublishedEvent {envelope: Value,}+#[derive(Clone, Default)]+enum DeliveryBehavior {+ #[default]+ Succeed,+ Fail,+ FailToJoin,+ Wait(Arc<Semaphore>),+}+#[derive(Clone, Default)]struct FakeEventBroker {published: Arc<Mutex<Vec<PublishedEvent>>>,- fail_publication: bool,+ fail_send: bool,+ delivery_behavior: DeliveryBehavior,}impl FakeEventBroker {- fn failing() -> Self {+ fn failing_send() -> Self {+ Self {+ fail_send: true,+ ..Self::default()+ }+ }++ fn failing_delivery() -> Self {+ Self {+ delivery_behavior: DeliveryBehavior::Fail,+ ..Self::default()+ }+ }++ fn failing_join() -> Self {+ Self {+ delivery_behavior: DeliveryBehavior::FailToJoin,+ ..Self::default()+ }+ }++ fn waiting_for_delivery(delivery_gate: Arc<Semaphore>) -> Self {Self {- fail_publication: true,+ delivery_behavior: DeliveryBehavior::Wait(delivery_gate),..Self::default()}}@@ -36,22 +68,40 @@ impl MacroEventBroker for FakeEventBroker {&self,event: &E,) -> Result<tokio::task::JoinHandle<Result<(), EventBrokerError>>, EventBrokerError> {+ if self.fail_send {+ return Err(EventBrokerError::Publish(+ "event enqueue rejected".to_string(),+ ));+ }+self.published.lock().unwrap().push(PublishedEvent {topic: event.topic().to_string(),key: event.key().to_string(),envelope: serde_json::to_value(event.event())?,});- let fail_publication = self.fail_publication;- Ok(tokio::spawn(async move {- if fail_publication {+ let delivery_handle = match self.delivery_behavior.clone() {+ DeliveryBehavior::Succeed => tokio::spawn(async { Ok(()) }),+ DeliveryBehavior::Fail => tokio::spawn(async {Err(EventBrokerError::Publish("publisher unavailable".to_string(),))- } else {- Ok(())+ }),+ DeliveryBehavior::FailToJoin => {+ let handle = tokio::spawn(std::future::pending::<Result<(), EventBrokerError>>());+ handle.abort();+ handle}- }))+ DeliveryBehavior::Wait(delivery_gate) => tokio::spawn(async move {+ let _permit = delivery_gate+ .acquire_owned()+ .await+ .expect("delivery gate should remain open");+ Ok(())+ }),+ };++ Ok(delivery_handle)}}@@ -98,7 +148,7 @@ async fn publish_project_purge_events_publishes_separately_keyed_events() {#[tokio::test]async fn publish_project_purge_events_returns_publication_failures() {- let event_broker = FakeEventBroker::failing();+ let event_broker = FakeEventBroker::failing_delivery();let projects = vec![ProjectToDelete {project_id: "project-one".to_string(),user_id: "macro|[email protected]".to_string(),@@ -126,3 +176,92 @@ async fn publish_project_purge_events_rejects_malformed_owners() {assert!(format!("{error:#}").contains("invalid owner for project project-one"));assert!(event_broker.published().is_empty());}++#[tokio::test]+async fn publish_chat_purge_events_publishes_separately_keyed_events() {+ let event_broker = FakeEventBroker::default();+ let chat_ids = vec!["chat-one".to_string(), "chat-two".to_string()];++ publish_chat_purge_events(&event_broker, &chat_ids)+ .await+ .unwrap();++ let published = event_broker.published();+ assert_eq!(published.len(), chat_ids.len());++ for (event, chat_id) in published.iter().zip(&chat_ids) {+ assert_eq!(event.topic, "macro.chats");+ assert_eq!(event.key, *chat_id);+ assert_eq!(event.envelope["schema_version"], json!(1));+ assert_eq!(+ event.envelope["event_type"],+ json!("chat.permanently_deleted")+ );++ let metadata = &event.envelope["metadata"];+ assert_eq!(metadata["chat_id"], json!(chat_id));+ assert_eq!(metadata["actor_user_id"], Value::Null);+ assert_eq!(metadata["project_id"], Value::Null);+ }+}++#[tokio::test]+async fn publish_chat_purge_events_returns_immediate_send_failures() {+ let event_broker = FakeEventBroker::failing_send();+ let chat_ids = vec!["chat-one".to_string()];++ let error = publish_chat_purge_events(&event_broker, &chat_ids)+ .await+ .expect_err("immediate send failure should be returned");++ assert!(format!("{error:#}").contains("event enqueue rejected"));+}++#[tokio::test]+async fn publish_chat_purge_events_returns_delivery_failures() {+ let event_broker = FakeEventBroker::failing_delivery();+ let chat_ids = vec!["chat-one".to_string()];++ let error = publish_chat_purge_events(&event_broker, &chat_ids)+ .await+ .expect_err("delivery failure should be returned");++ assert!(format!("{error:#}").contains("publisher unavailable"));+}++#[tokio::test]+async fn publish_chat_purge_events_returns_delivery_join_failures() {+ let event_broker = FakeEventBroker::failing_join();+ let chat_ids = vec!["chat-one".to_string()];++ let error = publish_chat_purge_events(&event_broker, &chat_ids)+ .await+ .expect_err("delivery join failure should be returned");++ assert!(+ format!("{error:#}").contains("chat purge event publication task failed"),+ "unexpected error: {error:#}"+ );+}++#[tokio::test]+async fn publish_chat_purge_events_waits_for_every_delivery() {+ let delivery_gate = Arc::new(Semaphore::new(0));+ let event_broker = FakeEventBroker::waiting_for_delivery(delivery_gate.clone());+ let chat_ids = vec!["chat-one".to_string(), "chat-two".to_string()];++ let publication_task = tokio::spawn({+ let event_broker = event_broker.clone();+ let chat_ids = chat_ids.clone();+ async move { publish_chat_purge_events(&event_broker, &chat_ids).await }+ });++ while event_broker.published().len() < chat_ids.len() {+ tokio::task::yield_now().await;+ }++ assert!(!publication_task.is_finished());++ delivery_gate.add_permits(chat_ids.len());+ publication_task.await.unwrap().unwrap();+}diff --git a/services/document_cognition_service/Cargo.toml b/services/document_cognition_service/Cargo.tomlindex 0534e8b949..093d2d0621 100644--- a/services/document_cognition_service/Cargo.toml+++ b/services/document_cognition_service/Cargo.toml@@ -120,7 +120,6 @@ sqs_client = { path = "../../crates/sqs_client", default-features = false, featu"document_text_extractor","email","gmail",- "search",] }sqs_worker = { path = "../../crates/sqs_worker" }static_file = { path = "../../crates/static_file", features = ["attachment"] }diff --git a/services/document_cognition_service/src/api/stream/chat_message/mod.rs b/services/document_cognition_service/src/api/stream/chat_message/mod.rsindex 5e62614c59..e3f1ccb9c9 100644--- a/services/document_cognition_service/src/api/stream/chat_message/mod.rs+++ b/services/document_cognition_service/src/api/stream/chat_message/mod.rs@@ -251,7 +251,7 @@ async fn send_chat_message_inner(};// Store the incoming user message and resolve its attachments- let resolved = store_incoming_message(ctx.clone(), user_id.0.as_ref(), &chat, &model, &payload)+ let resolved = store_incoming_message(ctx.clone(), user_id.0.as_ref(), &model, &payload).await.map_err(|err| {tracing::error!(error=?err, "failed to store incoming message");@@ -674,7 +674,6 @@ fn stream_and_save_message(if let Err(err) = store_conversation_messages(ctx.clone(),- user_id.0.as_ref(),&chat_id,new_messages,&model,diff --git a/services/document_cognition_service/src/api/stream/util/chat_message/mod.rs b/services/document_cognition_service/src/api/stream/util/chat_message/mod.rsindex 520adc9ea3..6385cdc79e 100644--- a/services/document_cognition_service/src/api/stream/util/chat_message/mod.rs+++ b/services/document_cognition_service/src/api/stream/util/chat_message/mod.rs@@ -1,11 +1,7 @@pub mod ai_request;pub mod toolset;-use crate::model::chats::ChatResponse;-use crate::{- api::{context::ApiContext, utils::search},- model::stream::SendChatMessagePayload,-};+use crate::{api::context::ApiContext, model::stream::SendChatMessagePayload};use agent::types::Role;use agent::types::{ChatMessage, ChatMessageContent};@@ -17,11 +13,10 @@ use model_entity::EntityType;use std::sync::Arc;/// Stores the incoming user message and resolves its attachments in one step.-#[tracing::instrument(err, skip(ctx, chat, incoming_message), fields(chat_id=?incoming_message.chat_id))]+#[tracing::instrument(err, skip(ctx, incoming_message), fields(chat_id=?incoming_message.chat_id))]pub async fn store_incoming_message(ctx: Arc<ApiContext>,user_id: &str,- chat: &ChatResponse,model: &str,incoming_message: &SendChatMessagePayload,) -> Result<ResolvedMessageContent> {@@ -56,22 +51,10 @@ pub async fn store_incoming_message(let user_id: macro_user_id::user_id::MacroUserIdStr<'static> =user_id.to_owned().try_into().map_err(anyhow::Error::msg)?;- let resolved = ctx- .message_service+ ctx.message_service.create(&user_id, &incoming_message.chat_id, new_chat_message).await- .context("failed to create chat message")?;-- search::send_chat_message_to_search(- &ctx,- &chat.id,- &resolved.message_id,- user_id.as_ref(),- created_at,- created_at,- );-- Ok(resolved)+ .context("failed to create chat message")}/// Stores multiple conversation messages to the database.@@ -79,7 +62,6 @@ pub async fn store_incoming_message(#[tracing::instrument(err, skip(ctx, messages), fields(chat_id=?chat_id, message_count=messages.len()))]pub async fn store_conversation_messages(ctx: Arc<ApiContext>,- user_id: &str,chat_id: &str,messages: Vec<ChatMessage>,model: &str,@@ -118,16 +100,7 @@ pub async fn store_conversation_messages(.await.context("failed to create chat message")?;- message_ids.push(message_id.clone());-- search::send_chat_message_to_search(- &ctx,- chat_id,- &message_id,- user_id,- created_at,- created_at,- );+ message_ids.push(message_id);}Ok(message_ids)diff --git a/services/document_cognition_service/src/api/utils/mod.rs b/services/document_cognition_service/src/api/utils/mod.rsindex b117ff4ffd..f4ee9bc8d0 100644--- a/services/document_cognition_service/src/api/utils/mod.rs+++ b/services/document_cognition_service/src/api/utils/mod.rs@@ -1,2 +1 @@pub mod log;-pub mod search;diff --git a/services/document_cognition_service/src/api/utils/search.rs b/services/document_cognition_service/src/api/utils/search.rsdeleted file mode 100644index 396128dea9..0000000000--- a/services/document_cognition_service/src/api/utils/search.rs+++ /dev/null@@ -1,58 +0,0 @@-use crate::api::context::ApiContext;-use chrono::{DateTime, Utc};-use sqs_client::search::SearchQueueMessage;--/// Spawns a tokio task to send a remove chat message from the search text extractor-// used in edit_message which is dead for now-#[allow(dead_code)]-pub fn send_remove_chat_message_to_search(ctx: &ApiContext, chat_id: &str, message_id: &str) {- let chat_id = chat_id.to_string();- let message_id = message_id.to_string();- tokio::spawn({- let sqs_client = ctx.sqs_client.clone();- async move {- let _ = sqs_client- .send_message_to_search_event_queue(- sqs_client::search::SearchQueueMessage::RemoveChatMessage(- sqs_client::search::chat::RemoveChatMessage {- chat_id: chat_id.to_string(),- message_id: Some(message_id.to_string()),- index_override: None,- },- ),- )- .await;- }- });-}--/// Spawns a tokio task to send the chat message to the search text extractor-pub fn send_chat_message_to_search(- ctx: &ApiContext,- chat_id: &str,- message_id: &str,- user_id: &str,- created_at: DateTime<Utc>,- updated_at: DateTime<Utc>,-) {- let chat_id = chat_id.to_string();- let message_id = message_id.to_string();- let user_id = user_id.to_string();- tokio::spawn({- let sqs_client = ctx.sqs_client.clone();- async move {- let _ = sqs_client- .send_message_to_search_event_queue(SearchQueueMessage::ChatMessage(- sqs_client::search::chat::ChatMessage {- chat_id,- message_id,- user_id,- created_at,- updated_at,- index_override: None,- },- ))- .await;- }- });-}diff --git a/services/document_cognition_service/src/main.rs b/services/document_cognition_service/src/main.rsindex 85fd706fc6..9065cd6417 100644--- a/services/document_cognition_service/src/main.rs+++ b/services/document_cognition_service/src/main.rs@@ -103,7 +103,6 @@ async fn main() -> anyhow::Result<()> {let chat_delete_queue = macro_queues::ChatDeleteQueue::new();let email_scheduled_queue = macro_queues::EmailScheduledQueue::new();let gmail_ops_queue = macro_queues::GmailOpsQueue::new();- let search_event_queue = macro_queues::SearchEventQueue::new();let ai_projection_queue = macro_queues::AiProjectionQueue::new();let notification_queue = macro_queues::NotificationIngressQueue::new();let sqs_client = sqs_client::SQS::new(queue_aws_client)@@ -111,7 +110,6 @@ async fn main() -> anyhow::Result<()> {.chat_delete_queue(&chat_delete_queue).email_scheduled_queue(&email_scheduled_queue).gmail_ops_queue(&gmail_ops_queue)- .search_event_queue(&search_event_queue).ai_projection_queue(&ai_projection_queue);let internal_api_key = config.internal_api_key.to_string();diff --git a/services/search_processing_service/Cargo.toml b/services/search_processing_service/Cargo.tomlindex bfc46f9bab..4b3957c742 100644--- a/services/search_processing_service/Cargo.toml+++ b/services/search_processing_service/Cargo.toml@@ -10,6 +10,7 @@ default = ["processing", "service", "pdf"]processing = ["dep:call","dep:channels",+ "dep:chat","dep:kafka_util","dep:macro_event_broker","dep:macro_event_topics",@@ -38,6 +39,7 @@ aws-sdk-sqs = { workspace = true }axum = { workspace = true }call = { path = "../../crates/call", default-features = false, optional = true }channels = { path = "../../crates/channels", default-features = false, optional = true }+chat = { path = "../../crates/chat", default-features = false, optional = true }chrono = { workspace = true }comms_db_client = { path = "../../crates/comms_db_client" }email_db_client = { path = "../../crates/email_db_client" }diff --git a/services/search_processing_service/src/inbound/kafka_consumer.rs b/services/search_processing_service/src/inbound/kafka_consumer.rsindex 298924a806..cde317db90 100644--- a/services/search_processing_service/src/inbound/kafka_consumer.rs+++ b/services/search_processing_service/src/inbound/kafka_consumer.rs@@ -7,13 +7,14 @@//! offsets are already committed.//!//! Per-entity event mapping and processing live in the [`call`], [`channel`],-//! and [`project`] submodules; this module owns the poll loop, worker, retry-//! policy, and commit semantics.+//! [`chat`], and [`project`] submodules; this module owns the poll loop, worker,+//! retry policy, and commit semantics.#![allow(clippy::enum_variant_names)]mod call;mod channel;+mod chat;mod project;#[cfg(test)]mod test;@@ -21,6 +22,7 @@ mod test;use std::{future::Future, time::Duration};use ::call::domain::events::CallMacroEvent;+use ::chat::domain::events::ChatMacroEvent;use channels::domain::broker_events::ChannelMacroEvent;use kafka_util::{GroupName, KafkaEventConsumer};use macro_event_broker::{KafkaConsumerAdapter, MacroEventCollection, MacroEventConsumerService};@@ -36,7 +38,8 @@ use tokio::sync::mpsc;use tokio_retry::{Retry, strategy::ExponentialBackoff};use self::{- call::process_call_event, channel::process_channel_event, project::process_project_event,+ call::process_call_event, channel::process_channel_event, chat::process_chat_event,+ project::process_project_event,};/// Consumer group used for live search-index event offsets.@@ -55,6 +58,7 @@ macro_event_broker::declare_topics!(DeclaredMacroEvent:CallMacroEvent,ChannelMacroEvent,+ ChatMacroEvent,ProjectMacroEvent,);@@ -170,6 +174,9 @@ async fn process_event(DeclaredMacroEvent::ChannelMacroEvent(event) => {process_channel_event(db, opensearch_client, event, partition, offset).await}+ DeclaredMacroEvent::ChatMacroEvent(event) => {+ process_chat_event(db, opensearch_client, event, partition, offset).await+ }DeclaredMacroEvent::ProjectMacroEvent(event) => {process_project_event(db, opensearch_client, event, partition, offset).await}diff --git a/services/search_processing_service/src/inbound/kafka_consumer/chat.rs b/services/search_processing_service/src/inbound/kafka_consumer/chat.rsnew file mode 100644index 0000000000..5e43256589--- /dev/null+++ b/services/search_processing_service/src/inbound/kafka_consumer/chat.rs@@ -0,0 +1,172 @@+//! Maps chat lifecycle events to search-index actions and processes them.++use ::chat::domain::events::{ChatMacroEvent, ChatTopicEvent};+use macro_event_broker::MacroEvent as _;+use opensearch_client::OpensearchClient;+use sqlx::PgPool;++use super::{EventOutcome, MAX_PROCESSING_ATTEMPTS, PROCESSING_RETRY_BASE_DELAY, retry_processing};+use crate::process::chat::{remove_chat_message, upsert_chat_message_by_ids};++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub(super) enum ChatIndexAction<'a> {+ UpsertMessage {+ chat_id: &'a str,+ message_id: &'a str,+ },+ RemoveMessage {+ chat_id: &'a str,+ message_id: &'a str,+ },+ RemoveChat {+ chat_id: &'a str,+ },+ Ignore,+}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub(super) struct ChatEventDescription<'a> {+ pub(super) action: ChatIndexAction<'a>,+ pub(super) chat_id: &'a str,+ pub(super) event_type: &'static str,+}++pub(super) fn describe_chat_event(event: &ChatTopicEvent) -> ChatEventDescription<'_> {+ match event {+ ChatTopicEvent::Created(metadata) => ChatEventDescription {+ action: ChatIndexAction::Ignore,+ chat_id: &metadata.chat_id,+ event_type: "chat.created",+ },+ ChatTopicEvent::Updated(metadata) => ChatEventDescription {+ action: ChatIndexAction::Ignore,+ chat_id: &metadata.chat_id,+ event_type: "chat.updated",+ },+ ChatTopicEvent::Deleted(metadata) => ChatEventDescription {+ action: ChatIndexAction::Ignore,+ chat_id: &metadata.chat_id,+ event_type: "chat.deleted",+ },+ ChatTopicEvent::PermanentlyDeleted(metadata) => ChatEventDescription {+ action: ChatIndexAction::RemoveChat {+ chat_id: &metadata.chat_id,+ },+ chat_id: &metadata.chat_id,+ event_type: "chat.permanently_deleted",+ },+ ChatTopicEvent::Restored(metadata) => ChatEventDescription {+ action: ChatIndexAction::Ignore,+ chat_id: &metadata.chat_id,+ event_type: "chat.restored",+ },+ ChatTopicEvent::Copied(metadata) => ChatEventDescription {+ action: ChatIndexAction::Ignore,+ chat_id: &metadata.chat_id,+ event_type: "chat.copied",+ },+ ChatTopicEvent::MessageSent(metadata) => ChatEventDescription {+ action: ChatIndexAction::UpsertMessage {+ chat_id: &metadata.chat_id,+ message_id: &metadata.message_id,+ },+ chat_id: &metadata.chat_id,+ event_type: "chat.message_sent",+ },+ ChatTopicEvent::MessageDeleted(metadata) => ChatEventDescription {+ action: ChatIndexAction::RemoveMessage {+ chat_id: &metadata.chat_id,+ message_id: &metadata.message_id,+ },+ chat_id: &metadata.chat_id,+ event_type: "chat.message_deleted",+ },+ }+}++async fn process_chat_index_action(+ db: &PgPool,+ opensearch_client: &OpensearchClient,+ action: ChatIndexAction<'_>,+) -> anyhow::Result<()> {+ match action {+ ChatIndexAction::UpsertMessage {+ chat_id,+ message_id,+ } => upsert_chat_message_by_ids(opensearch_client, db, chat_id, message_id, None).await,+ ChatIndexAction::RemoveMessage {+ chat_id,+ message_id,+ } => remove_chat_message(opensearch_client, chat_id, Some(message_id), None).await,+ ChatIndexAction::RemoveChat { chat_id } => {+ remove_chat_message(opensearch_client, chat_id, None, None).await+ }+ ChatIndexAction::Ignore => Ok(()),+ }+}++pub(super) async fn process_chat_event(+ db: &PgPool,+ opensearch_client: &OpensearchClient,+ event: &ChatMacroEvent,+ partition: i32,+ offset: i64,+) -> EventOutcome {+ let description = describe_chat_event(&event.event().event);+ if description.action == ChatIndexAction::Ignore {+ tracing::trace!(+ chat_id = description.chat_id,+ event_type = description.event_type,+ partition,+ offset,+ "ignoring chat event without a search-index action"+ );+ return EventOutcome::Ignored;+ }++ let result = retry_processing(|attempt| async move {+ tracing::trace!(+ chat_id = description.chat_id,+ event_type = description.event_type,+ partition,+ offset,+ attempt,+ "processing chat search-index event"+ );+ process_chat_index_action(db, opensearch_client, description.action)+ .await+ .inspect_err(|error| {+ if attempt < MAX_PROCESSING_ATTEMPTS {+ let retry_delay =+ PROCESSING_RETRY_BASE_DELAY * 2u32.pow(attempt.saturating_sub(1));+ tracing::warn!(+ error = ?error,+ chat_id = description.chat_id,+ event_type = description.event_type,+ partition,+ offset,+ attempt,+ delay_secs = retry_delay.as_secs(),+ "chat search-index processing failed, retrying"+ );+ }+ })+ })+ .await;++ match result {+ Ok(()) => EventOutcome::Indexed,+ Err(error) => {+ tracing::error!(+ error = ?error,+ chat_id = description.chat_id,+ event_type = description.event_type,+ partition,+ offset,+ attempts = MAX_PROCESSING_ATTEMPTS,+ "dropping chat event after processing retries were exhausted"+ );+ EventOutcome::Dropped+ }+ }+}diff --git a/services/search_processing_service/src/inbound/kafka_consumer/project.rs b/services/search_processing_service/src/inbound/kafka_consumer/project.rsindex 2619764745..c4af0d3a0d 100644--- a/services/search_processing_service/src/inbound/kafka_consumer/project.rs+++ b/services/search_processing_service/src/inbound/kafka_consumer/project.rs@@ -9,11 +9,14 @@ use sqlx::PgPool;use sqs_client::search::project::UpsertProject;use super::{EventOutcome, MAX_PROCESSING_ATTEMPTS, PROCESSING_RETRY_BASE_DELAY, retry_processing};-use crate::process::project::upsert_project;+use crate::process::{chat::remove_chat_message, project::upsert_project};#[derive(Debug, Clone, PartialEq, Eq)]pub(super) enum ProjectIndexAction {- Reconcile { project_ids: Vec<String> },+ Reconcile {+ project_ids: Vec<String>,+ purged_chat_ids: Vec<String>,+ },}#[derive(Debug, Clone, PartialEq, Eq)]@@ -46,6 +49,7 @@ pub(super) fn describe_project_event(event: &ProjectTopicEvent) -> ProjectEventDSome(metadata.project_id.as_str()),metadata.parent_project_id.as_deref(),]),+ purged_chat_ids: Vec::new(),},project_id: &metadata.project_id,event_type: "project.created",@@ -57,6 +61,7 @@ pub(super) fn describe_project_event(event: &ProjectTopicEvent) -> ProjectEventDmetadata.previous_parent_id.as_deref(),metadata.parent_id.as_deref(),]),+ purged_chat_ids: Vec::new(),},project_id: &metadata.project_id,event_type: "project.updated",@@ -70,6 +75,7 @@ pub(super) fn describe_project_event(event: &ProjectTopicEvent) -> ProjectEventD.map(|project_id| Some(project_id.as_str())).chain([metadata.parent_project_id.as_deref()]),),+ purged_chat_ids: Vec::new(),},project_id: &metadata.project_id,event_type: "project.deleted",@@ -83,6 +89,7 @@ pub(super) fn describe_project_event(event: &ProjectTopicEvent) -> ProjectEventD.map(|project_id| Some(project_id.as_str())).chain([metadata.parent_project_id.as_deref()]),),+ purged_chat_ids: Vec::new(),},project_id: &metadata.project_id,event_type: "project.restored",@@ -96,6 +103,7 @@ pub(super) fn describe_project_event(event: &ProjectTopicEvent) -> ProjectEventD.map(|project_id| Some(project_id.as_str())).chain([metadata.parent_project_id.as_deref()]),),+ purged_chat_ids: metadata.purged_chat_ids.clone(),},project_id: &metadata.project_id,event_type: "project.permanently_deleted",@@ -109,6 +117,7 @@ pub(super) fn describe_project_event(event: &ProjectTopicEvent) -> ProjectEventD.map(|project_id| Some(project_id.as_str())).chain([metadata.parent_project_id.as_deref()]),),+ purged_chat_ids: Vec::new(),},project_id: &metadata.root_project_id,event_type: "project.uploaded",@@ -122,7 +131,10 @@ async fn process_project_index_action(action: ProjectIndexAction,) -> anyhow::Result<()> {match action {- ProjectIndexAction::Reconcile { project_ids } => {+ ProjectIndexAction::Reconcile {+ project_ids,+ purged_chat_ids,+ } => {for project_id in project_ids {upsert_project(opensearch_client,@@ -134,6 +146,11 @@ async fn process_project_index_action().await?;}++ for chat_id in purged_chat_ids {+ remove_chat_message(opensearch_client, &chat_id, None, None).await?;+ }+Ok(())}}diff --git a/services/search_processing_service/src/inbound/kafka_consumer/test.rs b/services/search_processing_service/src/inbound/kafka_consumer/test.rsindex dd55691cae..b62352a43f 100644--- a/services/search_processing_service/src/inbound/kafka_consumer/test.rs+++ b/services/search_processing_service/src/inbound/kafka_consumer/test.rs@@ -6,6 +6,11 @@ use ::call::domain::events::{CallRecordSummarizedMetadata, CallRecordUpdatedMetadata, CallRecordingReadyMetadata,CallStartedMetadata, CallTopicEvent,};+use ::chat::domain::events::{+ ChatCopiedMetadata, ChatCreatedMetadata, ChatDeletedMetadata, ChatMessageDeletedMetadata,+ ChatMessageRole, ChatMessageSentMetadata, ChatPermanentlyDeletedMetadata, ChatRestoredMetadata,+ ChatTopicEvent, ChatUpdatedMetadata,+};use channels::domain::{broker_events::{ChannelCreatedMetadata, ChannelDeletedMetadata, ChannelMessageAttachmentCreatedMetadata,@@ -18,7 +23,9 @@ use channels::domain::{};use chrono::Utc;use macro_event_broker::{Event, EventBrokerError, MacroEvent as _, MessageParts};-use macro_event_topics::{MacroCallsTopic, MacroChannelsTopic, MacroProjectsTopic, Topic as _};+use macro_event_topics::{+ MacroCallsTopic, MacroChannelsTopic, MacroChatsTopic, MacroProjectsTopic, Topic as _,+};use macro_user_id::user_id::MacroUserIdStr;use projects::domain::events::{ProjectCreatedMetadata, ProjectDeletedMetadata, ProjectPermanentlyDeletedMetadata,@@ -29,6 +36,7 @@ use uuid::Uuid;use super::{call::{CallEventDescription, CallIndexAction, describe_call_event},channel::{ChannelEventDescription, ChannelIndexAction, describe_channel_event},+ chat::{ChatEventDescription, ChatIndexAction, describe_chat_event},project::{ProjectEventDescription, ProjectIndexAction, collect_project_ids, describe_project_event,},@@ -38,6 +46,9 @@ use super::{const CALL_ID: Uuid = Uuid::from_u128(1);const CHANNEL_ID: Uuid = Uuid::from_u128(2);const MESSAGE_ID: Uuid = Uuid::from_u128(3);+const CHAT_ID: &str = "chat-id";+const SECOND_CHAT_ID: &str = "second-chat-id";+const CHAT_MESSAGE_ID: &str = "chat-message-id";const PROJECT_ID: &str = "project-root";const CHILD_PROJECT_ID: &str = "project-child";const PARENT_PROJECT_ID: &str = "project-parent";@@ -140,6 +151,122 @@ fn encoded_message<E: serde::Serialize>(}}+fn chat_event_cases() -> Vec<(ChatTopicEvent, ChatEventDescription<'static>)> {+ let owner = user_id();++ vec![+ (+ ChatTopicEvent::Created(ChatCreatedMetadata {+ chat_id: CHAT_ID.to_string(),+ owner: owner.clone(),+ name: "Chat".to_string(),+ project_id: Some(PROJECT_ID.to_string()),+ }),+ ChatEventDescription {+ action: ChatIndexAction::Ignore,+ chat_id: CHAT_ID,+ event_type: "chat.created",+ },+ ),+ (+ ChatTopicEvent::Updated(ChatUpdatedMetadata {+ chat_id: CHAT_ID.to_string(),+ actor_user_id: owner.clone(),+ name: Some("Renamed chat".to_string()),+ previous_project_id: Some(PROJECT_ID.to_string()),+ project_id: Some(PARENT_PROJECT_ID.to_string()),+ share_permission_updated: true,+ }),+ ChatEventDescription {+ action: ChatIndexAction::Ignore,+ chat_id: CHAT_ID,+ event_type: "chat.updated",+ },+ ),+ (+ ChatTopicEvent::Deleted(ChatDeletedMetadata {+ chat_id: CHAT_ID.to_string(),+ actor_user_id: Some(owner.clone()),+ project_id: Some(PROJECT_ID.to_string()),+ }),+ ChatEventDescription {+ action: ChatIndexAction::Ignore,+ chat_id: CHAT_ID,+ event_type: "chat.deleted",+ },+ ),+ (+ ChatTopicEvent::PermanentlyDeleted(ChatPermanentlyDeletedMetadata {+ chat_id: CHAT_ID.to_string(),+ actor_user_id: Some(owner.clone()),+ project_id: Some(PROJECT_ID.to_string()),+ }),+ ChatEventDescription {+ action: ChatIndexAction::RemoveChat { chat_id: CHAT_ID },+ chat_id: CHAT_ID,+ event_type: "chat.permanently_deleted",+ },+ ),+ (+ ChatTopicEvent::Restored(ChatRestoredMetadata {+ chat_id: CHAT_ID.to_string(),+ actor_user_id: Some(owner.clone()),+ project_id: Some(PROJECT_ID.to_string()),+ }),+ ChatEventDescription {+ action: ChatIndexAction::Ignore,+ chat_id: CHAT_ID,+ event_type: "chat.restored",+ },+ ),+ (+ ChatTopicEvent::Copied(ChatCopiedMetadata {+ chat_id: CHAT_ID.to_string(),+ source_chat_id: "source-chat-id".to_string(),+ owner: owner.clone(),+ name: "Copied chat".to_string(),+ }),+ ChatEventDescription {+ action: ChatIndexAction::Ignore,+ chat_id: CHAT_ID,+ event_type: "chat.copied",+ },+ ),+ (+ ChatTopicEvent::MessageSent(ChatMessageSentMetadata {+ chat_id: CHAT_ID.to_string(),+ message_id: CHAT_MESSAGE_ID.to_string(),+ role: ChatMessageRole::User,+ model: "chat-model".to_string(),+ actor_user_id: Some(owner),+ attachment_count: 1,+ }),+ ChatEventDescription {+ action: ChatIndexAction::UpsertMessage {+ chat_id: CHAT_ID,+ message_id: CHAT_MESSAGE_ID,+ },+ chat_id: CHAT_ID,+ event_type: "chat.message_sent",+ },+ ),+ (+ ChatTopicEvent::MessageDeleted(ChatMessageDeletedMetadata {+ chat_id: CHAT_ID.to_string(),+ message_id: CHAT_MESSAGE_ID.to_string(),+ }),+ ChatEventDescription {+ action: ChatIndexAction::RemoveMessage {+ chat_id: CHAT_ID,+ message_id: CHAT_MESSAGE_ID,+ },+ chat_id: CHAT_ID,+ event_type: "chat.message_deleted",+ },+ ),+ ]+}+fn channel_event_cases() -> Vec<(ChannelTopicEvent, ChannelEventDescription)> {let sender = channel_sender();@@ -318,6 +445,7 @@ fn project_event_cases() -> Vec<(ProjectTopicEvent, ProjectEventDescription<'staProjectEventDescription {action: ProjectIndexAction::Reconcile {project_ids: vec![PROJECT_ID.to_string(), PARENT_PROJECT_ID.to_string()],+ purged_chat_ids: Vec::new(),},project_id: PROJECT_ID,event_type: "project.created",@@ -340,6 +468,7 @@ fn project_event_cases() -> Vec<(ProjectTopicEvent, ProjectEventDescription<'staPARENT_PROJECT_ID.to_string(),NEW_PARENT_PROJECT_ID.to_string(),],+ purged_chat_ids: Vec::new(),},project_id: PROJECT_ID,event_type: "project.updated",@@ -358,6 +487,7 @@ fn project_event_cases() -> Vec<(ProjectTopicEvent, ProjectEventDescription<'staProjectEventDescription {action: ProjectIndexAction::Reconcile {project_ids: vec![PROJECT_ID.to_string(), PARENT_PROJECT_ID.to_string()],+ purged_chat_ids: Vec::new(),},project_id: PROJECT_ID,event_type: "project.updated",@@ -385,6 +515,7 @@ fn project_event_cases() -> Vec<(ProjectTopicEvent, ProjectEventDescription<'staCHILD_PROJECT_ID.to_string(),PARENT_PROJECT_ID.to_string(),],+ purged_chat_ids: Vec::new(),},project_id: PROJECT_ID,event_type: "project.deleted",@@ -405,6 +536,7 @@ fn project_event_cases() -> Vec<(ProjectTopicEvent, ProjectEventDescription<'staCHILD_PROJECT_ID.to_string(),PARENT_PROJECT_ID.to_string(),],+ purged_chat_ids: Vec::new(),},project_id: PROJECT_ID,event_type: "project.restored",@@ -418,7 +550,7 @@ fn project_event_cases() -> Vec<(ProjectTopicEvent, ProjectEventDescription<'staparent_project_id: Some(PARENT_PROJECT_ID.to_string()),purged_project_ids: vec![PROJECT_ID.to_string(), CHILD_PROJECT_ID.to_string()],purged_document_ids: vec!["document-id".to_string()],- purged_chat_ids: vec!["chat-id".to_string()],+ purged_chat_ids: vec![CHAT_ID.to_string(), SECOND_CHAT_ID.to_string()],}),ProjectEventDescription {action: ProjectIndexAction::Reconcile {@@ -427,6 +559,7 @@ fn project_event_cases() -> Vec<(ProjectTopicEvent, ProjectEventDescription<'staCHILD_PROJECT_ID.to_string(),PARENT_PROJECT_ID.to_string(),],+ purged_chat_ids: vec![CHAT_ID.to_string(), SECOND_CHAT_ID.to_string()],},project_id: PROJECT_ID,event_type: "project.permanently_deleted",@@ -447,6 +580,7 @@ fn project_event_cases() -> Vec<(ProjectTopicEvent, ProjectEventDescription<'staCHILD_PROJECT_ID.to_string(),PARENT_PROJECT_ID.to_string(),],+ purged_chat_ids: Vec::new(),},project_id: PROJECT_ID,event_type: "project.uploaded",@@ -462,8 +596,10 @@ fn subscribes_to_declared_search_processing_topics_with_durable_group() {"search-processing-service");let topics = DeclaredMacroEvent::topics();+ assert_eq!(MacroChatsTopic::TOPIC_STR, "macro.chats");assert!(topics.contains(&MacroCallsTopic::TOPIC_STR));assert!(topics.contains(&MacroChannelsTopic::TOPIC_STR));+ assert!(topics.contains(&MacroChatsTopic::TOPIC_STR));assert!(topics.contains(&MacroProjectsTopic::TOPIC_STR));}@@ -522,6 +658,18 @@ fn maps_all_call_lifecycle_events_to_index_actions() {);}+#[test]+fn maps_all_chat_lifecycle_events_to_index_actions() {+ let cases = chat_event_cases();+ assert_eq!(cases.len(), 8);++ for (event, expected) in cases {+ let serialized = serde_json::to_value(&event).expect("serializable chat event");+ assert_eq!(serialized["event_type"], expected.event_type);+ assert_eq!(describe_chat_event(&event), expected);+ }+}+#[test]fn maps_all_channel_lifecycle_events_to_index_actions() {let cases = channel_event_cases();@@ -560,6 +708,30 @@ fn project_id_collection_is_stable_and_drops_missing_or_empty_ids() {);}+#[test]+fn chat_envelope_decodes_round_trip() {+ let event = ChatTopicEvent::MessageSent(ChatMessageSentMetadata {+ chat_id: CHAT_ID.to_string(),+ message_id: CHAT_MESSAGE_ID.to_string(),+ role: ChatMessageRole::Assistant,+ model: "chat-model".to_string(),+ actor_user_id: None,+ attachment_count: 0,+ });+ let message = encoded_message(+ MacroChatsTopic::TOPIC_STR,+ CHAT_ID,+ Event::new(event.clone()),+ );++ let decoded = DeclaredMacroEvent::decode(&message).expect("decodable chat event");+ let DeclaredMacroEvent::ChatMacroEvent(decoded_event) = decoded else {+ panic!("expected chat event");+ };+ assert_eq!(decoded_event.key(), CHAT_ID);+ assert_eq!(decoded_event.event().event, event);+}+#[test]fn channel_envelope_decodes_round_trip() {let event = ChannelTopicEvent::Deleted(ChannelDeletedMetadata {@@ -603,6 +775,50 @@ fn project_envelope_decodes_round_trip_with_string_key() {assert_eq!(decoded_event.event().event, event);}+#[tokio::test]+async fn malformed_and_unsupported_chat_messages_are_commit_safe() {+ let malformed = TestMessage {+ topic: MacroChatsTopic::TOPIC_STR,+ key: Some(CHAT_ID.to_string()),+ payload: Some(b"not json".to_vec()),+ };+ let malformed = attach_event_coordinates(DeclaredMacroEvent::decode(&malformed), 5, 40);+ let (sender, mut receiver) = mpsc::channel(1);++ assert!(matches!(+ handoff_decoded(&sender, malformed).await,+ HandoffOutcome::MalformedRecord(EventBrokerError::Serialization(_))+ ));++ let unsupported_event = ChatTopicEvent::MessageDeleted(ChatMessageDeletedMetadata {+ chat_id: CHAT_ID.to_string(),+ message_id: CHAT_MESSAGE_ID.to_string(),+ });+ let unsupported = encoded_message(+ MacroChatsTopic::TOPIC_STR,+ CHAT_ID,+ Event::with_schema_version(unsupported_event, 2),+ );+ let unsupported = attach_event_coordinates(DeclaredMacroEvent::decode(&unsupported), 5, 41);+ match handoff_decoded(&sender, unsupported).await {+ HandoffOutcome::MalformedRecord(EventBrokerError::UnsupportedSchemaVersion {+ topic,+ expected,+ actual,+ }) => {+ assert_eq!(topic, MacroChatsTopic::TOPIC_STR);+ assert_eq!(expected, 1);+ assert_eq!(actual, 2);+ }+ outcome => panic!("expected malformed chat record, got {outcome:?}"),+ }++ assert!(matches!(+ receiver.try_recv(),+ Err(mpsc::error::TryRecvError::Empty)+ ));+}+#[tokio::test]async fn unsupported_project_schema_message_is_commit_safe() {let event = ProjectTopicEvent::Restored(ProjectRestoredMetadata {diff --git a/services/search_processing_service/src/process/chat.rs b/services/search_processing_service/src/process/chat.rsindex fb33b30bd0..8750876b39 100644--- a/services/search_processing_service/src/process/chat.rs+++ b/services/search_processing_service/src/process/chat.rs@@ -4,38 +4,48 @@ use opensearch_client::{OpensearchClient, date_format::EpochMillis, upsert::chat_message::UpsertChatMessageArgs,};use properties::outbound::entity_properties_get_query::get_entity_properties_for_index;-use sqs_client::search::chat::{ChatMessage, RemoveChatMessage};+use sqs_client::search::chat::ChatMessage;use crate::process::properties::to_indexed_properties;-/// Handles the processing of chat messages-#[tracing::instrument(skip(opensearch_client, db))]+/// Handles the SQS backfill request for a chat message.+#[tracing::instrument(skip(opensearch_client, db), err)]pub async fn insert_chat_message(opensearch_client: &OpensearchClient,db: &sqlx::Pool<sqlx::Postgres>,chat_message: &ChatMessage,) -> anyhow::Result<()> {- let info = macro_db_client::chat::get::get_chat_message_info(+ upsert_chat_message_by_ids(+ opensearch_client,db,chat_message.chat_id.as_str(),chat_message.message_id.as_str(),+ chat_message.index_override.as_deref(),).await- .context("failed to get chat message info")?;+}++/// Upserts a chat message using the authoritative database values.+#[tracing::instrument(skip(opensearch_client, db), err)]+pub(crate) async fn upsert_chat_message_by_ids(+ opensearch_client: &OpensearchClient,+ db: &sqlx::Pool<sqlx::Postgres>,+ chat_id: &str,+ message_id: &str,+ index_override: Option<&str>,+) -> anyhow::Result<()> {+ let info = macro_db_client::chat::get::get_chat_message_info(db, chat_id, message_id)+ .await+ .context("failed to get chat message info")?;let Some(info) = info else {return Ok(());};- let index_override = chat_message.index_override.as_deref();if info.deleted_at.is_some() {tracing::trace!("chat is deleted, removing message from search index");opensearch_client- .delete_chat_message(- chat_message.chat_id.as_str(),- chat_message.message_id.as_str(),- index_override,- )+ .delete_chat_message(chat_id, message_id, index_override).await.context("failed to delete chat message from search")?;return Ok(());@@ -45,7 +55,7 @@ pub async fn insert_chat_message(// write or values set by the property-update path get wiped. A fetch// failure propagates (retry) rather than being mistaken for "empty".let properties = to_indexed_properties(- get_entity_properties_for_index(db, chat_message.chat_id.as_str(), EntityType::Chat)+ get_entity_properties_for_index(db, chat_id, EntityType::Chat).await.context("failed to fetch chat properties for search index")?,);@@ -53,11 +63,11 @@ pub async fn insert_chat_message(opensearch_client.upsert_chat_message(&UpsertChatMessageArgs {- chat_id: chat_message.chat_id.clone(),- chat_message_id: chat_message.message_id.clone(),- user_id: chat_message.user_id.clone(),- created_at_millis: EpochMillis::new(chat_message.created_at.timestamp_millis())?,- updated_at_millis: EpochMillis::new(chat_message.updated_at.timestamp_millis())?,+ chat_id: chat_id.to_string(),+ chat_message_id: message_id.to_string(),+ user_id: info.owner_user_id,+ created_at_millis: EpochMillis::new(info.created_at.timestamp_millis())?,+ updated_at_millis: EpochMillis::new(info.updated_at.timestamp_millis())?,title: info.name,content: info.content,role: info.role,@@ -71,22 +81,23 @@ pub async fn insert_chat_message(Ok(())}-/// Handles the removal of chat message(s) from the opensearch index-#[tracing::instrument(skip(opensearch_client))]-pub async fn remove_chat_message(+/// Removes a chat message or all messages for a chat from the search index.+#[tracing::instrument(skip(opensearch_client), err)]+pub(crate) async fn remove_chat_message(opensearch_client: &OpensearchClient,- remove_message: &RemoveChatMessage,+ chat_id: &str,+ message_id: Option<&str>,+ index_override: Option<&str>,) -> anyhow::Result<()> {- let index_override = remove_message.index_override.as_deref();- if let Some(message_id) = remove_message.message_id.as_ref() {+ if let Some(message_id) = message_id {tracing::trace!("deleting chat message");opensearch_client- .delete_chat_message(remove_message.chat_id.as_str(), message_id, index_override)+ .delete_chat_message(chat_id, message_id, index_override).await?;} else {tracing::trace!("deleting chat");opensearch_client- .delete_chat(remove_message.chat_id.as_str(), index_override)+ .delete_chat(chat_id, index_override).await?;}diff --git a/services/search_processing_service/src/process/mod.rs b/services/search_processing_service/src/process/mod.rsindex 7a21f79c7e..b7c1b81daa 100644--- a/services/search_processing_service/src/process/mod.rs+++ b/services/search_processing_service/src/process/mod.rs@@ -1,6 +1,6 @@pub(crate) mod call;pub(crate) mod channel;-mod chat;+pub(crate) mod chat;pub mod context;mod document;mod email;@@ -115,9 +115,6 @@ pub async fn process_message(SearchQueueMessage::ChatMessage(message) => {chat::insert_chat_message(&ctx.opensearch_client, &ctx.db, &message).await?;}- SearchQueueMessage::RemoveChatMessage(message) => {- chat::remove_chat_message(&ctx.opensearch_client, &message).await?;- }SearchQueueMessage::CallRecord(message) => {let call_id = message.call_id