Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(search_processing_service): chats kafka (#5304)

Will Hutchinson7183fca6f74b
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.json
index 09e2587f83..da5f74eea2 100644
--- a/.github/workspace-dep-closures.json
+++ b/.github/workspace-dep-closures.json
@@ -1304,6 +1304,7 @@
"crates/authentication_service_client",
"crates/bot_id",
"crates/channel_sender",
+ "crates/chat",
"crates/cowlike",
"crates/crm",
"crates/document_sub_type",
@@ -1359,6 +1360,7 @@
"crates/properties",
"crates/rate_limit",
"crates/remote_env_var",
+ "crates/roles_and_permissions",
"crates/s3_key",
"crates/sqs_client",
"crates/system_properties",
@@ -4924,6 +4926,7 @@
"crates/call",
"crates/channel_sender",
"crates/channels",
+ "crates/chat",
"crates/comms_db_client",
"crates/connection",
"crates/connection_gateway_client",
@@ -4996,6 +4999,7 @@
"crates/properties",
"crates/rate_limit",
"crates/remote_env_var",
+ "crates/roles_and_permissions",
"crates/s3_client",
"crates/s3_key",
"crates/share_permission_db_utils",
diff --git a/.sqlx/query-92e2916a16d165600f457c778ac3db9ae11c6625c948798e19ca19da4a30f3c1.json b/.sqlx/query-eb360ba6aad69fe2c501baa8b61b4243051bd3bde6e517014cd5768334fd7757.json
similarity index 51%
rename from .sqlx/query-92e2916a16d165600f457c778ac3db9ae11c6625c948798e19ca19da4a30f3c1.json
rename to .sqlx/query-eb360ba6aad69fe2c501baa8b61b4243051bd3bde6e517014cd5768334fd7757.json
index f2a02380a7..d6a8a3072f 100644
--- a/.sqlx/query-92e2916a16d165600f457c778ac3db9ae11c6625c948798e19ca19da4a30f3c1.json
+++ b/.sqlx/query-eb360ba6aad69fe2c501baa8b61b4243051bd3bde6e517014cd5768334fd7757.json
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
- "query": "\n SELECT\n m.content as \"content\",\n c.name as \"name\",\n m.role as \"role\",\n c.\"deletedAt\"::timestamptz as \"deleted_at\"\n FROM\n \"ChatMessage\" m\n JOIN\n \"Chat\" c on c.\"id\" = m.\"chatId\"\n WHERE\n m.id = $1 AND m.\"chatId\" = $2\n ",
+ "query": "\n SELECT\n m.content as \"content\",\n c.name as \"name\",\n m.role as \"role\",\n c.\"deletedAt\"::timestamptz as \"deleted_at\",\n c.\"userId\" as \"owner_user_id\",\n m.\"createdAt\" as \"created_at\",\n m.\"updatedAt\" as \"updated_at\"\n FROM\n \"ChatMessage\" m\n JOIN\n \"Chat\" c on c.\"id\" = m.\"chatId\"\n WHERE\n m.id = $1 AND m.\"chatId\" = $2\n ",
"describe": {
"columns": [
{
@@ -22,6 +22,21 @@
"ordinal": 3,
"name": "deleted_at",
"type_info": "Timestamptz"
+ },
+ {
+ "ordinal": 4,
+ "name": "owner_user_id",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 5,
+ "name": "created_at",
+ "type_info": "Timestamp"
+ },
+ {
+ "ordinal": 6,
+ "name": "updated_at",
+ "type_info": "Timestamp"
}
],
"parameters": {
@@ -34,8 +49,11 @@
false,
false,
false,
- null
+ null,
+ false,
+ false,
+ false
]
},
- "hash": "92e2916a16d165600f457c778ac3db9ae11c6625c948798e19ca19da4a30f3c1"
+ "hash": "eb360ba6aad69fe2c501baa8b61b4243051bd3bde6e517014cd5768334fd7757"
}
diff --git a/Cargo.lock b/Cargo.lock
index a33935b223..f5d94018ca 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3995,6 +3995,7 @@ dependencies = [
"aws-sdk-sqs",
"aws-smithy-types",
"aws_lambda_events",
+ "chat",
"chrono",
"futures",
"lambda_runtime",
@@ -12888,6 +12889,7 @@ dependencies = [
"axum",
"call",
"channels",
+ "chat",
"chrono",
"comms_db_client",
"document_sub_type",
diff --git a/crates/macro_db_client/fixtures/chat_message_info.sql b/crates/macro_db_client/fixtures/chat_message_info.sql
index 02bc75042c..0bf201b794 100644
--- a/crates/macro_db_client/fixtures/chat_message_info.sql
+++ b/crates/macro_db_client/fixtures/chat_message_info.sql
@@ -4,12 +4,14 @@ VALUES ('a1111111-1111-1111-1111-111111111111', 'user', '[email protected]', 'stripe
INSERT INTO public."User" ("id","email","stripeCustomerId","macro_user_id")
VALUES ('macro|[email protected]', '[email protected]','stripe_id','a1111111-1111-1111-1111-111111111111');
-INSERT INTO public."Chat" ("id","name","userId","model","createdAt","updatedAt","isPersistent")
+INSERT INTO public."Chat" ("id","name","userId","model","createdAt","updatedAt","deletedAt","isPersistent")
VALUES
- ('chat-persistent', 'persistent chat', 'macro|[email protected]', 'gpt-4o', '2024-01-01 00:00:00', '2024-01-01 00:00:00', true),
- ('chat-ephemeral', 'ephemeral chat', 'macro|[email protected]', 'gpt-4o', '2024-01-01 00:00:00', '2024-01-01 00:00:00', false);
+ ('chat-persistent', 'persistent chat', 'macro|[email protected]', 'gpt-4o', '2024-01-01 00:00:00', '2024-01-01 00:00:00', NULL, true),
+ ('chat-ephemeral', 'ephemeral chat', 'macro|[email protected]', 'gpt-4o', '2024-02-01 00:00:00', '2024-02-01 00:00:00', NULL, false),
+ ('chat-deleted', 'deleted chat', 'macro|[email protected]', 'gpt-4o', '2024-03-01 00:00:00', '2024-03-01 00:00:00', '2024-03-04 05:06:07.890', true);
-INSERT INTO public."ChatMessage" ("id","content","role","chatId")
+INSERT INTO public."ChatMessage" ("id","content","role","chatId","createdAt","updatedAt")
VALUES
- ('msg-persistent', '"codebase brighter"', 'user', 'chat-persistent'),
- ('msg-ephemeral', '"another message"', 'user', 'chat-ephemeral');
+ ('msg-persistent', '"codebase brighter"', 'user', 'chat-persistent', '2024-01-02 03:04:05.123', '2024-01-03 04:05:06.789'),
+ ('msg-ephemeral', '"another message"', 'assistant', 'chat-ephemeral', '2024-02-02 03:04:05.123', '2024-02-03 04:05:06.789'),
+ ('msg-deleted', '"remove from search"', 'assistant', 'chat-deleted', '2024-03-02 03:04:05.123', '2024-03-03 04:05:06.789');
diff --git a/crates/macro_db_client/src/chat/get.rs b/crates/macro_db_client/src/chat/get.rs
index 02cc071baf..cdda6dfe2a 100644
--- a/crates/macro_db_client/src/chat/get.rs
+++ b/crates/macro_db_client/src/chat/get.rs
@@ -258,6 +258,9 @@ pub struct ChatMessageInfo {
pub name: String,
pub content: String,
pub role: String,
+ pub owner_user_id: String,
+ pub created_at: DateTime<Utc>,
+ pub updated_at: DateTime<Utc>,
pub deleted_at: Option<DateTime<Utc>>,
}
@@ -277,7 +280,10 @@ pub async fn get_chat_message_info(
m.content as "content",
c.name as "name",
m.role as "role",
- c."deletedAt"::timestamptz as "deleted_at"
+ c."deletedAt"::timestamptz as "deleted_at",
+ c."userId" as "owner_user_id",
+ m."createdAt" as "created_at",
+ m."updatedAt" as "updated_at"
FROM
"ChatMessage" m
JOIN
@@ -300,6 +306,9 @@ pub async fn get_chat_message_info(
name: row.name,
content,
role: row.role,
+ owner_user_id: row.owner_user_id,
+ created_at: DateTime::<Utc>::from_naive_utc_and_offset(row.created_at, Utc),
+ updated_at: DateTime::<Utc>::from_naive_utc_and_offset(row.updated_at, Utc),
deleted_at: row.deleted_at,
}))
}
diff --git a/crates/macro_db_client/src/chat/get/test.rs b/crates/macro_db_client/src/chat/get/test.rs
index 4533d82ec7..633eab9cc6 100644
--- a/crates/macro_db_client/src/chat/get/test.rs
+++ b/crates/macro_db_client/src/chat/get/test.rs
@@ -2,20 +2,86 @@ use super::*;
use sqlx::{Pool, Postgres};
#[sqlx::test(fixtures(path = "../../../fixtures", scripts("chat_message_info")))]
-async fn persistent_chat_messages_are_indexed_for_search(
+async fn returns_persistent_and_ephemeral_chat_message_metadata(
pool: Pool<Postgres>,
) -> anyhow::Result<()> {
let persistent = get_chat_message_info(&pool, "chat-persistent", "msg-persistent")
.await?
.expect("persistent chat message should be returned for indexing");
+ assert_eq!(persistent.name, "persistent chat");
assert_eq!(persistent.content, "codebase brighter");
+ assert_eq!(persistent.role, "user");
+ assert_eq!(persistent.owner_user_id, "macro|[email protected]");
+ assert_eq!(
+ persistent.created_at,
+ "2024-01-02T03:04:05.123Z".parse::<DateTime<Utc>>()?
+ );
+ assert_eq!(
+ persistent.updated_at,
+ "2024-01-03T04:05:06.789Z".parse::<DateTime<Utc>>()?
+ );
assert!(persistent.deleted_at.is_none());
let ephemeral = get_chat_message_info(&pool, "chat-ephemeral", "msg-ephemeral")
.await?
.expect("ephemeral chat message should be returned for indexing");
+ assert_eq!(ephemeral.name, "ephemeral chat");
assert_eq!(ephemeral.content, "another message");
+ assert_eq!(ephemeral.role, "assistant");
+ assert_eq!(ephemeral.owner_user_id, "macro|[email protected]");
+ assert_eq!(
+ ephemeral.created_at,
+ "2024-02-02T03:04:05.123Z".parse::<DateTime<Utc>>()?
+ );
+ assert_eq!(
+ ephemeral.updated_at,
+ "2024-02-03T04:05:06.789Z".parse::<DateTime<Utc>>()?
+ );
assert!(ephemeral.deleted_at.is_none());
Ok(())
}
+
+#[sqlx::test(fixtures(path = "../../../fixtures", scripts("chat_message_info")))]
+async fn returns_soft_deleted_chat_message_metadata(pool: Pool<Postgres>) -> anyhow::Result<()> {
+ let message = get_chat_message_info(&pool, "chat-deleted", "msg-deleted")
+ .await?
+ .expect("soft-deleted chat message should be returned for removal");
+
+ assert_eq!(message.name, "deleted chat");
+ assert_eq!(message.content, "remove from search");
+ assert_eq!(message.role, "assistant");
+ assert_eq!(message.owner_user_id, "macro|[email protected]");
+ assert_eq!(
+ message.created_at,
+ "2024-03-02T03:04:05.123Z".parse::<DateTime<Utc>>()?
+ );
+ assert_eq!(
+ message.updated_at,
+ "2024-03-03T04:05:06.789Z".parse::<DateTime<Utc>>()?
+ );
+ assert_eq!(
+ message.deleted_at,
+ Some("2024-03-04T05:06:07.890Z".parse::<DateTime<Utc>>()?)
+ );
+
+ Ok(())
+}
+
+#[sqlx::test(fixtures(path = "../../../fixtures", scripts("chat_message_info")))]
+async fn returns_none_for_missing_or_mismatched_message_id(
+ pool: Pool<Postgres>,
+) -> anyhow::Result<()> {
+ assert!(
+ get_chat_message_info(&pool, "chat-persistent", "msg-missing")
+ .await?
+ .is_none()
+ );
+ assert!(
+ get_chat_message_info(&pool, "chat-ephemeral", "msg-persistent")
+ .await?
+ .is_none()
+ );
+
+ Ok(())
+}
diff --git a/crates/projects/src/domain/ports.rs b/crates/projects/src/domain/ports.rs
index 5ec4e2e3cb..6031b00a16 100644
--- a/crates/projects/src/domain/ports.rs
+++ b/crates/projects/src/domain/ports.rs
@@ -206,15 +206,9 @@ pub trait ShaCounterPort: Send + Sync + 'static {
) -> impl Future<Output = anyhow::Result<()>> + Send;
}
-/// Port for publishing cross-family search and document-deletion work.
-// TODO: Remove this port and its SQS adapter after the chats and documents migrations.
+/// Port for publishing document search-removal and deletion work.
+// TODO: Remove this port and its SQS adapter after the documents migration.
pub trait ProjectSearchIndexer: Send + Sync + 'static {
- /// Remove chats from the search index.
- fn remove_chats(
- &self,
- chat_ids: Vec<String>,
- ) -> impl Future<Output = anyhow::Result<()>> + Send;
-
/// Remove documents from the search index.
fn remove_documents(
&self,
diff --git a/crates/projects/src/domain/service.rs b/crates/projects/src/domain/service.rs
index e63707d81f..2016320a17 100644
--- a/crates/projects/src/domain/service.rs
+++ b/crates/projects/src/domain/service.rs
@@ -77,7 +77,7 @@ where
pub sha_counter: Sha,
/// Entity-access inheritance manager.
pub entity_access_management_service: Eam,
- /// Search and deletion queue publisher.
+ /// Document search and deletion queue publisher.
pub search_indexer: Idx,
/// Project lifecycle event broker.
pub macro_event_broker: B,
@@ -534,13 +534,6 @@ where
.map(|(document_id, _)| document_id.clone())
.collect::<Vec<_>>();
- if !purged.chat_ids.is_empty() {
- let _ = self
- .search_indexer
- .remove_chats(purged.chat_ids)
- .await
- .inspect_err(|error| tracing::error!(error = ?error, "unable to enqueue purged chats for search"));
- }
if !purged.documents.is_empty() {
let _ = self
.search_indexer
diff --git a/crates/projects/src/domain/service/tests.rs b/crates/projects/src/domain/service/tests.rs
index b986ddb57c..5d2d5a7a82 100644
--- a/crates/projects/src/domain/service/tests.rs
+++ b/crates/projects/src/domain/service/tests.rs
@@ -153,10 +153,6 @@ impl EntityAccessManagementService for NullPort {
}
impl ProjectSearchIndexer for NullPort {
- async fn remove_chats(&self, _chat_ids: Vec<String>) -> anyhow::Result<()> {
- unreachable!()
- }
-
async fn remove_documents(&self, _document_ids: Vec<String>) -> anyhow::Result<()> {
unreachable!()
}
@@ -224,10 +220,6 @@ struct RecordingIndexer {
}
impl ProjectSearchIndexer for RecordingIndexer {
- async fn remove_chats(&self, _chat_ids: Vec<String>) -> anyhow::Result<()> {
- unreachable!()
- }
-
async fn remove_documents(&self, _document_ids: Vec<String>) -> anyhow::Result<()> {
unreachable!()
}
@@ -1574,10 +1566,6 @@ impl OrderedIndexer {
}
impl ProjectSearchIndexer for OrderedIndexer {
- async fn remove_chats(&self, _chat_ids: Vec<String>) -> anyhow::Result<()> {
- self.record("chats")
- }
-
async fn remove_documents(&self, _document_ids: Vec<String>) -> anyhow::Result<()> {
self.record("documents")
}
@@ -1740,7 +1728,7 @@ async fn permanent_delete_runs_external_work_after_committed_purge() {
assert_eq!(
*events.lock().unwrap(),
- vec!["repo", "sha", "chats", "documents", "document_deletes"]
+ vec!["repo", "sha", "documents", "document_deletes"]
);
let published = published.lock().unwrap();
assert_eq!(published.len(), 1);
diff --git a/crates/projects/src/outbound/sqs_search_indexer.rs b/crates/projects/src/outbound/sqs_search_indexer.rs
index 447429bc2c..4455625854 100644
--- a/crates/projects/src/outbound/sqs_search_indexer.rs
+++ b/crates/projects/src/outbound/sqs_search_indexer.rs
@@ -1,12 +1,12 @@
-//! SQS adapter for cross-family search cleanup and document deletion.
+//! SQS adapter for document search cleanup and deletion.
use std::sync::Arc;
-use sqs_client::search::{SearchQueueMessage, chat::RemoveChatMessage, document::DocumentId};
+use sqs_client::search::{SearchQueueMessage, document::DocumentId};
use crate::domain::ports::ProjectSearchIndexer;
-/// SQS-backed cross-family search cleanup and document-deletion adapter.
+/// SQS-backed document search cleanup and deletion adapter.
#[derive(Clone)]
pub struct SqsProjectSearchIndexer {
sqs: Arc<sqs_client::SQS>,
@@ -20,24 +20,6 @@ impl SqsProjectSearchIndexer {
}
impl ProjectSearchIndexer for SqsProjectSearchIndexer {
- #[tracing::instrument(skip(self), err)]
- async fn remove_chats(&self, chat_ids: Vec<String>) -> anyhow::Result<()> {
- let messages = chat_ids
- .into_iter()
- .map(|chat_id| {
- SearchQueueMessage::RemoveChatMessage(RemoveChatMessage {
- chat_id,
- message_id: None,
- index_override: None,
- })
- })
- .collect();
-
- self.sqs
- .bulk_send_message_to_search_event_queue(messages)
- .await
- }
-
#[tracing::instrument(skip(self), err)]
async fn remove_documents(&self, document_ids: Vec<String>) -> anyhow::Result<()> {
let messages = document_ids
diff --git a/crates/sqs_client/src/search/chat.rs b/crates/sqs_client/src/search/chat.rs
index 1180d5db73..f6f0b8dae3 100644
--- a/crates/sqs_client/src/search/chat.rs
+++ b/crates/sqs_client/src/search/chat.rs
@@ -1,5 +1,13 @@
use chrono::{DateTime, Utc};
+#[cfg(test)]
+mod test;
+
+/// SQS backfill work-queue contract for reconciling a chat message.
+///
+/// Live chat updates use Kafka. This payload remains available so search
+/// backfills can target an alternate OpenSearch index with
+/// [`Self::index_override`].
#[derive(serde::Serialize, serde::Deserialize, Debug, PartialEq, Eq)]
pub struct ChatMessage {
/// The chat id
@@ -14,14 +22,3 @@ pub struct ChatMessage {
#[serde(skip_serializing_if = "Option::is_none", default)]
pub index_override: Option<String>,
}
-
-#[derive(serde::Serialize, serde::Deserialize, Debug, PartialEq, Eq)]
-pub struct RemoveChatMessage {
- /// The chat id to remove
- pub chat_id: String,
- /// The message id to remove, if None then all messages for the chat will be removed
- pub message_id: Option<String>,
- /// Optional override for the target OpenSearch index
- #[serde(skip_serializing_if = "Option::is_none", default)]
- pub index_override: Option<String>,
-}
diff --git a/crates/sqs_client/src/search/chat/test.rs b/crates/sqs_client/src/search/chat/test.rs
new file mode 100644
index 0000000000..d7d850bf41
--- /dev/null
+++ b/crates/sqs_client/src/search/chat/test.rs
@@ -0,0 +1,51 @@
+use super::*;
+use crate::{
+ PrimaryId,
+ search::{Operation, SearchQueueMessage},
+};
+
+#[test]
+fn chat_message_with_index_override_round_trips() {
+ let message = SearchQueueMessage::ChatMessage(ChatMessage {
+ chat_id: "chat-1".to_string(),
+ message_id: "message-1".to_string(),
+ user_id: "user-1".to_string(),
+ created_at: "2026-01-02T03:04:05Z".parse().unwrap(),
+ updated_at: "2026-01-02T04:05:06Z".parse().unwrap(),
+ index_override: Some("chats-backfill-v2".to_string()),
+ });
+
+ let serialized = serde_json::to_string(&message).unwrap();
+ let deserialized: SearchQueueMessage = serde_json::from_str(&serialized).unwrap();
+
+ assert_eq!(deserialized, message);
+ assert_eq!(deserialized.id(), "message-1");
+ assert!(matches!(deserialized.operation(), Operation::ExtractText));
+}
+
+#[test]
+fn legacy_chat_backfill_without_index_override_deserializes() {
+ let payload = r#"{
+ "ChatMessage": {
+ "chat_id": "chat-1",
+ "message_id": "message-1",
+ "user_id": "user-1",
+ "created_at": "2026-01-02T03:04:05Z",
+ "updated_at": "2026-01-02T04:05:06Z"
+ }
+ }"#;
+
+ let message: SearchQueueMessage = serde_json::from_str(payload).unwrap();
+
+ assert_eq!(
+ message,
+ SearchQueueMessage::ChatMessage(ChatMessage {
+ chat_id: "chat-1".to_string(),
+ message_id: "message-1".to_string(),
+ user_id: "user-1".to_string(),
+ created_at: "2026-01-02T03:04:05Z".parse().unwrap(),
+ updated_at: "2026-01-02T04:05:06Z".parse().unwrap(),
+ index_override: None,
+ })
+ );
+}
diff --git a/crates/sqs_client/src/search/mod.rs b/crates/sqs_client/src/search/mod.rs
index e1bd1f8bfc..44fd2d6b78 100644
--- a/crates/sqs_client/src/search/mod.rs
+++ b/crates/sqs_client/src/search/mod.rs
@@ -3,7 +3,7 @@ use crate::{
search::{
call::{CallRecordMessage, RemoveCallRecord},
channel::ChannelMessageUpdate,
- chat::{ChatMessage, RemoveChatMessage},
+ chat::ChatMessage,
document::{DocumentId, DocumentPropertiesUpdate, SearchExtractorMessage},
email::{EmailLinkMessage, EmailMessage, EmailThreadBatchMessage, EmailThreadMessage},
project::UpsertProject,
@@ -75,8 +75,8 @@ pub enum SearchQueueMessage {
UpdateDocumentProperties(DocumentPropertiesUpdate),
UpdateDocumentName(DocumentId),
// Chat
+ /// SQS backfill work-queue contract for reconciling a chat message.
ChatMessage(ChatMessage),
- RemoveChatMessage(RemoveChatMessage),
// Email
ExtractEmailMessage(EmailMessage),
RemoveEmailMessage(EmailMessage),
@@ -103,9 +103,8 @@ impl PrimaryId for SearchQueueMessage {
SearchQueueMessage::ExtractSync(message) => message.document_id.clone(),
SearchQueueMessage::UpdateDocumentProperties(message) => message.document_id.clone(),
SearchQueueMessage::UpdateDocumentName(message) => message.document_id.clone(),
- SearchQueueMessage::ChatMessage(message) => message.message_id.clone(), // needs
- // to be the message id to ensure it's unique for batch
- SearchQueueMessage::RemoveChatMessage(message) => message.chat_id.clone(),
+ // The message id keeps entries unique within an SQS batch.
+ SearchQueueMessage::ChatMessage(message) => message.message_id.clone(),
SearchQueueMessage::ExtractEmailMessage(message)
| SearchQueueMessage::RemoveEmailMessage(message) => message.message_id.clone(),
SearchQueueMessage::ExtractEmailThreadMessage(message) => message.thread_id.clone(),
@@ -138,7 +137,6 @@ impl SearchQueueMessage {
SearchQueueMessage::UpdateDocumentName(_) => Operation::UpdateMetadata,
// Chat
SearchQueueMessage::ChatMessage(_) => Operation::ExtractText,
- SearchQueueMessage::RemoveChatMessage(_) => Operation::Remove,
// Email
SearchQueueMessage::ExtractEmailMessage(_) => Operation::ExtractText,
SearchQueueMessage::RemoveEmailMessage(_) => Operation::Remove,
diff --git a/services/deleted_item_poller/Cargo.toml b/services/deleted_item_poller/Cargo.toml
index eba4186fcf..1280e61cb1 100644
--- a/services/deleted_item_poller/Cargo.toml
+++ b/services/deleted_item_poller/Cargo.toml
@@ -10,6 +10,7 @@ aws-sdk-sqs = { workspace = true }
macro_aws_config = { path = "../../crates/macro_aws_config" }
aws-smithy-types = { workspace = true }
aws_lambda_events = { workspace = true, features = ["eventbridge"] }
+chat = { path = "../../crates/chat", default-features = false }
chrono = { workspace = true }
futures = { workspace = true }
lambda_runtime = { workspace = true }
diff --git a/services/deleted_item_poller/src/config.rs b/services/deleted_item_poller/src/config.rs
index 5780c8e957..ee3db73e7b 100644
--- a/services/deleted_item_poller/src/config.rs
+++ b/services/deleted_item_poller/src/config.rs
@@ -13,7 +13,7 @@ pub struct Config {
/// The connection URL for the Postgres database this application should use.
pub database_url: String,
- /// Comma-separated Kafka bootstrap servers for project lifecycle events.
+ /// Comma-separated Kafka bootstrap servers for project and chat lifecycle events.
pub kafka_brokers: KafkaBrokers,
/// The environment we are in
diff --git a/services/deleted_item_poller/src/handler.rs b/services/deleted_item_poller/src/handler.rs
index 4599839b54..5fe629c4cf 100644
--- a/services/deleted_item_poller/src/handler.rs
+++ b/services/deleted_item_poller/src/handler.rs
@@ -4,6 +4,7 @@ mod test;
use crate::context::{self};
use anyhow::Context;
use aws_lambda_events::eventbridge::EventBridgeEvent;
+use chat::domain::events::{ChatMacroEvent, ChatPermanentlyDeletedMetadata};
use futures::future::join_all;
use lambda_runtime::{
Error, LambdaEvent,
@@ -13,7 +14,7 @@ use macro_db_client::projects::ProjectToDelete;
use macro_event_broker::MacroEventBroker;
use macro_user_id::user_id::MacroUserIdStr;
use projects::domain::events::{ProjectMacroEvent, ProjectPermanentlyDeletedMetadata};
-use sqs_client::search::{SearchQueueMessage, chat::RemoveChatMessage, document::DocumentId};
+use sqs_client::search::{SearchQueueMessage, document::DocumentId};
#[tracing::instrument(skip(ctx, _event), err)]
pub async fn handler(
@@ -76,6 +77,42 @@ async fn publish_project_purge_events<B: MacroEventBroker>(
Ok(())
}
+#[tracing::instrument(skip(event_broker, chat_ids), err)]
+async fn publish_chat_purge_events<B: MacroEventBroker>(
+ event_broker: &B,
+ chat_ids: &[String],
+) -> anyhow::Result<()> {
+ let events = chat_ids
+ .iter()
+ .map(|chat_id| {
+ ChatMacroEvent::permanently_deleted(ChatPermanentlyDeletedMetadata {
+ chat_id: chat_id.clone(),
+ actor_user_id: None,
+ project_id: None,
+ })
+ })
+ .collect::<Vec<_>>();
+
+ let publications = events
+ .iter()
+ .map(|event| event_broker.send_event(event))
+ .collect::<Vec<_>>();
+ let publication_results = join_all(publications.into_iter().map(|publication| async move {
+ let handle = publication.context("failed to enqueue chat purge event")?;
+ handle
+ .await
+ .context("chat purge event publication task failed")?
+ .context("failed to publish chat purge event")
+ }))
+ .await;
+
+ for result in publication_results {
+ result?;
+ }
+
+ Ok(())
+}
+
#[tracing::instrument(skip(ctx), err)]
async fn handle_projects(ctx: &context::Context) -> anyhow::Result<()> {
let date = chrono::Utc::now().naive_utc() - chrono::Duration::days(30);
@@ -121,20 +158,9 @@ async fn handle_chats(ctx: &context::Context) -> anyhow::Result<()> {
tracing::debug!(chats_to_delete=?chats_to_delete, "chats to delete");
- ctx.sqs_client
- .bulk_send_message_to_search_event_queue(
- chats_to_delete
- .iter()
- .map(|id| {
- SearchQueueMessage::RemoveChatMessage(RemoveChatMessage {
- chat_id: id.to_string(),
- message_id: None,
- index_override: None,
- })
- })
- .collect(),
- )
- .await?;
+ publish_chat_purge_events(&ctx.macro_event_broker, &chats_to_delete)
+ .await
+ .context("unable to publish chat purge events")?;
ctx.sqs_client
.bulk_enqueue_chat_delete(chats_to_delete)
diff --git a/services/deleted_item_poller/src/handler/test.rs b/services/deleted_item_poller/src/handler/test.rs
index c342399143..10f77529c0 100644
--- a/services/deleted_item_poller/src/handler/test.rs
+++ b/services/deleted_item_poller/src/handler/test.rs
@@ -2,6 +2,7 @@ use std::sync::{Arc, Mutex};
use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker};
use serde_json::{Value, json};
+use tokio::sync::Semaphore;
use super::*;
@@ -12,16 +13,47 @@ struct PublishedEvent {
envelope: Value,
}
+#[derive(Clone, Default)]
+enum DeliveryBehavior {
+ #[default]
+ Succeed,
+ Fail,
+ FailToJoin,
+ Wait(Arc<Semaphore>),
+}
+
#[derive(Clone, Default)]
struct FakeEventBroker {
published: Arc<Mutex<Vec<PublishedEvent>>>,
- fail_publication: bool,
+ fail_send: bool,
+ delivery_behavior: DeliveryBehavior,
}
impl FakeEventBroker {
- fn failing() -> Self {
+ fn failing_send() -> Self {
+ Self {
+ fail_send: true,
+ ..Self::default()
+ }
+ }
+
+ fn failing_delivery() -> Self {
+ Self {
+ delivery_behavior: DeliveryBehavior::Fail,
+ ..Self::default()
+ }
+ }
+
+ fn failing_join() -> Self {
+ Self {
+ delivery_behavior: DeliveryBehavior::FailToJoin,
+ ..Self::default()
+ }
+ }
+
+ fn waiting_for_delivery(delivery_gate: Arc<Semaphore>) -> Self {
Self {
- fail_publication: true,
+ delivery_behavior: DeliveryBehavior::Wait(delivery_gate),
..Self::default()
}
}
@@ -36,22 +68,40 @@ impl MacroEventBroker for FakeEventBroker {
&self,
event: &E,
) -> Result<tokio::task::JoinHandle<Result<(), EventBrokerError>>, EventBrokerError> {
+ if self.fail_send {
+ return Err(EventBrokerError::Publish(
+ "event enqueue rejected".to_string(),
+ ));
+ }
+
self.published.lock().unwrap().push(PublishedEvent {
topic: event.topic().to_string(),
key: event.key().to_string(),
envelope: serde_json::to_value(event.event())?,
});
- let fail_publication = self.fail_publication;
- Ok(tokio::spawn(async move {
- if fail_publication {
+ let delivery_handle = match self.delivery_behavior.clone() {
+ DeliveryBehavior::Succeed => tokio::spawn(async { Ok(()) }),
+ DeliveryBehavior::Fail => tokio::spawn(async {
Err(EventBrokerError::Publish(
"publisher unavailable".to_string(),
))
- } else {
- Ok(())
+ }),
+ DeliveryBehavior::FailToJoin => {
+ let handle = tokio::spawn(std::future::pending::<Result<(), EventBrokerError>>());
+ handle.abort();
+ handle
}
- }))
+ DeliveryBehavior::Wait(delivery_gate) => tokio::spawn(async move {
+ let _permit = delivery_gate
+ .acquire_owned()
+ .await
+ .expect("delivery gate should remain open");
+ Ok(())
+ }),
+ };
+
+ Ok(delivery_handle)
}
}
@@ -98,7 +148,7 @@ async fn publish_project_purge_events_publishes_separately_keyed_events() {
#[tokio::test]
async fn publish_project_purge_events_returns_publication_failures() {
- let event_broker = FakeEventBroker::failing();
+ let event_broker = FakeEventBroker::failing_delivery();
let projects = vec![ProjectToDelete {
project_id: "project-one".to_string(),
user_id: "macro|[email protected]".to_string(),
@@ -126,3 +176,92 @@ async fn publish_project_purge_events_rejects_malformed_owners() {
assert!(format!("{error:#}").contains("invalid owner for project project-one"));
assert!(event_broker.published().is_empty());
}
+
+#[tokio::test]
+async fn publish_chat_purge_events_publishes_separately_keyed_events() {
+ let event_broker = FakeEventBroker::default();
+ let chat_ids = vec!["chat-one".to_string(), "chat-two".to_string()];
+
+ publish_chat_purge_events(&event_broker, &chat_ids)
+ .await
+ .unwrap();
+
+ let published = event_broker.published();
+ assert_eq!(published.len(), chat_ids.len());
+
+ for (event, chat_id) in published.iter().zip(&chat_ids) {
+ assert_eq!(event.topic, "macro.chats");
+ assert_eq!(event.key, *chat_id);
+ assert_eq!(event.envelope["schema_version"], json!(1));
+ assert_eq!(
+ event.envelope["event_type"],
+ json!("chat.permanently_deleted")
+ );
+
+ let metadata = &event.envelope["metadata"];
+ assert_eq!(metadata["chat_id"], json!(chat_id));
+ assert_eq!(metadata["actor_user_id"], Value::Null);
+ assert_eq!(metadata["project_id"], Value::Null);
+ }
+}
+
+#[tokio::test]
+async fn publish_chat_purge_events_returns_immediate_send_failures() {
+ let event_broker = FakeEventBroker::failing_send();
+ let chat_ids = vec!["chat-one".to_string()];
+
+ let error = publish_chat_purge_events(&event_broker, &chat_ids)
+ .await
+ .expect_err("immediate send failure should be returned");
+
+ assert!(format!("{error:#}").contains("event enqueue rejected"));
+}
+
+#[tokio::test]
+async fn publish_chat_purge_events_returns_delivery_failures() {
+ let event_broker = FakeEventBroker::failing_delivery();
+ let chat_ids = vec!["chat-one".to_string()];
+
+ let error = publish_chat_purge_events(&event_broker, &chat_ids)
+ .await
+ .expect_err("delivery failure should be returned");
+
+ assert!(format!("{error:#}").contains("publisher unavailable"));
+}
+
+#[tokio::test]
+async fn publish_chat_purge_events_returns_delivery_join_failures() {
+ let event_broker = FakeEventBroker::failing_join();
+ let chat_ids = vec!["chat-one".to_string()];
+
+ let error = publish_chat_purge_events(&event_broker, &chat_ids)
+ .await
+ .expect_err("delivery join failure should be returned");
+
+ assert!(
+ format!("{error:#}").contains("chat purge event publication task failed"),
+ "unexpected error: {error:#}"
+ );
+}
+
+#[tokio::test]
+async fn publish_chat_purge_events_waits_for_every_delivery() {
+ let delivery_gate = Arc::new(Semaphore::new(0));
+ let event_broker = FakeEventBroker::waiting_for_delivery(delivery_gate.clone());
+ let chat_ids = vec!["chat-one".to_string(), "chat-two".to_string()];
+
+ let publication_task = tokio::spawn({
+ let event_broker = event_broker.clone();
+ let chat_ids = chat_ids.clone();
+ async move { publish_chat_purge_events(&event_broker, &chat_ids).await }
+ });
+
+ while event_broker.published().len() < chat_ids.len() {
+ tokio::task::yield_now().await;
+ }
+
+ assert!(!publication_task.is_finished());
+
+ delivery_gate.add_permits(chat_ids.len());
+ publication_task.await.unwrap().unwrap();
+}
diff --git a/services/document_cognition_service/Cargo.toml b/services/document_cognition_service/Cargo.toml
index 0534e8b949..093d2d0621 100644
--- a/services/document_cognition_service/Cargo.toml
+++ b/services/document_cognition_service/Cargo.toml
@@ -120,7 +120,6 @@ sqs_client = { path = "../../crates/sqs_client", default-features = false, featu
"document_text_extractor",
"email",
"gmail",
- "search",
] }
sqs_worker = { path = "../../crates/sqs_worker" }
static_file = { path = "../../crates/static_file", features = ["attachment"] }
diff --git a/services/document_cognition_service/src/api/stream/chat_message/mod.rs b/services/document_cognition_service/src/api/stream/chat_message/mod.rs
index 5e62614c59..e3f1ccb9c9 100644
--- a/services/document_cognition_service/src/api/stream/chat_message/mod.rs
+++ b/services/document_cognition_service/src/api/stream/chat_message/mod.rs
@@ -251,7 +251,7 @@ async fn send_chat_message_inner(
};
// Store the incoming user message and resolve its attachments
- let resolved = store_incoming_message(ctx.clone(), user_id.0.as_ref(), &chat, &model, &payload)
+ let resolved = store_incoming_message(ctx.clone(), user_id.0.as_ref(), &model, &payload)
.await
.map_err(|err| {
tracing::error!(error=?err, "failed to store incoming message");
@@ -674,7 +674,6 @@ fn stream_and_save_message(
if let Err(err) = store_conversation_messages(
ctx.clone(),
- user_id.0.as_ref(),
&chat_id,
new_messages,
&model,
diff --git a/services/document_cognition_service/src/api/stream/util/chat_message/mod.rs b/services/document_cognition_service/src/api/stream/util/chat_message/mod.rs
index 520adc9ea3..6385cdc79e 100644
--- a/services/document_cognition_service/src/api/stream/util/chat_message/mod.rs
+++ b/services/document_cognition_service/src/api/stream/util/chat_message/mod.rs
@@ -1,11 +1,7 @@
pub mod ai_request;
pub mod toolset;
-use crate::model::chats::ChatResponse;
-use crate::{
- api::{context::ApiContext, utils::search},
- model::stream::SendChatMessagePayload,
-};
+use crate::{api::context::ApiContext, model::stream::SendChatMessagePayload};
use agent::types::Role;
use agent::types::{ChatMessage, ChatMessageContent};
@@ -17,11 +13,10 @@ use model_entity::EntityType;
use std::sync::Arc;
/// Stores the incoming user message and resolves its attachments in one step.
-#[tracing::instrument(err, skip(ctx, chat, incoming_message), fields(chat_id=?incoming_message.chat_id))]
+#[tracing::instrument(err, skip(ctx, incoming_message), fields(chat_id=?incoming_message.chat_id))]
pub async fn store_incoming_message(
ctx: Arc<ApiContext>,
user_id: &str,
- chat: &ChatResponse,
model: &str,
incoming_message: &SendChatMessagePayload,
) -> Result<ResolvedMessageContent> {
@@ -56,22 +51,10 @@ pub async fn store_incoming_message(
let user_id: macro_user_id::user_id::MacroUserIdStr<'static> =
user_id.to_owned().try_into().map_err(anyhow::Error::msg)?;
- let resolved = ctx
- .message_service
+ ctx.message_service
.create(&user_id, &incoming_message.chat_id, new_chat_message)
.await
- .context("failed to create chat message")?;
-
- search::send_chat_message_to_search(
- &ctx,
- &chat.id,
- &resolved.message_id,
- user_id.as_ref(),
- created_at,
- created_at,
- );
-
- Ok(resolved)
+ .context("failed to create chat message")
}
/// Stores multiple conversation messages to the database.
@@ -79,7 +62,6 @@ pub async fn store_incoming_message(
#[tracing::instrument(err, skip(ctx, messages), fields(chat_id=?chat_id, message_count=messages.len()))]
pub async fn store_conversation_messages(
ctx: Arc<ApiContext>,
- user_id: &str,
chat_id: &str,
messages: Vec<ChatMessage>,
model: &str,
@@ -118,16 +100,7 @@ pub async fn store_conversation_messages(
.await
.context("failed to create chat message")?;
- message_ids.push(message_id.clone());
-
- search::send_chat_message_to_search(
- &ctx,
- chat_id,
- &message_id,
- user_id,
- created_at,
- created_at,
- );
+ message_ids.push(message_id);
}
Ok(message_ids)
diff --git a/services/document_cognition_service/src/api/utils/mod.rs b/services/document_cognition_service/src/api/utils/mod.rs
index b117ff4ffd..f4ee9bc8d0 100644
--- a/services/document_cognition_service/src/api/utils/mod.rs
+++ b/services/document_cognition_service/src/api/utils/mod.rs
@@ -1,2 +1 @@
pub mod log;
-pub mod search;
diff --git a/services/document_cognition_service/src/api/utils/search.rs b/services/document_cognition_service/src/api/utils/search.rs
deleted file mode 100644
index 396128dea9..0000000000
--- a/services/document_cognition_service/src/api/utils/search.rs
+++ /dev/null
@@ -1,58 +0,0 @@
-use crate::api::context::ApiContext;
-use chrono::{DateTime, Utc};
-use sqs_client::search::SearchQueueMessage;
-
-/// Spawns a tokio task to send a remove chat message from the search text extractor
-// used in edit_message which is dead for now
-#[allow(dead_code)]
-pub fn send_remove_chat_message_to_search(ctx: &ApiContext, chat_id: &str, message_id: &str) {
- let chat_id = chat_id.to_string();
- let message_id = message_id.to_string();
- tokio::spawn({
- let sqs_client = ctx.sqs_client.clone();
- async move {
- let _ = sqs_client
- .send_message_to_search_event_queue(
- sqs_client::search::SearchQueueMessage::RemoveChatMessage(
- sqs_client::search::chat::RemoveChatMessage {
- chat_id: chat_id.to_string(),
- message_id: Some(message_id.to_string()),
- index_override: None,
- },
- ),
- )
- .await;
- }
- });
-}
-
-/// Spawns a tokio task to send the chat message to the search text extractor
-pub fn send_chat_message_to_search(
- ctx: &ApiContext,
- chat_id: &str,
- message_id: &str,
- user_id: &str,
- created_at: DateTime<Utc>,
- updated_at: DateTime<Utc>,
-) {
- let chat_id = chat_id.to_string();
- let message_id = message_id.to_string();
- let user_id = user_id.to_string();
- tokio::spawn({
- let sqs_client = ctx.sqs_client.clone();
- async move {
- let _ = sqs_client
- .send_message_to_search_event_queue(SearchQueueMessage::ChatMessage(
- sqs_client::search::chat::ChatMessage {
- chat_id,
- message_id,
- user_id,
- created_at,
- updated_at,
- index_override: None,
- },
- ))
- .await;
- }
- });
-}
diff --git a/services/document_cognition_service/src/main.rs b/services/document_cognition_service/src/main.rs
index 85fd706fc6..9065cd6417 100644
--- a/services/document_cognition_service/src/main.rs
+++ b/services/document_cognition_service/src/main.rs
@@ -103,7 +103,6 @@ async fn main() -> anyhow::Result<()> {
let chat_delete_queue = macro_queues::ChatDeleteQueue::new();
let email_scheduled_queue = macro_queues::EmailScheduledQueue::new();
let gmail_ops_queue = macro_queues::GmailOpsQueue::new();
- let search_event_queue = macro_queues::SearchEventQueue::new();
let ai_projection_queue = macro_queues::AiProjectionQueue::new();
let notification_queue = macro_queues::NotificationIngressQueue::new();
let sqs_client = sqs_client::SQS::new(queue_aws_client)
@@ -111,7 +110,6 @@ async fn main() -> anyhow::Result<()> {
.chat_delete_queue(&chat_delete_queue)
.email_scheduled_queue(&email_scheduled_queue)
.gmail_ops_queue(&gmail_ops_queue)
- .search_event_queue(&search_event_queue)
.ai_projection_queue(&ai_projection_queue);
let internal_api_key = config.internal_api_key.to_string();
diff --git a/services/search_processing_service/Cargo.toml b/services/search_processing_service/Cargo.toml
index bfc46f9bab..4b3957c742 100644
--- a/services/search_processing_service/Cargo.toml
+++ b/services/search_processing_service/Cargo.toml
@@ -10,6 +10,7 @@ default = ["processing", "service", "pdf"]
processing = [
"dep:call",
"dep:channels",
+ "dep:chat",
"dep:kafka_util",
"dep:macro_event_broker",
"dep:macro_event_topics",
@@ -38,6 +39,7 @@ aws-sdk-sqs = { workspace = true }
axum = { workspace = true }
call = { path = "../../crates/call", default-features = false, optional = true }
channels = { path = "../../crates/channels", default-features = false, optional = true }
+chat = { path = "../../crates/chat", default-features = false, optional = true }
chrono = { workspace = true }
comms_db_client = { path = "../../crates/comms_db_client" }
email_db_client = { path = "../../crates/email_db_client" }
diff --git a/services/search_processing_service/src/inbound/kafka_consumer.rs b/services/search_processing_service/src/inbound/kafka_consumer.rs
index 298924a806..cde317db90 100644
--- a/services/search_processing_service/src/inbound/kafka_consumer.rs
+++ b/services/search_processing_service/src/inbound/kafka_consumer.rs
@@ -7,13 +7,14 @@
//! offsets are already committed.
//!
//! Per-entity event mapping and processing live in the [`call`], [`channel`],
-//! and [`project`] submodules; this module owns the poll loop, worker, retry
-//! policy, and commit semantics.
+//! [`chat`], and [`project`] submodules; this module owns the poll loop, worker,
+//! retry policy, and commit semantics.
#![allow(clippy::enum_variant_names)]
mod call;
mod channel;
+mod chat;
mod project;
#[cfg(test)]
mod test;
@@ -21,6 +22,7 @@ mod test;
use std::{future::Future, time::Duration};
use ::call::domain::events::CallMacroEvent;
+use ::chat::domain::events::ChatMacroEvent;
use channels::domain::broker_events::ChannelMacroEvent;
use kafka_util::{GroupName, KafkaEventConsumer};
use macro_event_broker::{KafkaConsumerAdapter, MacroEventCollection, MacroEventConsumerService};
@@ -36,7 +38,8 @@ use tokio::sync::mpsc;
use tokio_retry::{Retry, strategy::ExponentialBackoff};
use self::{
- call::process_call_event, channel::process_channel_event, project::process_project_event,
+ call::process_call_event, channel::process_channel_event, chat::process_chat_event,
+ project::process_project_event,
};
/// Consumer group used for live search-index event offsets.
@@ -55,6 +58,7 @@ macro_event_broker::declare_topics!(
DeclaredMacroEvent:
CallMacroEvent,
ChannelMacroEvent,
+ ChatMacroEvent,
ProjectMacroEvent,
);
@@ -170,6 +174,9 @@ async fn process_event(
DeclaredMacroEvent::ChannelMacroEvent(event) => {
process_channel_event(db, opensearch_client, event, partition, offset).await
}
+ DeclaredMacroEvent::ChatMacroEvent(event) => {
+ process_chat_event(db, opensearch_client, event, partition, offset).await
+ }
DeclaredMacroEvent::ProjectMacroEvent(event) => {
process_project_event(db, opensearch_client, event, partition, offset).await
}
diff --git a/services/search_processing_service/src/inbound/kafka_consumer/chat.rs b/services/search_processing_service/src/inbound/kafka_consumer/chat.rs
new file mode 100644
index 0000000000..5e43256589
--- /dev/null
+++ b/services/search_processing_service/src/inbound/kafka_consumer/chat.rs
@@ -0,0 +1,172 @@
+//! Maps chat lifecycle events to search-index actions and processes them.
+
+use ::chat::domain::events::{ChatMacroEvent, ChatTopicEvent};
+use macro_event_broker::MacroEvent as _;
+use opensearch_client::OpensearchClient;
+use sqlx::PgPool;
+
+use super::{EventOutcome, MAX_PROCESSING_ATTEMPTS, PROCESSING_RETRY_BASE_DELAY, retry_processing};
+use crate::process::chat::{remove_chat_message, upsert_chat_message_by_ids};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(super) enum ChatIndexAction<'a> {
+ UpsertMessage {
+ chat_id: &'a str,
+ message_id: &'a str,
+ },
+ RemoveMessage {
+ chat_id: &'a str,
+ message_id: &'a str,
+ },
+ RemoveChat {
+ chat_id: &'a str,
+ },
+ Ignore,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(super) struct ChatEventDescription<'a> {
+ pub(super) action: ChatIndexAction<'a>,
+ pub(super) chat_id: &'a str,
+ pub(super) event_type: &'static str,
+}
+
+pub(super) fn describe_chat_event(event: &ChatTopicEvent) -> ChatEventDescription<'_> {
+ match event {
+ ChatTopicEvent::Created(metadata) => ChatEventDescription {
+ action: ChatIndexAction::Ignore,
+ chat_id: &metadata.chat_id,
+ event_type: "chat.created",
+ },
+ ChatTopicEvent::Updated(metadata) => ChatEventDescription {
+ action: ChatIndexAction::Ignore,
+ chat_id: &metadata.chat_id,
+ event_type: "chat.updated",
+ },
+ ChatTopicEvent::Deleted(metadata) => ChatEventDescription {
+ action: ChatIndexAction::Ignore,
+ chat_id: &metadata.chat_id,
+ event_type: "chat.deleted",
+ },
+ ChatTopicEvent::PermanentlyDeleted(metadata) => ChatEventDescription {
+ action: ChatIndexAction::RemoveChat {
+ chat_id: &metadata.chat_id,
+ },
+ chat_id: &metadata.chat_id,
+ event_type: "chat.permanently_deleted",
+ },
+ ChatTopicEvent::Restored(metadata) => ChatEventDescription {
+ action: ChatIndexAction::Ignore,
+ chat_id: &metadata.chat_id,
+ event_type: "chat.restored",
+ },
+ ChatTopicEvent::Copied(metadata) => ChatEventDescription {
+ action: ChatIndexAction::Ignore,
+ chat_id: &metadata.chat_id,
+ event_type: "chat.copied",
+ },
+ ChatTopicEvent::MessageSent(metadata) => ChatEventDescription {
+ action: ChatIndexAction::UpsertMessage {
+ chat_id: &metadata.chat_id,
+ message_id: &metadata.message_id,
+ },
+ chat_id: &metadata.chat_id,
+ event_type: "chat.message_sent",
+ },
+ ChatTopicEvent::MessageDeleted(metadata) => ChatEventDescription {
+ action: ChatIndexAction::RemoveMessage {
+ chat_id: &metadata.chat_id,
+ message_id: &metadata.message_id,
+ },
+ chat_id: &metadata.chat_id,
+ event_type: "chat.message_deleted",
+ },
+ }
+}
+
+async fn process_chat_index_action(
+ db: &PgPool,
+ opensearch_client: &OpensearchClient,
+ action: ChatIndexAction<'_>,
+) -> anyhow::Result<()> {
+ match action {
+ ChatIndexAction::UpsertMessage {
+ chat_id,
+ message_id,
+ } => upsert_chat_message_by_ids(opensearch_client, db, chat_id, message_id, None).await,
+ ChatIndexAction::RemoveMessage {
+ chat_id,
+ message_id,
+ } => remove_chat_message(opensearch_client, chat_id, Some(message_id), None).await,
+ ChatIndexAction::RemoveChat { chat_id } => {
+ remove_chat_message(opensearch_client, chat_id, None, None).await
+ }
+ ChatIndexAction::Ignore => Ok(()),
+ }
+}
+
+pub(super) async fn process_chat_event(
+ db: &PgPool,
+ opensearch_client: &OpensearchClient,
+ event: &ChatMacroEvent,
+ partition: i32,
+ offset: i64,
+) -> EventOutcome {
+ let description = describe_chat_event(&event.event().event);
+ if description.action == ChatIndexAction::Ignore {
+ tracing::trace!(
+ chat_id = description.chat_id,
+ event_type = description.event_type,
+ partition,
+ offset,
+ "ignoring chat event without a search-index action"
+ );
+ return EventOutcome::Ignored;
+ }
+
+ let result = retry_processing(|attempt| async move {
+ tracing::trace!(
+ chat_id = description.chat_id,
+ event_type = description.event_type,
+ partition,
+ offset,
+ attempt,
+ "processing chat search-index event"
+ );
+ process_chat_index_action(db, opensearch_client, description.action)
+ .await
+ .inspect_err(|error| {
+ if attempt < MAX_PROCESSING_ATTEMPTS {
+ let retry_delay =
+ PROCESSING_RETRY_BASE_DELAY * 2u32.pow(attempt.saturating_sub(1));
+ tracing::warn!(
+ error = ?error,
+ chat_id = description.chat_id,
+ event_type = description.event_type,
+ partition,
+ offset,
+ attempt,
+ delay_secs = retry_delay.as_secs(),
+ "chat search-index processing failed, retrying"
+ );
+ }
+ })
+ })
+ .await;
+
+ match result {
+ Ok(()) => EventOutcome::Indexed,
+ Err(error) => {
+ tracing::error!(
+ error = ?error,
+ chat_id = description.chat_id,
+ event_type = description.event_type,
+ partition,
+ offset,
+ attempts = MAX_PROCESSING_ATTEMPTS,
+ "dropping chat event after processing retries were exhausted"
+ );
+ EventOutcome::Dropped
+ }
+ }
+}
diff --git a/services/search_processing_service/src/inbound/kafka_consumer/project.rs b/services/search_processing_service/src/inbound/kafka_consumer/project.rs
index 2619764745..c4af0d3a0d 100644
--- a/services/search_processing_service/src/inbound/kafka_consumer/project.rs
+++ b/services/search_processing_service/src/inbound/kafka_consumer/project.rs
@@ -9,11 +9,14 @@ use sqlx::PgPool;
use sqs_client::search::project::UpsertProject;
use super::{EventOutcome, MAX_PROCESSING_ATTEMPTS, PROCESSING_RETRY_BASE_DELAY, retry_processing};
-use crate::process::project::upsert_project;
+use crate::process::{chat::remove_chat_message, project::upsert_project};
#[derive(Debug, Clone, PartialEq, Eq)]
pub(super) enum ProjectIndexAction {
- Reconcile { project_ids: Vec<String> },
+ Reconcile {
+ project_ids: Vec<String>,
+ purged_chat_ids: Vec<String>,
+ },
}
#[derive(Debug, Clone, PartialEq, Eq)]
@@ -46,6 +49,7 @@ pub(super) fn describe_project_event(event: &ProjectTopicEvent) -> ProjectEventD
Some(metadata.project_id.as_str()),
metadata.parent_project_id.as_deref(),
]),
+ purged_chat_ids: Vec::new(),
},
project_id: &metadata.project_id,
event_type: "project.created",
@@ -57,6 +61,7 @@ pub(super) fn describe_project_event(event: &ProjectTopicEvent) -> ProjectEventD
metadata.previous_parent_id.as_deref(),
metadata.parent_id.as_deref(),
]),
+ purged_chat_ids: Vec::new(),
},
project_id: &metadata.project_id,
event_type: "project.updated",
@@ -70,6 +75,7 @@ pub(super) fn describe_project_event(event: &ProjectTopicEvent) -> ProjectEventD
.map(|project_id| Some(project_id.as_str()))
.chain([metadata.parent_project_id.as_deref()]),
),
+ purged_chat_ids: Vec::new(),
},
project_id: &metadata.project_id,
event_type: "project.deleted",
@@ -83,6 +89,7 @@ pub(super) fn describe_project_event(event: &ProjectTopicEvent) -> ProjectEventD
.map(|project_id| Some(project_id.as_str()))
.chain([metadata.parent_project_id.as_deref()]),
),
+ purged_chat_ids: Vec::new(),
},
project_id: &metadata.project_id,
event_type: "project.restored",
@@ -96,6 +103,7 @@ pub(super) fn describe_project_event(event: &ProjectTopicEvent) -> ProjectEventD
.map(|project_id| Some(project_id.as_str()))
.chain([metadata.parent_project_id.as_deref()]),
),
+ purged_chat_ids: metadata.purged_chat_ids.clone(),
},
project_id: &metadata.project_id,
event_type: "project.permanently_deleted",
@@ -109,6 +117,7 @@ pub(super) fn describe_project_event(event: &ProjectTopicEvent) -> ProjectEventD
.map(|project_id| Some(project_id.as_str()))
.chain([metadata.parent_project_id.as_deref()]),
),
+ purged_chat_ids: Vec::new(),
},
project_id: &metadata.root_project_id,
event_type: "project.uploaded",
@@ -122,7 +131,10 @@ async fn process_project_index_action(
action: ProjectIndexAction,
) -> anyhow::Result<()> {
match action {
- ProjectIndexAction::Reconcile { project_ids } => {
+ ProjectIndexAction::Reconcile {
+ project_ids,
+ purged_chat_ids,
+ } => {
for project_id in project_ids {
upsert_project(
opensearch_client,
@@ -134,6 +146,11 @@ async fn process_project_index_action(
)
.await?;
}
+
+ for chat_id in purged_chat_ids {
+ remove_chat_message(opensearch_client, &chat_id, None, None).await?;
+ }
+
Ok(())
}
}
diff --git a/services/search_processing_service/src/inbound/kafka_consumer/test.rs b/services/search_processing_service/src/inbound/kafka_consumer/test.rs
index dd55691cae..b62352a43f 100644
--- a/services/search_processing_service/src/inbound/kafka_consumer/test.rs
+++ b/services/search_processing_service/src/inbound/kafka_consumer/test.rs
@@ -6,6 +6,11 @@ use ::call::domain::events::{
CallRecordSummarizedMetadata, CallRecordUpdatedMetadata, CallRecordingReadyMetadata,
CallStartedMetadata, CallTopicEvent,
};
+use ::chat::domain::events::{
+ ChatCopiedMetadata, ChatCreatedMetadata, ChatDeletedMetadata, ChatMessageDeletedMetadata,
+ ChatMessageRole, ChatMessageSentMetadata, ChatPermanentlyDeletedMetadata, ChatRestoredMetadata,
+ ChatTopicEvent, ChatUpdatedMetadata,
+};
use channels::domain::{
broker_events::{
ChannelCreatedMetadata, ChannelDeletedMetadata, ChannelMessageAttachmentCreatedMetadata,
@@ -18,7 +23,9 @@ use channels::domain::{
};
use chrono::Utc;
use macro_event_broker::{Event, EventBrokerError, MacroEvent as _, MessageParts};
-use macro_event_topics::{MacroCallsTopic, MacroChannelsTopic, MacroProjectsTopic, Topic as _};
+use macro_event_topics::{
+ MacroCallsTopic, MacroChannelsTopic, MacroChatsTopic, MacroProjectsTopic, Topic as _,
+};
use macro_user_id::user_id::MacroUserIdStr;
use projects::domain::events::{
ProjectCreatedMetadata, ProjectDeletedMetadata, ProjectPermanentlyDeletedMetadata,
@@ -29,6 +36,7 @@ use uuid::Uuid;
use super::{
call::{CallEventDescription, CallIndexAction, describe_call_event},
channel::{ChannelEventDescription, ChannelIndexAction, describe_channel_event},
+ chat::{ChatEventDescription, ChatIndexAction, describe_chat_event},
project::{
ProjectEventDescription, ProjectIndexAction, collect_project_ids, describe_project_event,
},
@@ -38,6 +46,9 @@ use super::{
const CALL_ID: Uuid = Uuid::from_u128(1);
const CHANNEL_ID: Uuid = Uuid::from_u128(2);
const MESSAGE_ID: Uuid = Uuid::from_u128(3);
+const CHAT_ID: &str = "chat-id";
+const SECOND_CHAT_ID: &str = "second-chat-id";
+const CHAT_MESSAGE_ID: &str = "chat-message-id";
const PROJECT_ID: &str = "project-root";
const CHILD_PROJECT_ID: &str = "project-child";
const PARENT_PROJECT_ID: &str = "project-parent";
@@ -140,6 +151,122 @@ fn encoded_message<E: serde::Serialize>(
}
}
+fn chat_event_cases() -> Vec<(ChatTopicEvent, ChatEventDescription<'static>)> {
+ let owner = user_id();
+
+ vec![
+ (
+ ChatTopicEvent::Created(ChatCreatedMetadata {
+ chat_id: CHAT_ID.to_string(),
+ owner: owner.clone(),
+ name: "Chat".to_string(),
+ project_id: Some(PROJECT_ID.to_string()),
+ }),
+ ChatEventDescription {
+ action: ChatIndexAction::Ignore,
+ chat_id: CHAT_ID,
+ event_type: "chat.created",
+ },
+ ),
+ (
+ ChatTopicEvent::Updated(ChatUpdatedMetadata {
+ chat_id: CHAT_ID.to_string(),
+ actor_user_id: owner.clone(),
+ name: Some("Renamed chat".to_string()),
+ previous_project_id: Some(PROJECT_ID.to_string()),
+ project_id: Some(PARENT_PROJECT_ID.to_string()),
+ share_permission_updated: true,
+ }),
+ ChatEventDescription {
+ action: ChatIndexAction::Ignore,
+ chat_id: CHAT_ID,
+ event_type: "chat.updated",
+ },
+ ),
+ (
+ ChatTopicEvent::Deleted(ChatDeletedMetadata {
+ chat_id: CHAT_ID.to_string(),
+ actor_user_id: Some(owner.clone()),
+ project_id: Some(PROJECT_ID.to_string()),
+ }),
+ ChatEventDescription {
+ action: ChatIndexAction::Ignore,
+ chat_id: CHAT_ID,
+ event_type: "chat.deleted",
+ },
+ ),
+ (
+ ChatTopicEvent::PermanentlyDeleted(ChatPermanentlyDeletedMetadata {
+ chat_id: CHAT_ID.to_string(),
+ actor_user_id: Some(owner.clone()),
+ project_id: Some(PROJECT_ID.to_string()),
+ }),
+ ChatEventDescription {
+ action: ChatIndexAction::RemoveChat { chat_id: CHAT_ID },
+ chat_id: CHAT_ID,
+ event_type: "chat.permanently_deleted",
+ },
+ ),
+ (
+ ChatTopicEvent::Restored(ChatRestoredMetadata {
+ chat_id: CHAT_ID.to_string(),
+ actor_user_id: Some(owner.clone()),
+ project_id: Some(PROJECT_ID.to_string()),
+ }),
+ ChatEventDescription {
+ action: ChatIndexAction::Ignore,
+ chat_id: CHAT_ID,
+ event_type: "chat.restored",
+ },
+ ),
+ (
+ ChatTopicEvent::Copied(ChatCopiedMetadata {
+ chat_id: CHAT_ID.to_string(),
+ source_chat_id: "source-chat-id".to_string(),
+ owner: owner.clone(),
+ name: "Copied chat".to_string(),
+ }),
+ ChatEventDescription {
+ action: ChatIndexAction::Ignore,
+ chat_id: CHAT_ID,
+ event_type: "chat.copied",
+ },
+ ),
+ (
+ ChatTopicEvent::MessageSent(ChatMessageSentMetadata {
+ chat_id: CHAT_ID.to_string(),
+ message_id: CHAT_MESSAGE_ID.to_string(),
+ role: ChatMessageRole::User,
+ model: "chat-model".to_string(),
+ actor_user_id: Some(owner),
+ attachment_count: 1,
+ }),
+ ChatEventDescription {
+ action: ChatIndexAction::UpsertMessage {
+ chat_id: CHAT_ID,
+ message_id: CHAT_MESSAGE_ID,
+ },
+ chat_id: CHAT_ID,
+ event_type: "chat.message_sent",
+ },
+ ),
+ (
+ ChatTopicEvent::MessageDeleted(ChatMessageDeletedMetadata {
+ chat_id: CHAT_ID.to_string(),
+ message_id: CHAT_MESSAGE_ID.to_string(),
+ }),
+ ChatEventDescription {
+ action: ChatIndexAction::RemoveMessage {
+ chat_id: CHAT_ID,
+ message_id: CHAT_MESSAGE_ID,
+ },
+ chat_id: CHAT_ID,
+ event_type: "chat.message_deleted",
+ },
+ ),
+ ]
+}
+
fn channel_event_cases() -> Vec<(ChannelTopicEvent, ChannelEventDescription)> {
let sender = channel_sender();
@@ -318,6 +445,7 @@ fn project_event_cases() -> Vec<(ProjectTopicEvent, ProjectEventDescription<'sta
ProjectEventDescription {
action: ProjectIndexAction::Reconcile {
project_ids: vec![PROJECT_ID.to_string(), PARENT_PROJECT_ID.to_string()],
+ purged_chat_ids: Vec::new(),
},
project_id: PROJECT_ID,
event_type: "project.created",
@@ -340,6 +468,7 @@ fn project_event_cases() -> Vec<(ProjectTopicEvent, ProjectEventDescription<'sta
PARENT_PROJECT_ID.to_string(),
NEW_PARENT_PROJECT_ID.to_string(),
],
+ purged_chat_ids: Vec::new(),
},
project_id: PROJECT_ID,
event_type: "project.updated",
@@ -358,6 +487,7 @@ fn project_event_cases() -> Vec<(ProjectTopicEvent, ProjectEventDescription<'sta
ProjectEventDescription {
action: ProjectIndexAction::Reconcile {
project_ids: vec![PROJECT_ID.to_string(), PARENT_PROJECT_ID.to_string()],
+ purged_chat_ids: Vec::new(),
},
project_id: PROJECT_ID,
event_type: "project.updated",
@@ -385,6 +515,7 @@ fn project_event_cases() -> Vec<(ProjectTopicEvent, ProjectEventDescription<'sta
CHILD_PROJECT_ID.to_string(),
PARENT_PROJECT_ID.to_string(),
],
+ purged_chat_ids: Vec::new(),
},
project_id: PROJECT_ID,
event_type: "project.deleted",
@@ -405,6 +536,7 @@ fn project_event_cases() -> Vec<(ProjectTopicEvent, ProjectEventDescription<'sta
CHILD_PROJECT_ID.to_string(),
PARENT_PROJECT_ID.to_string(),
],
+ purged_chat_ids: Vec::new(),
},
project_id: PROJECT_ID,
event_type: "project.restored",
@@ -418,7 +550,7 @@ fn project_event_cases() -> Vec<(ProjectTopicEvent, ProjectEventDescription<'sta
parent_project_id: Some(PARENT_PROJECT_ID.to_string()),
purged_project_ids: vec![PROJECT_ID.to_string(), CHILD_PROJECT_ID.to_string()],
purged_document_ids: vec!["document-id".to_string()],
- purged_chat_ids: vec!["chat-id".to_string()],
+ purged_chat_ids: vec![CHAT_ID.to_string(), SECOND_CHAT_ID.to_string()],
}),
ProjectEventDescription {
action: ProjectIndexAction::Reconcile {
@@ -427,6 +559,7 @@ fn project_event_cases() -> Vec<(ProjectTopicEvent, ProjectEventDescription<'sta
CHILD_PROJECT_ID.to_string(),
PARENT_PROJECT_ID.to_string(),
],
+ purged_chat_ids: vec![CHAT_ID.to_string(), SECOND_CHAT_ID.to_string()],
},
project_id: PROJECT_ID,
event_type: "project.permanently_deleted",
@@ -447,6 +580,7 @@ fn project_event_cases() -> Vec<(ProjectTopicEvent, ProjectEventDescription<'sta
CHILD_PROJECT_ID.to_string(),
PARENT_PROJECT_ID.to_string(),
],
+ purged_chat_ids: Vec::new(),
},
project_id: PROJECT_ID,
event_type: "project.uploaded",
@@ -462,8 +596,10 @@ fn subscribes_to_declared_search_processing_topics_with_durable_group() {
"search-processing-service"
);
let topics = DeclaredMacroEvent::topics();
+ assert_eq!(MacroChatsTopic::TOPIC_STR, "macro.chats");
assert!(topics.contains(&MacroCallsTopic::TOPIC_STR));
assert!(topics.contains(&MacroChannelsTopic::TOPIC_STR));
+ assert!(topics.contains(&MacroChatsTopic::TOPIC_STR));
assert!(topics.contains(&MacroProjectsTopic::TOPIC_STR));
}
@@ -522,6 +658,18 @@ fn maps_all_call_lifecycle_events_to_index_actions() {
);
}
+#[test]
+fn maps_all_chat_lifecycle_events_to_index_actions() {
+ let cases = chat_event_cases();
+ assert_eq!(cases.len(), 8);
+
+ for (event, expected) in cases {
+ let serialized = serde_json::to_value(&event).expect("serializable chat event");
+ assert_eq!(serialized["event_type"], expected.event_type);
+ assert_eq!(describe_chat_event(&event), expected);
+ }
+}
+
#[test]
fn maps_all_channel_lifecycle_events_to_index_actions() {
let cases = channel_event_cases();
@@ -560,6 +708,30 @@ fn project_id_collection_is_stable_and_drops_missing_or_empty_ids() {
);
}
+#[test]
+fn chat_envelope_decodes_round_trip() {
+ let event = ChatTopicEvent::MessageSent(ChatMessageSentMetadata {
+ chat_id: CHAT_ID.to_string(),
+ message_id: CHAT_MESSAGE_ID.to_string(),
+ role: ChatMessageRole::Assistant,
+ model: "chat-model".to_string(),
+ actor_user_id: None,
+ attachment_count: 0,
+ });
+ let message = encoded_message(
+ MacroChatsTopic::TOPIC_STR,
+ CHAT_ID,
+ Event::new(event.clone()),
+ );
+
+ let decoded = DeclaredMacroEvent::decode(&message).expect("decodable chat event");
+ let DeclaredMacroEvent::ChatMacroEvent(decoded_event) = decoded else {
+ panic!("expected chat event");
+ };
+ assert_eq!(decoded_event.key(), CHAT_ID);
+ assert_eq!(decoded_event.event().event, event);
+}
+
#[test]
fn channel_envelope_decodes_round_trip() {
let event = ChannelTopicEvent::Deleted(ChannelDeletedMetadata {
@@ -603,6 +775,50 @@ fn project_envelope_decodes_round_trip_with_string_key() {
assert_eq!(decoded_event.event().event, event);
}
+#[tokio::test]
+async fn malformed_and_unsupported_chat_messages_are_commit_safe() {
+ let malformed = TestMessage {
+ topic: MacroChatsTopic::TOPIC_STR,
+ key: Some(CHAT_ID.to_string()),
+ payload: Some(b"not json".to_vec()),
+ };
+ let malformed = attach_event_coordinates(DeclaredMacroEvent::decode(&malformed), 5, 40);
+ let (sender, mut receiver) = mpsc::channel(1);
+
+ assert!(matches!(
+ handoff_decoded(&sender, malformed).await,
+ HandoffOutcome::MalformedRecord(EventBrokerError::Serialization(_))
+ ));
+
+ let unsupported_event = ChatTopicEvent::MessageDeleted(ChatMessageDeletedMetadata {
+ chat_id: CHAT_ID.to_string(),
+ message_id: CHAT_MESSAGE_ID.to_string(),
+ });
+ let unsupported = encoded_message(
+ MacroChatsTopic::TOPIC_STR,
+ CHAT_ID,
+ Event::with_schema_version(unsupported_event, 2),
+ );
+ let unsupported = attach_event_coordinates(DeclaredMacroEvent::decode(&unsupported), 5, 41);
+ match handoff_decoded(&sender, unsupported).await {
+ HandoffOutcome::MalformedRecord(EventBrokerError::UnsupportedSchemaVersion {
+ topic,
+ expected,
+ actual,
+ }) => {
+ assert_eq!(topic, MacroChatsTopic::TOPIC_STR);
+ assert_eq!(expected, 1);
+ assert_eq!(actual, 2);
+ }
+ outcome => panic!("expected malformed chat record, got {outcome:?}"),
+ }
+
+ assert!(matches!(
+ receiver.try_recv(),
+ Err(mpsc::error::TryRecvError::Empty)
+ ));
+}
+
#[tokio::test]
async fn unsupported_project_schema_message_is_commit_safe() {
let event = ProjectTopicEvent::Restored(ProjectRestoredMetadata {
diff --git a/services/search_processing_service/src/process/chat.rs b/services/search_processing_service/src/process/chat.rs
index fb33b30bd0..8750876b39 100644
--- a/services/search_processing_service/src/process/chat.rs
+++ b/services/search_processing_service/src/process/chat.rs
@@ -4,38 +4,48 @@ use opensearch_client::{
OpensearchClient, date_format::EpochMillis, upsert::chat_message::UpsertChatMessageArgs,
};
use properties::outbound::entity_properties_get_query::get_entity_properties_for_index;
-use sqs_client::search::chat::{ChatMessage, RemoveChatMessage};
+use sqs_client::search::chat::ChatMessage;
use crate::process::properties::to_indexed_properties;
-/// Handles the processing of chat messages
-#[tracing::instrument(skip(opensearch_client, db))]
+/// Handles the SQS backfill request for a chat message.
+#[tracing::instrument(skip(opensearch_client, db), err)]
pub async fn insert_chat_message(
opensearch_client: &OpensearchClient,
db: &sqlx::Pool<sqlx::Postgres>,
chat_message: &ChatMessage,
) -> anyhow::Result<()> {
- let info = macro_db_client::chat::get::get_chat_message_info(
+ upsert_chat_message_by_ids(
+ opensearch_client,
db,
chat_message.chat_id.as_str(),
chat_message.message_id.as_str(),
+ chat_message.index_override.as_deref(),
)
.await
- .context("failed to get chat message info")?;
+}
+
+/// Upserts a chat message using the authoritative database values.
+#[tracing::instrument(skip(opensearch_client, db), err)]
+pub(crate) async fn upsert_chat_message_by_ids(
+ opensearch_client: &OpensearchClient,
+ db: &sqlx::Pool<sqlx::Postgres>,
+ chat_id: &str,
+ message_id: &str,
+ index_override: Option<&str>,
+) -> anyhow::Result<()> {
+ let info = macro_db_client::chat::get::get_chat_message_info(db, chat_id, message_id)
+ .await
+ .context("failed to get chat message info")?;
let Some(info) = info else {
return Ok(());
};
- let index_override = chat_message.index_override.as_deref();
if info.deleted_at.is_some() {
tracing::trace!("chat is deleted, removing message from search index");
opensearch_client
- .delete_chat_message(
- chat_message.chat_id.as_str(),
- chat_message.message_id.as_str(),
- index_override,
- )
+ .delete_chat_message(chat_id, message_id, index_override)
.await
.context("failed to delete chat message from search")?;
return Ok(());
@@ -45,7 +55,7 @@ pub async fn insert_chat_message(
// write or values set by the property-update path get wiped. A fetch
// failure propagates (retry) rather than being mistaken for "empty".
let properties = to_indexed_properties(
- get_entity_properties_for_index(db, chat_message.chat_id.as_str(), EntityType::Chat)
+ get_entity_properties_for_index(db, chat_id, EntityType::Chat)
.await
.context("failed to fetch chat properties for search index")?,
);
@@ -53,11 +63,11 @@ pub async fn insert_chat_message(
opensearch_client
.upsert_chat_message(
&UpsertChatMessageArgs {
- chat_id: chat_message.chat_id.clone(),
- chat_message_id: chat_message.message_id.clone(),
- user_id: chat_message.user_id.clone(),
- created_at_millis: EpochMillis::new(chat_message.created_at.timestamp_millis())?,
- updated_at_millis: EpochMillis::new(chat_message.updated_at.timestamp_millis())?,
+ chat_id: chat_id.to_string(),
+ chat_message_id: message_id.to_string(),
+ user_id: info.owner_user_id,
+ created_at_millis: EpochMillis::new(info.created_at.timestamp_millis())?,
+ updated_at_millis: EpochMillis::new(info.updated_at.timestamp_millis())?,
title: info.name,
content: info.content,
role: info.role,
@@ -71,22 +81,23 @@ pub async fn insert_chat_message(
Ok(())
}
-/// Handles the removal of chat message(s) from the opensearch index
-#[tracing::instrument(skip(opensearch_client))]
-pub async fn remove_chat_message(
+/// Removes a chat message or all messages for a chat from the search index.
+#[tracing::instrument(skip(opensearch_client), err)]
+pub(crate) async fn remove_chat_message(
opensearch_client: &OpensearchClient,
- remove_message: &RemoveChatMessage,
+ chat_id: &str,
+ message_id: Option<&str>,
+ index_override: Option<&str>,
) -> anyhow::Result<()> {
- let index_override = remove_message.index_override.as_deref();
- if let Some(message_id) = remove_message.message_id.as_ref() {
+ if let Some(message_id) = message_id {
tracing::trace!("deleting chat message");
opensearch_client
- .delete_chat_message(remove_message.chat_id.as_str(), message_id, index_override)
+ .delete_chat_message(chat_id, message_id, index_override)
.await?;
} else {
tracing::trace!("deleting chat");
opensearch_client
- .delete_chat(remove_message.chat_id.as_str(), index_override)
+ .delete_chat(chat_id, index_override)
.await?;
}
diff --git a/services/search_processing_service/src/process/mod.rs b/services/search_processing_service/src/process/mod.rs
index 7a21f79c7e..b7c1b81daa 100644
--- a/services/search_processing_service/src/process/mod.rs
+++ b/services/search_processing_service/src/process/mod.rs
@@ -1,6 +1,6 @@
pub(crate) mod call;
pub(crate) mod channel;
-mod chat;
+pub(crate) mod chat;
pub mod context;
mod document;
mod email;
@@ -115,9 +115,6 @@ pub async fn process_message(
SearchQueueMessage::ChatMessage(message) => {
chat::insert_chat_message(&ctx.opensearch_client, &ctx.db, &message).await?;
}
- SearchQueueMessage::RemoveChatMessage(message) => {
- chat::remove_chat_message(&ctx.opensearch_client, &message).await?;
- }
SearchQueueMessage::CallRecord(message) => {
let call_id = message
.call_id