Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(search_processing_service): projects kafka (#5295)
Will Hutchinsonabf9de570bb1
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.jsonindex 4a5b2a25ae..09e2587f83 100644--- a/.github/workspace-dep-closures.json+++ b/.github/workspace-dep-closures.json@@ -1307,6 +1307,7 @@"crates/cowlike","crates/crm","crates/document_sub_type",+ "crates/dynamodb_client","crates/email","crates/email_db_client","crates/email_utils",@@ -1331,6 +1332,7 @@"crates/macro_event_topics","crates/macro_queues","crates/macro_service_urls",+ "crates/macro_sha_count_client","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -1352,6 +1354,7 @@"crates/models_team","crates/non_empty","crates/notification",+ "crates/projects","crates/prompt","crates/properties","crates/rate_limit",@@ -4929,6 +4932,7 @@"crates/cowlike","crates/crm","crates/document_sub_type",+ "crates/dynamodb_client","crates/email","crates/email_db_client","crates/email_utils",@@ -4960,6 +4964,7 @@"crates/macro_middleware","crates/macro_queues","crates/macro_service_urls",+ "crates/macro_sha_count_client","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -4986,6 +4991,7 @@"crates/notification","crates/opensearch_client","crates/opensearch_query_builder",+ "crates/projects","crates/prompt","crates/properties","crates/rate_limit",diff --git a/.sqlx/query-306950934a036e99471a1646c6f3b97a9ca9da460ba754db95f8772ab18e9f54.json b/.sqlx/query-306950934a036e99471a1646c6f3b97a9ca9da460ba754db95f8772ab18e9f54.jsonnew file mode 100644index 0000000000..8ddafbf80f--- /dev/null+++ b/.sqlx/query-306950934a036e99471a1646c6f3b97a9ca9da460ba754db95f8772ab18e9f54.json@@ -0,0 +1,28 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT p.id as project_id, p.\"userId\" as user_id\n FROM \"Project\" p\n WHERE p.\"deletedAt\" IS NOT NULL AND p.\"deletedAt\" <= $1\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "project_id",+ "type_info": "Text"+ },+ {+ "ordinal": 1,+ "name": "user_id",+ "type_info": "Text"+ }+ ],+ "parameters": {+ "Left": [+ "Timestamp"+ ]+ },+ "nullable": [+ false,+ false+ ]+ },+ "hash": "306950934a036e99471a1646c6f3b97a9ca9da460ba754db95f8772ab18e9f54"+}diff --git a/.sqlx/query-9f595d4ae9aa9111c316efa5d4be2b8e4a6cc69374b90600457a32b14a91ae61.json b/.sqlx/query-9f595d4ae9aa9111c316efa5d4be2b8e4a6cc69374b90600457a32b14a91ae61.jsondeleted file mode 100644index 2620ca0739..0000000000--- a/.sqlx/query-9f595d4ae9aa9111c316efa5d4be2b8e4a6cc69374b90600457a32b14a91ae61.json+++ /dev/null@@ -1,22 +0,0 @@-{- "db_name": "PostgreSQL",- "query": "\n SELECT p.id\n FROM \"Project\" p\n WHERE p.\"deletedAt\" IS NOT NULL AND p.\"deletedAt\" <= $1\n ",- "describe": {- "columns": [- {- "ordinal": 0,- "name": "id",- "type_info": "Text"- }- ],- "parameters": {- "Left": [- "Timestamp"- ]- },- "nullable": [- false- ]- },- "hash": "9f595d4ae9aa9111c316efa5d4be2b8e4a6cc69374b90600457a32b14a91ae61"-}diff --git a/Cargo.lock b/Cargo.lockindex 267d784166..5fb66cbeda 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -4003,9 +4003,12 @@ dependencies = ["macro_entrypoint","macro_env","macro_env_var",+ "macro_event_broker","macro_queues",+ "macro_user_id","macro_uuid","model",+ "projects","serde","serde_json","sqlx",@@ -12920,6 +12923,7 @@ dependencies = ["nom 8.0.0","opensearch_client","pdfium-render",+ "projects","properties","rdkafka","rootcause",diff --git a/crates/macro_db_client/src/projects/get_projects.rs b/crates/macro_db_client/src/projects/get_projects.rsindex 7a4dddce09..a5a177e128 100644--- a/crates/macro_db_client/src/projects/get_projects.rs+++ b/crates/macro_db_client/src/projects/get_projects.rs@@ -1,6 +1,9 @@use chrono::{DateTime, Utc};use sqlx::{Pool, Postgres};+#[cfg(test)]+mod test;+/// One row of the search-backfill scan: just enough to build an upsert/// message and the keyset cursor.pub struct ProjectSearchBackfillRow {@@ -73,21 +76,30 @@ pub async fn get_sub_project_ids(Ok(result)}-/// Gets all projects that are older than the provided date+/// A deleted project and the user who owns it.+#[derive(Debug, Eq, PartialEq)]+pub struct ProjectToDelete {+ /// The deleted project's ID.+ pub project_id: String,+ /// The owning user's ID.+ pub user_id: String,+}++/// Gets all projects and their owners that are older than the provided date.#[tracing::instrument(skip(db))]pub async fn get_projects_to_delete(db: &Pool<Postgres>,date: &chrono::NaiveDateTime,-) -> anyhow::Result<Vec<String>> {- let result = sqlx::query!(+) -> anyhow::Result<Vec<ProjectToDelete>> {+ let result = sqlx::query_as!(+ ProjectToDelete,r#"- SELECT p.id+ SELECT p.id as project_id, p."userId" as user_idFROM "Project" pWHERE p."deletedAt" IS NOT NULL AND p."deletedAt" <= $1"#,date)- .map(|row| row.id).fetch_all(db).await?;diff --git a/crates/macro_db_client/src/projects/get_projects/test.rs b/crates/macro_db_client/src/projects/get_projects/test.rsnew file mode 100644index 0000000000..9ad8711019--- /dev/null+++ b/crates/macro_db_client/src/projects/get_projects/test.rs@@ -0,0 +1,29 @@+use super::*;+use chrono::NaiveDateTime;++#[sqlx::test(fixtures(path = "../../../fixtures", scripts("recently_deleted")))]+async fn returns_old_deleted_projects_with_owners(pool: Pool<Postgres>) -> anyhow::Result<()> {+ let cutoff = NaiveDateTime::parse_from_str("2020-01-01 00:00:00", "%Y-%m-%d %H:%M:%S")?;+ let mut projects = get_projects_to_delete(&pool, &cutoff).await?;+ projects.sort_unstable_by(|left, right| left.project_id.cmp(&right.project_id));++ assert_eq!(+ projects,+ vec![+ ProjectToDelete {+ project_id: "p1".to_owned(),+ user_id: "macro|[email protected]".to_owned(),+ },+ ProjectToDelete {+ project_id: "p2".to_owned(),+ user_id: "macro|[email protected]".to_owned(),+ },+ ProjectToDelete {+ project_id: "p3".to_owned(),+ user_id: "macro|[email protected]".to_owned(),+ },+ ]+ );++ Ok(())+}diff --git a/crates/macro_db_client/src/projects/mod.rs b/crates/macro_db_client/src/projects/mod.rsindex dec5da97e6..68c0073d60 100644--- a/crates/macro_db_client/src/projects/mod.rs+++ b/crates/macro_db_client/src/projects/mod.rs@@ -2,7 +2,7 @@ pub mod get_project;mod get_projects;pub use get_projects::{- ProjectSearchBackfillRow, get_all_project_ids_with_users_paginated,+ ProjectSearchBackfillRow, ProjectToDelete, get_all_project_ids_with_users_paginated,get_projects_for_search_backfill, get_projects_to_delete, get_sub_project_ids,};pub mod delete;diff --git a/crates/macro_project_utils/Cargo.toml b/crates/macro_project_utils/Cargo.tomldeleted file mode 100644index 971ad5567d..0000000000--- a/crates/macro_project_utils/Cargo.toml+++ /dev/null@@ -1,15 +0,0 @@-[package]-edition = "2024"-name = "macro_project_utils"-publish = false-version = "0.1.0"--[dependencies]-anyhow = { workspace = true }-sqlx = { workspace = true }-sqs_client = { path = "../sqs_client", default-features = false, features = [- "search",-] }-tokio = { workspace = true }-tracing = { workspace = true }-workspace-hack = { version = "0.1", path = "../workspace-hack" }diff --git a/crates/macro_project_utils/justfile b/crates/macro_project_utils/justfiledeleted file mode 100644index 1849fc42e3..0000000000--- a/crates/macro_project_utils/justfile+++ /dev/null@@ -1,6 +0,0 @@-import '../../tooling/just/database.just'--mod sqlx '../../tooling/just/sqlx.just'--prepare_db:- just sqlx::prepare_db {{ DATABASE_URL }}diff --git a/crates/macro_project_utils/src/lib.rs b/crates/macro_project_utils/src/lib.rsdeleted file mode 100644index ddfa74c46a..0000000000--- a/crates/macro_project_utils/src/lib.rs+++ /dev/null@@ -1,91 +0,0 @@-use sqs_client::{- SQS,- search::{SearchQueueMessage, project::UpsertProject},-};--#[derive(Clone, Debug)]-pub struct ProjectModifiedArgs<T>-where- T: Clone + std::fmt::Debug + ToString + std::marker::Send + Sync,-{- /// The new project id of the item or the project that was modified- pub project_id: Option<T>,- /// The old project id if the item was moved- pub old_project_id: Option<T>,- /// The user who performed the action- pub user_id: String,-}--#[tracing::instrument(skip(db, sqs))]-pub async fn update_project_modified<T>(- db: &sqlx::Pool<sqlx::Postgres>,- sqs: Option<&SQS>,- project_modified_args: ProjectModifiedArgs<T>,-) where- T: Clone + std::fmt::Debug + ToString + std::marker::Send + Sync,-{- tracing::trace!("updating project modified date");-- let project_id = project_modified_args- .project_id- .as_ref()- .map(|s| s.to_string());- let old_project_id = project_modified_args- .old_project_id- .as_ref()- .map(|s| s.to_string());-- tokio::spawn({- let db = db.clone();- let sqs = sqs.cloned();- async move {- if let Some(old_project_id) = old_project_id- && !old_project_id.is_empty()- {- tracing::trace!(project_id=?old_project_id, "updating project modified date");- let _ = update_project_modified_date(&db, sqs.as_ref(), &old_project_id).await.inspect_err(|e| {- tracing::error!(error=?e, project_id=?old_project_id, "unable to update project modified date");- });- }-- if let Some(project_id) = project_id- && !project_id.is_empty()- {- tracing::trace!(project_id=?project_id, "updating project modified date");-- let _ = update_project_modified_date(&db, sqs.as_ref(), &project_id).await.inspect_err(|e| {- tracing::error!(error=?e, project_id=?project_id, "unable to update project modified date");- });- }- }- });-}--async fn update_project_modified_date(- db: &sqlx::Pool<sqlx::Postgres>,- sqs: Option<&SQS>,- project_id: &str,-) -> anyhow::Result<()> {- sqlx::query!(- r#"UPDATE "Project" SET "updatedAt" = NOW() WHERE id = $1"#,- project_id,- )- .execute(db)- .await?;-- // Refresh the project's search doc so ranking follows the bumped- // updatedAt. Best-effort like the bump itself.- if let Some(sqs) = sqs {- let _ = sqs- .send_message_to_search_event_queue(SearchQueueMessage::UpsertProject(UpsertProject {- project_id: project_id.to_string(),- index_override: None,- }))- .await- .inspect_err(|e| {- tracing::error!(error=?e, project_id=?project_id, "unable to enqueue project search upsert");- });- }-- Ok(())-}diff --git a/crates/projects/src/domain.rs b/crates/projects/src/domain.rsindex 4ed26cf913..9c75d16302 100644--- a/crates/projects/src/domain.rs+++ b/crates/projects/src/domain.rs@@ -1,6 +1,7 @@//! Domain models, ports, and project service definitions./// Unified entity-mutation capability impls.+#[cfg(feature = "service")]pub mod entity_mutation;pub mod events;pub mod models;diff --git a/crates/projects/src/domain/ports.rs b/crates/projects/src/domain/ports.rsindex 5677ec672c..5ec4e2e3cb 100644--- a/crates/projects/src/domain/ports.rs+++ b/crates/projects/src/domain/ports.rs@@ -206,20 +206,9 @@ pub trait ShaCounterPort: Send + Sync + 'static {) -> impl Future<Output = anyhow::Result<()>> + Send;}-/// Port for publishing project search and document-deletion work.+/// Port for publishing cross-family search and document-deletion work.+// TODO: Remove this port and its SQS adapter after the chats and documents migrations.pub trait ProjectSearchIndexer: Send + Sync + 'static {- /// Enqueue projects for indexing.- fn upsert_projects(- &self,- project_ids: Vec<String>,- ) -> impl Future<Output = anyhow::Result<()>> + Send;-- /// Remove projects from the search index.- fn remove_projects(- &self,- project_ids: Vec<String>,- ) -> impl Future<Output = anyhow::Result<()>> + Send;-/// Remove chats from the search index.fn remove_chats(&self,diff --git a/crates/projects/src/domain/service.rs b/crates/projects/src/domain/service.rsindex 4d82776eb6..e63707d81f 100644--- a/crates/projects/src/domain/service.rs+++ b/crates/projects/src/domain/service.rs@@ -10,7 +10,7 @@ use entity_access_management::domain::ports::EntityAccessManagementService;use futures::stream::{FuturesUnordered, StreamExt};use macro_event_broker::MacroEventBroker;use macro_user_id::user_id::MacroUserIdStr;-use model::folder::{UploadFolderRequest, UploadFolderResponseData};+use model::folder::{FileSystemNodeWithIds, UploadFolderRequest, UploadFolderResponseData};use model::item::{Item, ItemWithUserAccessLevel};use model::project::request::{CreateProjectRequest, PatchProjectRequestV2};use model::project::response::GetProjectResponseData;@@ -141,17 +141,6 @@ where"unable to update project modified date");});- let _ = self- .search_indexer- .upsert_projects(vec![project_id.to_string()])- .await- .inspect_err(|error| {- tracing::error!(- error = ?error,- project_id,- "unable to enqueue project search upsert"- );- });}}@@ -341,17 +330,6 @@ where.project_parent_id.map(|_| parse_internal_uuid(&project.id, "created project ID")).transpose()?;- let _ = self- .search_indexer- .upsert_projects(vec![project.id.clone()])- .await- .inspect_err(|error| {- tracing::error!(- error = ?error,- project_id = %project.id,- "unable to enqueue project search upsert"- );- });if let Some((parent_id, entity_id)) = args.project_parent_id.zip(entity_id) {let _ = self@@ -508,15 +486,6 @@ where.await.map_err(|error| internal_error(error, "unable to delete project"))?;- if !deleted.project_ids.is_empty() {- let _ = self- .search_indexer- .remove_projects(deleted.project_ids.clone())- .await- .inspect_err(|error| {- tracing::error!(error = ?error, "unable to enqueue deleted projects for search");- });- }if let Some(parent_id) = project.parent_id.as_deref() {self.bump_project_modified(parent_id).await;}@@ -565,13 +534,6 @@ where.map(|(document_id, _)| document_id.clone()).collect::<Vec<_>>();- if !purged.project_ids.is_empty() {- let _ = self- .search_indexer- .remove_projects(purged.project_ids)- .await- .inspect_err(|error| tracing::error!(error = ?error, "unable to enqueue purged projects for search"));- }if !purged.chat_ids.is_empty() {let _ = self.search_indexer@@ -624,16 +586,6 @@ where.map_err(|error| internal_error(error, "unable to revert project"))?;let result = restored.clone();- if !restored.project_ids.is_empty() {- let _ = self- .search_indexer- .upsert_projects(restored.project_ids.clone())- .await- .inspect_err(|error| {- tracing::error!(error = ?error, "unable to enqueue restored projects for search");- });- }-let project_id = receipt.entity().entity_id.clone();self.publish_project_event(&ProjectMacroEvent::restored(project_id.clone(),@@ -655,6 +607,9 @@ whereinternal: bool,args: UploadFolderRequest,) -> Result<UploadFolderResponseData, ProjectError> {+ let event_owner = actor.clone();+ let event_name = args.root_folder_name.clone();+ let event_parent_project_id = args.parent_id.clone();let root_folder = build_root_folder(&args.root_folder_name, args.content).map_err(|error| internal_error(error, "unable to prepare folder upload"))?;let uploaded = self@@ -699,13 +654,26 @@ where};if !uploaded.project_ids.is_empty() {- let _ = self- .search_indexer- .upsert_projects(uploaded.project_ids)- .await- .inspect_err(|error| {- tracing::error!(error = ?error, "unable to enqueue uploaded projects for search");- });+ match &uploaded.file_system {+ FileSystemNodeWithIds::Folder {+ project_id: root_project_id,+ ..+ } => {+ self.publish_project_event(&ProjectMacroEvent::uploaded(+ root_project_id.clone(),+ ProjectUploadedMetadata {+ root_project_id: root_project_id.clone(),+ owner: event_owner,+ name: event_name,+ parent_project_id: event_parent_project_id,+ project_ids: uploaded.project_ids,+ },+ ));+ }+ FileSystemNodeWithIds::File { .. } => {+ tracing::error!("unable to publish folder upload event: root node is a file");+ }+ }}Ok(UploadFolderResponseData {diff --git a/crates/projects/src/domain/service/tests.rs b/crates/projects/src/domain/service/tests.rsindex 6f4f3d04be..b986ddb57c 100644--- a/crates/projects/src/domain/service/tests.rs+++ b/crates/projects/src/domain/service/tests.rs@@ -153,14 +153,6 @@ impl EntityAccessManagementService for NullPort {}impl ProjectSearchIndexer for NullPort {- async fn upsert_projects(&self, _project_ids: Vec<String>) -> anyhow::Result<()> {- unreachable!()- }-- async fn remove_projects(&self, _project_ids: Vec<String>) -> anyhow::Result<()> {- unreachable!()- }-async fn remove_chats(&self, _chat_ids: Vec<String>) -> anyhow::Result<()> {unreachable!()}@@ -226,34 +218,12 @@ impl EntityAccessManagementService for RecordingEam {}}-#[derive(Clone, Debug, PartialEq, Eq)]-enum IndexCall {- Upsert(Vec<String>),- Remove(Vec<String>),-}--#[derive(Clone, Default)]+#[derive(Clone, Copy, Default)]struct RecordingIndexer {- calls: Arc<Mutex<Vec<IndexCall>>>,+ _unused: (),}impl ProjectSearchIndexer for RecordingIndexer {- async fn upsert_projects(&self, project_ids: Vec<String>) -> anyhow::Result<()> {- self.calls- .lock()- .unwrap()- .push(IndexCall::Upsert(project_ids));- Ok(())- }-- async fn remove_projects(&self, project_ids: Vec<String>) -> anyhow::Result<()> {- self.calls- .lock()- .unwrap()- .push(IndexCall::Remove(project_ids));- Ok(())- }-async fn remove_chats(&self, _chat_ids: Vec<String>) -> anyhow::Result<()> {unreachable!()}@@ -705,9 +675,7 @@ async fn create_uses_grapheme_limit_and_orchestrates_parent_side_effects() {.return_once(|_| Box::pin(async { Ok(()) }));let eam = RecordingEam::default();let eam_calls = eam.calls.clone();- let indexer = RecordingIndexer::default();- let index_calls = indexer.calls.clone();- let service = mutation_service(repo, eam, indexer);+ let service = mutation_service(repo, eam, RecordingIndexer::default());service.create_project(@@ -735,13 +703,6 @@ async fn create_uses_grapheme_limit_and_orchestrates_parent_side_effects() {*eam_calls.lock().unwrap(),vec![EamCall::Add(project_id, parent_id)]);- assert_eq!(- *index_calls.lock().unwrap(),- vec![- IndexCall::Upsert(vec![project_id.to_string()]),- IndexCall::Upsert(vec![parent_id.to_string()]),- ]- );}#[tokio::test]@@ -1168,7 +1129,7 @@ async fn edit_project_succeeds_when_event_publication_fails() {}#[tokio::test]-async fn owner_edit_propagates_move_and_pairs_every_bump_with_an_upsert() {+async fn owner_edit_propagates_move_and_updates_all_modified_timestamps() {let project_id = Uuid::new_v4();let old_parent_id = Uuid::new_v4();let new_parent_id = Uuid::new_v4();@@ -1200,9 +1161,7 @@ async fn owner_edit_propagates_move_and_pairs_every_bump_with_an_upsert() {});let eam = RecordingEam::default();let eam_calls = eam.calls.clone();- let indexer = RecordingIndexer::default();- let index_calls = indexer.calls.clone();- let service = mutation_service(repo, eam, indexer);+ let service = mutation_service(repo, eam, RecordingIndexer::default());service.edit_project(@@ -1221,17 +1180,14 @@ async fn owner_edit_propagates_move_and_pairs_every_bump_with_an_upsert() {Some(new_parent_id))]);- let bumps = bumped.lock().unwrap().clone();- let indexed = index_calls- .lock()- .unwrap()- .iter()- .filter_map(|call| match call {- IndexCall::Upsert(ids) if ids.len() == 1 => Some(ids[0].clone()),- _ => None,- })- .collect::<Vec<_>>();- assert_eq!(bumps, indexed);+ assert_eq!(+ *bumped.lock().unwrap(),+ vec![+ project_id.to_string(),+ old_parent_id.to_string(),+ new_parent_id.to_string(),+ ]+ );}#[tokio::test]@@ -1314,25 +1270,19 @@ async fn edit_name_limit_counts_unicode_graphemes() {}#[tokio::test]-async fn failed_database_bump_still_runs_the_paired_index_upsert() {+async fn failed_project_modified_update_is_non_fatal() {let project_id = Uuid::new_v4();let mut repo = MockProjectRepo::new();repo.expect_update_project_modified()+ .withf(move |id| id == project_id.to_string()).return_once(|_| Box::pin(async { Err(anyhow::anyhow!("database unavailable")) }));- let indexer = RecordingIndexer::default();- let calls = indexer.calls.clone();- let service = mutation_service(repo, RecordingEam::default(), indexer);+ let service = mutation_service(repo, RecordingEam::default(), RecordingIndexer::default());service.bump_project_modified(&project_id.to_string()).await;-- assert_eq!(- *calls.lock().unwrap(),- vec![IndexCall::Upsert(vec![project_id.to_string()])]- );}#[tokio::test]-async fn soft_delete_removes_index_entries_and_bumps_parent_with_upsert() {+async fn soft_delete_updates_parent_timestamp_and_publishes_event() {let project_id = Uuid::new_v4();let parent_id = Uuid::new_v4();let deleted_ids = vec![project_id.to_string(), Uuid::new_v4().to_string()];@@ -1350,12 +1300,14 @@ async fn soft_delete_removes_index_entries_and_bumps_parent_with_upsert() {repo.expect_update_project_modified().withf(move |id| id == parent_id.to_string()).return_once(|_| Box::pin(async { Ok(()) }));- let indexer = RecordingIndexer::default();- let calls = indexer.calls.clone();let event_broker = TestEventBroker::default();let published = event_broker.published();- let service =- mutation_service_with_event_broker(repo, RecordingEam::default(), indexer, event_broker);+ let service = mutation_service_with_event_broker(+ repo,+ RecordingEam::default(),+ RecordingIndexer::default(),+ event_broker,+ );service.soft_delete_project(@@ -1366,13 +1318,6 @@ async fn soft_delete_removes_index_entries_and_bumps_parent_with_upsert() {.await.unwrap();- assert_eq!(- *calls.lock().unwrap(),- vec![- IndexCall::Remove(expected_deleted_ids.clone()),- IndexCall::Upsert(vec![parent_id.to_string()]),- ]- );let published = published.lock().unwrap();assert_eq!(published.len(), 1);assert_project_event(&published[0], project_id, "project.deleted");@@ -1393,7 +1338,7 @@ async fn soft_delete_removes_index_entries_and_bumps_parent_with_upsert() {}#[tokio::test]-async fn revert_upserts_every_restored_project() {+async fn revert_publishes_every_restored_project_in_event() {let project_id = Uuid::new_v4();let parent_id = Uuid::new_v4();let restored_ids = vec![project_id.to_string(), Uuid::new_v4().to_string()];@@ -1413,12 +1358,14 @@ async fn revert_upserts_every_restored_project() {})})});- let indexer = RecordingIndexer::default();- let calls = indexer.calls.clone();let event_broker = TestEventBroker::default();let published = event_broker.published();- let service =- mutation_service_with_event_broker(repo, RecordingEam::default(), indexer, event_broker);+ let service = mutation_service_with_event_broker(+ repo,+ RecordingEam::default(),+ RecordingIndexer::default(),+ event_broker,+ );service.revert_delete_project(@@ -1432,10 +1379,6 @@ async fn revert_upserts_every_restored_project() {.await.unwrap();- assert_eq!(- *calls.lock().unwrap(),- vec![IndexCall::Upsert(expected_ids.clone())]- );let published = published.lock().unwrap();assert_eq!(published.len(), 1);assert_project_event(&published[0], project_id, "project.restored");@@ -1477,9 +1420,7 @@ async fn restore_entity_returns_document_and_chat_effects() {})})});- let indexer = RecordingIndexer::default();- let index_calls = indexer.calls.clone();- let service = mutation_service(repo, RecordingEam::default(), indexer);+ let service = mutation_service(repo, RecordingEam::default(), RecordingIndexer::default());let entity = EntityType::Project.with_entity_string(project_id.to_string());let effects = service@@ -1508,10 +1449,6 @@ async fn restore_entity_returns_document_and_chat_effects() {),]);- assert_eq!(- *index_calls.lock().unwrap(),- vec![IndexCall::Upsert(restored_project_ids)]- );}#[tokio::test]@@ -1637,14 +1574,6 @@ impl OrderedIndexer {}impl ProjectSearchIndexer for OrderedIndexer {- async fn upsert_projects(&self, _project_ids: Vec<String>) -> anyhow::Result<()> {- self.record("upsert")- }-- async fn remove_projects(&self, _project_ids: Vec<String>) -> anyhow::Result<()> {- self.record("projects")- }-async fn remove_chats(&self, _chat_ids: Vec<String>) -> anyhow::Result<()> {self.record("chats")}@@ -1811,14 +1740,7 @@ async fn permanent_delete_runs_external_work_after_committed_purge() {assert_eq!(*events.lock().unwrap(),- vec![- "repo",- "sha",- "projects",- "chats",- "documents",- "document_deletes"- ]+ vec!["repo", "sha", "chats", "documents", "document_deletes"]);let published = published.lock().unwrap();assert_eq!(published.len(), 1);@@ -2070,6 +1992,123 @@ async fn mark_projects_uploaded_publication_failure_is_non_fatal() {assert_eq!(result, vec!["root-project", "child-project"]);}+#[tokio::test]+async fn upload_folder_publishes_uploaded_event() {+ let project_ids = vec!["root-project".to_string(), "child-project".to_string()];+ let expected_project_ids = project_ids.clone();+ let mut repo = MockProjectRepo::new();+ repo.expect_upload_folder()+ .withf(|args| {+ args.user_id.as_ref() == "macro|[email protected]"+ && args.root_folder_name == "Uploaded tree"+ && args.upload_request_id == "request"+ && args.parent_id.as_deref() == Some("parent-project")+ })+ .return_once(move |_| {+ Box::pin(async move {+ Ok(UploadFolderWithIdsResponse {+ file_system: FileSystemNodeWithIds::Folder {+ content: Default::default(),+ project_id: "root-project".to_string(),+ },+ project_ids,+ documents: vec![upload_document("document", FileType::Pdf)],+ })+ })+ });+ let upload_urls = RecordingUploadUrls {+ calls: Arc::new(Mutex::new(Vec::new())),+ fail_document: false,+ };+ let upload_url_calls = upload_urls.calls.clone();+ let event_broker = TestEventBroker::default();+ let published = event_broker.published();+ let service = ProjectServiceImpl::new(+ repo,+ upload_urls,+ RecordingBulkUpload::default(),+ NullPort,+ NullPort,+ RecordingIndexer::default(),+ None,+ event_broker,+ );++ service+ .upload_folder(+ user_id("macro|[email protected]"),+ false,+ UploadFolderRequest {+ content: vec![FolderItem {+ name: "document".to_string(),+ full_name: "document.pdf".to_string(),+ file_type: Some(FileType::Pdf),+ relative_path: "Uploaded tree".to_string(),+ sha: "0123456789abcdef".to_string(),+ }],+ root_folder_name: "Uploaded tree".to_string(),+ upload_request_id: "request".to_string(),+ parent_id: Some("parent-project".to_string()),+ },+ )+ .await+ .unwrap();++ assert_eq!(upload_url_calls.lock().unwrap().len(), 1);+ let published = published.lock().unwrap();+ assert_eq!(published.len(), 1);+ assert_eq!(published[0].topic, "macro.projects");+ assert_eq!(published[0].key, "root-project");+ assert_eq!(published[0].payload["schema_version"], 1);+ assert_eq!(published[0].payload["event_type"], "project.uploaded");+ assert_eq!(+ published[0].payload["metadata"],+ serde_json::json!({+ "root_project_id": "root-project",+ "owner": "macro|[email protected]",+ "name": "Uploaded tree",+ "parent_project_id": "parent-project",+ "project_ids": expected_project_ids,+ })+ );+}++#[tokio::test]+async fn upload_folder_with_no_project_ids_publishes_no_event() {+ let mut repo = MockProjectRepo::new();+ repo.expect_upload_folder().return_once(|_| {+ Box::pin(async {+ Ok(UploadFolderWithIdsResponse {+ file_system: FileSystemNodeWithIds::Folder {+ content: Default::default(),+ project_id: "root-project".to_string(),+ },+ project_ids: Vec::new(),+ documents: Vec::new(),+ })+ })+ });+ let event_broker = TestEventBroker::default();+ let published = event_broker.published();+ let service = service_with_event_broker(repo, RecordingBulkUpload::default(), event_broker);++ service+ .upload_folder(+ user_id("macro|[email protected]"),+ false,+ UploadFolderRequest {+ content: Vec::new(),+ root_folder_name: "Uploaded tree".to_string(),+ upload_request_id: "request".to_string(),+ parent_id: None,+ },+ )+ .await+ .unwrap();++ assert!(published.lock().unwrap().is_empty());+}+#[tokio::test]async fn upload_folder_compensates_after_destination_failure() {let mut repo = MockProjectRepo::new();@@ -2088,6 +2127,8 @@ async fn upload_folder_compensates_after_destination_failure() {repo.expect_delete_uploaded_tree().withf(|projects, documents| projects == ["project"] && documents == ["document"]).return_once(|_, _| Box::pin(async { Ok(()) }));+ let event_broker = TestEventBroker::default();+ let published = event_broker.published();let service = ProjectServiceImpl::new(repo,RecordingUploadUrls {@@ -2099,7 +2140,7 @@ async fn upload_folder_compensates_after_destination_failure() {NullPort,NullPort,None,- TestEventBroker::default(),+ event_broker,);let result = service@@ -2122,6 +2163,7 @@ async fn upload_folder_compensates_after_destination_failure() {.await;assert!(matches!(result, Err(ProjectError::Internal(_))));+ assert!(published.lock().unwrap().is_empty());}#[tokio::test]diff --git a/crates/projects/src/outbound/sqs_search_indexer.rs b/crates/projects/src/outbound/sqs_search_indexer.rsindex eb89ae672f..447429bc2c 100644--- a/crates/projects/src/outbound/sqs_search_indexer.rs+++ b/crates/projects/src/outbound/sqs_search_indexer.rs@@ -1,17 +1,12 @@-//! SQS adapter for project search indexing and document deletion.+//! SQS adapter for cross-family search cleanup and document deletion.use std::sync::Arc;-use sqs_client::search::{- SearchQueueMessage,- chat::RemoveChatMessage,- document::DocumentId,- project::{RemoveProject, UpsertProject},-};+use sqs_client::search::{SearchQueueMessage, chat::RemoveChatMessage, document::DocumentId};use crate::domain::ports::ProjectSearchIndexer;-/// SQS-backed project search-index and document-deletion adapter.+/// SQS-backed cross-family search cleanup and document-deletion adapter.#[derive(Clone)]pub struct SqsProjectSearchIndexer {sqs: Arc<sqs_client::SQS>,@@ -25,40 +20,6 @@ impl SqsProjectSearchIndexer {}impl ProjectSearchIndexer for SqsProjectSearchIndexer {- #[tracing::instrument(skip(self), err)]- async fn upsert_projects(&self, project_ids: Vec<String>) -> anyhow::Result<()> {- let messages = project_ids- .into_iter()- .map(|project_id| {- SearchQueueMessage::UpsertProject(UpsertProject {- project_id,- index_override: None,- })- })- .collect();-- self.sqs- .bulk_send_message_to_search_event_queue(messages)- .await- }-- #[tracing::instrument(skip(self), err)]- async fn remove_projects(&self, project_ids: Vec<String>) -> anyhow::Result<()> {- let messages = project_ids- .into_iter()- .map(|project_id| {- SearchQueueMessage::RemoveProject(RemoveProject {- project_id,- index_override: None,- })- })- .collect();-- self.sqs- .bulk_send_message_to_search_event_queue(messages)- .await- }-#[tracing::instrument(skip(self), err)]async fn remove_chats(&self, chat_ids: Vec<String>) -> anyhow::Result<()> {let messages = chat_idsdiff --git a/crates/sqs_client/src/search/mod.rs b/crates/sqs_client/src/search/mod.rsindex 0f0f0c82a4..e1bd1f8bfc 100644--- a/crates/sqs_client/src/search/mod.rs+++ b/crates/sqs_client/src/search/mod.rs@@ -6,7 +6,7 @@ use crate::{chat::{ChatMessage, RemoveChatMessage},document::{DocumentId, DocumentPropertiesUpdate, SearchExtractorMessage},email::{EmailLinkMessage, EmailMessage, EmailThreadBatchMessage, EmailThreadMessage},- project::{RemoveProject, UpsertProject},+ project::UpsertProject,},};use anyhow::Context;@@ -90,7 +90,6 @@ pub enum SearchQueueMessage {RemoveCallRecord(RemoveCallRecord),// ProjectUpsertProject(UpsertProject),- RemoveProject(RemoveProject),// UserRemoveUserProfile(String),@@ -122,7 +121,6 @@ impl PrimaryId for SearchQueueMessage {message.call_id.clone().unwrap_or_default()),SearchQueueMessage::UpsertProject(message) => message.project_id.clone(),- SearchQueueMessage::RemoveProject(message) => message.project_id.clone(),SearchQueueMessage::RemoveUserProfile(message) => message.clone(),}@@ -154,7 +152,6 @@ impl SearchQueueMessage {SearchQueueMessage::RemoveCallRecord(_) => Operation::Remove,// ProjectsSearchQueueMessage::UpsertProject(_) => Operation::UpdateMetadata,- SearchQueueMessage::RemoveProject(_) => Operation::Remove,// UsersSearchQueueMessage::RemoveUserProfile(_) => Operation::Remove,}diff --git a/crates/sqs_client/src/search/project.rs b/crates/sqs_client/src/search/project.rsindex 5c45b9c420..8e2af56ab5 100644--- a/crates/sqs_client/src/search/project.rs+++ b/crates/sqs_client/src/search/project.rs@@ -1,3 +1,7 @@+/// Requests project reconciliation through the SQS backfill path.+///+/// Project lifecycle updates use Kafka. This payload remains available only so+/// search backfills can use `index_override` to target an alternate index.#[derive(serde::Serialize, serde::Deserialize, Debug, PartialEq, Eq)]pub struct UpsertProject {/// The project id@@ -6,12 +10,3 @@ pub struct UpsertProject {#[serde(skip_serializing_if = "Option::is_none", default)]pub index_override: Option<String>,}--#[derive(serde::Serialize, serde::Deserialize, Debug, PartialEq, Eq)]-pub struct RemoveProject {- /// The project id to remove- pub project_id: String,- /// Optional override for the target OpenSearch index- #[serde(skip_serializing_if = "Option::is_none", default)]- pub index_override: Option<String>,-}diff --git a/infra/stacks/deleted-item-poller/index.ts b/infra/stacks/deleted-item-poller/index.tsindex 485bc0dd79..90ff7e1e06 100644--- a/infra/stacks/deleted-item-poller/index.ts+++ b/infra/stacks/deleted-item-poller/index.ts@@ -23,6 +23,17 @@ const cloudStorageServiceStack = new pulumi.StackReference(});+const kafkaClusterStack = new pulumi.StackReference(+ 'kafka-cluster-brokers-stack',+ {+ name: `macro-inc/kafka-cluster/${stack}`,+ }+);++const kafkaBrokers = kafkaClusterStack+ .getOutput('bootstrapBrokersSaslIam')+ .apply((brokers) => brokers as string);+const deleteDocumentQueueArn: pulumi.Output<string> = cloudStorageServiceStack.getOutput('deleteDocumentQueueArn').apply((arn) => arn as string);@@ -41,6 +52,7 @@ const deletedItemPoller = new DeleteItemPoller('deleted-item-poller', {envVars: {DATABASE_URL: pulumi.interpolate`${DATABASE_URL}`,ENVIRONMENT: stack,+ KAFKA_BROKERS: kafkaBrokers,RUST_LOG: 'deleted_item_poller=info,macro_http_request=info',},tags,diff --git a/infra/stacks/deleted-item-poller/lambda.ts b/infra/stacks/deleted-item-poller/lambda.tsindex d6426ffcaa..5a554aeb55 100644--- a/infra/stacks/deleted-item-poller/lambda.ts+++ b/infra/stacks/deleted-item-poller/lambda.ts@@ -1,7 +1,11 @@import * as aws from '@pulumi/aws';import * as pulumi from '@pulumi/pulumi';import { Lambda } from '../../packages/lambda';-import { CLOUD_TRAIL_SNS_TOPIC_ARN, stack } from '../../packages/shared';+import {+ CLOUD_TRAIL_SNS_TOPIC_ARN,+ getKafkaClusterPolicy,+ stack,+} from '../../packages/shared';const LAMBDA_BASE_NAME = 'deleted_item_poller';const REPO_ROOT = '../../..';@@ -11,6 +15,7 @@ const ZIP_LOCATION = `${REPO_ROOT}/target/lambda/${LAMBDA_BASE_NAME}/bootstrap.zexport type EnvVars = {DATABASE_URL: pulumi.Output<string> | string;ENVIRONMENT: pulumi.Output<string> | string;+ KAFKA_BROKERS: pulumi.Output<string> | string;RUST_LOG: pulumi.Output<string> | string;};@@ -80,6 +85,7 @@ export class DeleteItemPoller extends pulumi.ComponentResource {aws.iam.ManagedPolicy.AWSLambdaVPCAccessExecutionRole,aws.iam.ManagedPolicy.CloudWatchLogsFullAccess,sqsPolicy.arn,+ getKafkaClusterPolicy(),],tags: this.tags,},diff --git a/services/deleted_item_poller/Cargo.toml b/services/deleted_item_poller/Cargo.tomlindex 9b8ce72308..eba4186fcf 100644--- a/services/deleted_item_poller/Cargo.toml+++ b/services/deleted_item_poller/Cargo.toml@@ -17,9 +17,12 @@ macro_db_client = { path = "../../crates/macro_db_client" }macro_entrypoint = { path = "../../crates/macro_entrypoint" }macro_env = { path = "../../crates/macro_env" }macro_env_var = { path = "../../crates/macro_env_var" }+macro_event_broker = { path = "../../crates/macro_event_broker" }macro_queues = { path = "../../crates/macro_queues" }+macro_user_id = { path = "../../crates/macro_user_id", default-features = false }macro_uuid = { path = "../../crates/macro_uuid" }model = { path = "../../crates/model" }+projects = { path = "../../crates/projects", default-features = false }serde = { workspace = true }serde_json = { workspace = true }sqlx = { workspace = true }diff --git a/services/deleted_item_poller/src/config.rs b/services/deleted_item_poller/src/config.rsindex 3bf0211428..5780c8e957 100644--- a/services/deleted_item_poller/src/config.rs+++ b/services/deleted_item_poller/src/config.rs@@ -4,6 +4,8 @@ use macro_env_var::env_vars;env_vars! {struct DatabaseUrl;+ #[derive(Debug, Clone)]+ pub struct KafkaBrokers;}#[derive(Debug, Clone)]@@ -11,6 +13,9 @@ pub struct Config {/// The connection URL for the Postgres database this application should use.pub database_url: String,+ /// Comma-separated Kafka bootstrap servers for project lifecycle events.+ pub kafka_brokers: KafkaBrokers,+/// The environment we are in#[allow(dead_code)]pub environment: Environment,@@ -21,9 +26,11 @@ impl Config {let database_url = DatabaseUrl::new().context("DATABASE_URL must be provided")?.to_string();+ let kafka_brokers = KafkaBrokers::new().context("KAFKA_BROKERS must be provided")?;Ok(Config {database_url,+ kafka_brokers,environment: Environment::new_or_prod(),})}diff --git a/services/deleted_item_poller/src/context.rs b/services/deleted_item_poller/src/context.rsindex 7b9c0db64e..a92c89477a 100644--- a/services/deleted_item_poller/src/context.rs+++ b/services/deleted_item_poller/src/context.rs@@ -1,7 +1,12 @@use std::sync::Arc;+use macro_event_broker::{GlobalSpawner, KafkaEventPublisher, MacroEventBrokerService};++pub type PollerEventBroker = MacroEventBrokerService<KafkaEventPublisher, GlobalSpawner>;+#[derive(Clone)]pub struct Context {pub db: sqlx::Pool<sqlx::Postgres>,+ pub macro_event_broker: PollerEventBroker,pub sqs_client: Arc<sqs_client::SQS>,}diff --git a/services/deleted_item_poller/src/handler.rs b/services/deleted_item_poller/src/handler.rsindex 590a2bffa6..4599839b54 100644--- a/services/deleted_item_poller/src/handler.rs+++ b/services/deleted_item_poller/src/handler.rs@@ -1,13 +1,19 @@+#[cfg(test)]+mod test;+use crate::context::{self};use anyhow::Context;use aws_lambda_events::eventbridge::EventBridgeEvent;+use futures::future::join_all;use lambda_runtime::{Error, LambdaEvent,tracing::{self},};-use sqs_client::search::{- SearchQueueMessage, chat::RemoveChatMessage, document::DocumentId, project::RemoveProject,-};+use macro_db_client::projects::ProjectToDelete;+use macro_event_broker::MacroEventBroker;+use macro_user_id::user_id::MacroUserIdStr;+use projects::domain::events::{ProjectMacroEvent, ProjectPermanentlyDeletedMetadata};+use sqs_client::search::{SearchQueueMessage, chat::RemoveChatMessage, document::DocumentId};#[tracing::instrument(skip(ctx, _event), err)]pub async fn handler(@@ -23,6 +29,53 @@ pub async fn handler(Ok(())}+#[tracing::instrument(skip(event_broker, projects_to_delete), err)]+async fn publish_project_purge_events<B: MacroEventBroker>(+ event_broker: &B,+ projects_to_delete: &[ProjectToDelete],+) -> anyhow::Result<()> {+ let events = projects_to_delete+ .iter()+ .map(|project| {+ let project_id = project.project_id.clone();+ let owner: MacroUserIdStr<'static> = MacroUserIdStr::try_from(project.user_id.clone())+ .with_context(|| format!("invalid owner for project {}", project.project_id))?;++ Ok(ProjectMacroEvent::permanently_deleted(+ project_id.clone(),+ ProjectPermanentlyDeletedMetadata {+ project_id: project_id.clone(),+ owner,+ actor_user_id: None,+ parent_project_id: None,+ purged_project_ids: vec![project_id],+ purged_document_ids: Vec::new(),+ purged_chat_ids: Vec::new(),+ },+ ))+ })+ .collect::<anyhow::Result<Vec<_>>>()?;++ let publications = events+ .iter()+ .map(|event| event_broker.send_event(event))+ .collect::<Vec<_>>();+ let publication_results = join_all(publications.into_iter().map(|publication| async move {+ let handle = publication.context("failed to enqueue project purge event")?;+ handle+ .await+ .context("project purge event publication task failed")?+ .context("failed to publish project purge event")+ }))+ .await;++ for result in publication_results {+ result?;+ }++ Ok(())+}+#[tracing::instrument(skip(ctx), err)]async fn handle_projects(ctx: &context::Context) -> anyhow::Result<()> {let date = chrono::Utc::now().naive_utc() - chrono::Duration::days(30);@@ -37,25 +90,18 @@ async fn handle_projects(ctx: &context::Context) -> anyhow::Result<()> {tracing::debug!(projects_to_delete=?projects_to_delete, "projects to delete");- // Idempotent safety net — the soft delete already removed these from the- // search index.- ctx.sqs_client- .bulk_send_message_to_search_event_queue(- projects_to_delete- .iter()- .map(|id| {- SearchQueueMessage::RemoveProject(RemoveProject {- project_id: id.to_string(),- index_override: None,- })- })- .collect(),- )- .await?;+ publish_project_purge_events(&ctx.macro_event_broker, &projects_to_delete)+ .await+ .context("unable to publish project purge events")?;++ let project_ids = projects_to_delete+ .into_iter()+ .map(|project| project.project_id)+ .collect::<Vec<String>>();// We can actually perform the project deletion here as we will automatically be queuing all// the items in the project for deletion as well- macro_db_client::projects::delete::delete_projects_bulk(&ctx.db, &projects_to_delete)+ macro_db_client::projects::delete::delete_projects_bulk(&ctx.db, &project_ids).await.context("unable to delete projects")?;diff --git a/services/deleted_item_poller/src/handler/test.rs b/services/deleted_item_poller/src/handler/test.rsnew file mode 100644index 0000000000..c342399143--- /dev/null+++ b/services/deleted_item_poller/src/handler/test.rs@@ -0,0 +1,128 @@+use std::sync::{Arc, Mutex};++use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker};+use serde_json::{Value, json};++use super::*;++#[derive(Clone, Debug, PartialEq)]+struct PublishedEvent {+ topic: String,+ key: String,+ envelope: Value,+}++#[derive(Clone, Default)]+struct FakeEventBroker {+ published: Arc<Mutex<Vec<PublishedEvent>>>,+ fail_publication: bool,+}++impl FakeEventBroker {+ fn failing() -> Self {+ Self {+ fail_publication: true,+ ..Self::default()+ }+ }++ fn published(&self) -> Vec<PublishedEvent> {+ self.published.lock().unwrap().clone()+ }+}++impl MacroEventBroker for FakeEventBroker {+ fn send_event<E: MacroEvent + ?Sized>(+ &self,+ event: &E,+ ) -> Result<tokio::task::JoinHandle<Result<(), EventBrokerError>>, EventBrokerError> {+ self.published.lock().unwrap().push(PublishedEvent {+ topic: event.topic().to_string(),+ key: event.key().to_string(),+ envelope: serde_json::to_value(event.event())?,+ });++ let fail_publication = self.fail_publication;+ Ok(tokio::spawn(async move {+ if fail_publication {+ Err(EventBrokerError::Publish(+ "publisher unavailable".to_string(),+ ))+ } else {+ Ok(())+ }+ }))+ }+}++#[tokio::test]+async fn publish_project_purge_events_publishes_separately_keyed_events() {+ let event_broker = FakeEventBroker::default();+ let projects = vec![+ ProjectToDelete {+ project_id: "project-one".to_string(),+ user_id: "macro|[email protected]".to_string(),+ },+ ProjectToDelete {+ project_id: "project-two".to_string(),+ user_id: "macro|[email protected]".to_string(),+ },+ ];++ publish_project_purge_events(&event_broker, &projects)+ .await+ .unwrap();++ let published = event_broker.published();+ assert_eq!(published.len(), projects.len());++ for (event, project) in published.iter().zip(&projects) {+ assert_eq!(event.topic, "macro.projects");+ assert_eq!(event.key, project.project_id);+ assert_eq!(event.envelope["schema_version"], json!(1));+ assert_eq!(+ event.envelope["event_type"],+ json!("project.permanently_deleted")+ );++ let metadata = &event.envelope["metadata"];+ assert_eq!(metadata["project_id"], json!(project.project_id));+ assert_eq!(metadata["owner"], json!(project.user_id));+ assert_eq!(metadata["actor_user_id"], Value::Null);+ assert_eq!(metadata["parent_project_id"], Value::Null);+ assert_eq!(metadata["purged_project_ids"], json!([project.project_id]));+ assert_eq!(metadata["purged_document_ids"], json!([]));+ assert_eq!(metadata["purged_chat_ids"], json!([]));+ }+}++#[tokio::test]+async fn publish_project_purge_events_returns_publication_failures() {+ let event_broker = FakeEventBroker::failing();+ let projects = vec![ProjectToDelete {+ project_id: "project-one".to_string(),+ user_id: "macro|[email protected]".to_string(),+ }];++ let error = publish_project_purge_events(&event_broker, &projects)+ .await+ .expect_err("publication failure should be returned");++ assert!(format!("{error:#}").contains("publisher unavailable"));+}++#[tokio::test]+async fn publish_project_purge_events_rejects_malformed_owners() {+ let event_broker = FakeEventBroker::default();+ let projects = vec![ProjectToDelete {+ project_id: "project-one".to_string(),+ user_id: "not-a-macro-user".to_string(),+ }];++ let error = publish_project_purge_events(&event_broker, &projects)+ .await+ .expect_err("malformed owners should be returned");++ assert!(format!("{error:#}").contains("invalid owner for project project-one"));+ assert!(event_broker.published().is_empty());+}diff --git a/services/deleted_item_poller/src/main.rs b/services/deleted_item_poller/src/main.rsindex 7033137cee..e4022b7175 100644--- a/services/deleted_item_poller/src/main.rs+++ b/services/deleted_item_poller/src/main.rs@@ -12,6 +12,7 @@ use lambda_runtime::{tracing::{self},};use macro_entrypoint::MacroEntrypoint;+use macro_event_broker::{GlobalSpawner, KafkaEventPublisher};use sqlx::postgres::PgPoolOptions;use std::sync::Arc;@@ -24,6 +25,12 @@ async fn main() -> Result<(), Error> {tracing::trace!("initialized config");+ let macro_event_broker = context::PollerEventBroker::new(+ KafkaEventPublisher::new(config.kafka_brokers.as_ref())+ .context("failed to create kafka event publisher")?,+ GlobalSpawner,+ );+// We should only ever need 1 connectionlet db = PgPoolOptions::new().min_connections(3)@@ -44,6 +51,7 @@ async fn main() -> Result<(), Error> {let ctx = context::Context {db,+ macro_event_broker,sqs_client: Arc::new(sqs_client),};diff --git a/services/document_storage_service/src/api/context.rs b/services/document_storage_service/src/api/context.rsindex 7a94cdc840..9f2ab31b72 100644--- a/services/document_storage_service/src/api/context.rs+++ b/services/document_storage_service/src/api/context.rs@@ -451,6 +451,7 @@ pub(crate) struct ApiContext {pub favorites_state: DssFavoritesState,pub favorites_service: Arc<FavoritesServiceType>,pub foreign_entity_state: DssForeignEntityState,+ pub macro_event_broker: DssEventBroker,pub sqs_client: Arc<sqs_client::SQS>,pub contacts_ingress: Arc<SqsContactsIngress<SqsContactsQueue>>,pub notification_ingress_service: Arc<NotificationIngressType>,diff --git a/services/document_storage_service/src/api/documents/initialize_user_documents.rs b/services/document_storage_service/src/api/documents/initialize_user_documents.rsindex 74f7819804..21547b92a3 100644--- a/services/document_storage_service/src/api/documents/initialize_user_documents.rs+++ b/services/document_storage_service/src/api/documents/initialize_user_documents.rs@@ -5,15 +5,16 @@ use axum::{};use futures::StreamExt;use macro_authorization::{MacroAuthorizationExtractor, UserOrInternal};+use macro_event_broker::MacroEventBroker;use macro_user_id::cowlike::CowLike;use model::{document::BasicDocument,response::{ErrorResponse, GenericErrorResponse, GenericSuccessResponse},};use models_permissions::share_permission::SharePermissionV2;+use projects_hex::domain::events::{ProjectCreatedMetadata, ProjectMacroEvent};use reqwest::StatusCode;use s3_key::build_cloud_storage_bucket_document_key;-use sqs_client::search::{SearchQueueMessage, project::UpsertProject};const ONBOARDING_FOLDER_NAME: &str = "ONBOARDING_DOCUMENTS";const PROJECT_NAME: &str = "Starter Docs";@@ -246,23 +247,26 @@ pub async fn handler(.into_response()})?;- tokio::spawn({- let sqs_client = state.sqs_client.clone();- let project_id = project.id.clone();- async move {- let _ = sqs_client- .send_message_to_search_event_queue(SearchQueueMessage::UpsertProject(- UpsertProject {- project_id: project_id.clone(),- index_override: None,- },- ))- .await- .inspect_err(- |e| tracing::error!(error=?e, project_id=?project_id, "unable to enqueue project search upsert"),- );- }- });+ let project_event = ProjectMacroEvent::created(+ project.id.clone(),+ ProjectCreatedMetadata {+ project_id: project.id.clone(),+ owner: user_context.authorization.user.macro_user_id.clone(),+ name: PROJECT_NAME.to_string(),+ parent_project_id: None,+ created_at: project.created_at,+ },+ );+ let _ = state+ .macro_event_broker+ .send_event(&project_event)+ .inspect_err(|error| {+ tracing::error!(+ error=?error,+ project_id = %project.id,+ "unable to enqueue project created event"+ );+ });Ok((StatusCode::OK, Json(GenericSuccessResponse::default())).into_response())}diff --git a/services/document_storage_service/src/main.rs b/services/document_storage_service/src/main.rsindex 49f7abd9cc..13cdbf1773 100644--- a/services/document_storage_service/src/main.rs+++ b/services/document_storage_service/src/main.rs@@ -1044,6 +1044,7 @@ async fn main() -> anyhow::Result<()> {s3_client: s3,dynamodb_client: Arc::new(dynamodb_client),dynamo_db,+ macro_event_broker: macro_event_broker.clone(),sqs_client: sqs_client.clone(),notification_ingress_service: notification_ingress_service.clone(),conn_gateway_client: conn_gateway_client.clone(),diff --git a/services/search_processing_service/Cargo.toml b/services/search_processing_service/Cargo.tomlindex 23cb6e3773..bfc46f9bab 100644--- a/services/search_processing_service/Cargo.toml+++ b/services/search_processing_service/Cargo.toml@@ -13,6 +13,7 @@ processing = ["dep:kafka_util","dep:macro_event_broker","dep:macro_event_topics",+ "dep:projects","dep:rdkafka","dep:rootcause","dep:tokio-retry",@@ -78,6 +79,7 @@ pdfium-render = { workspace = true, optional = true }properties = { path = "../../crates/properties", default-features = false, features = ["outbound",], optional = true }+projects = { path = "../../crates/projects", default-features = false, optional = true }rdkafka = { workspace = true, optional = true }rootcause = { workspace = true, optional = true }rust-embed = "8.6.0"diff --git a/services/search_processing_service/src/inbound/kafka_consumer.rs b/services/search_processing_service/src/inbound/kafka_consumer.rsindex 391070dee2..298924a806 100644--- a/services/search_processing_service/src/inbound/kafka_consumer.rs+++ b/services/search_processing_service/src/inbound/kafka_consumer.rs@@ -5,19 +5,27 @@//! committed without a handoff so they cannot wedge a partition. Processing is//! retried in-process; exhausted events are logged and dropped because their//! offsets are already committed.+//!+//! Per-entity event mapping and processing live in the [`call`], [`channel`],+//! and [`project`] submodules; this module owns the poll loop, worker, retry+//! policy, and commit semantics.++#![allow(clippy::enum_variant_names)]+mod call;+mod channel;+mod project;#[cfg(test)]mod test;use std::{future::Future, time::Duration};-use call::domain::events::{CallMacroEvent, CallTopicEvent};-use channels::domain::broker_events::{ChannelMacroEvent, ChannelTopicEvent};+use ::call::domain::events::CallMacroEvent;+use channels::domain::broker_events::ChannelMacroEvent;use kafka_util::{GroupName, KafkaEventConsumer};-use macro_event_broker::{- KafkaConsumerAdapter, MacroEvent as _, MacroEventCollection as _, MacroEventConsumerService,-};+use macro_event_broker::{KafkaConsumerAdapter, MacroEventCollection, MacroEventConsumerService};use opensearch_client::OpensearchClient;+use projects::domain::events::ProjectMacroEvent;use rdkafka::{consumer::CommitMode,message::{BorrowedMessage, Message as _},@@ -26,11 +34,9 @@ use rootcause::prelude::{Report, ResultExt as _};use sqlx::PgPool;use tokio::sync::mpsc;use tokio_retry::{Retry, strategy::ExponentialBackoff};-use uuid::Uuid;-use crate::process::{- call::{process_call_record, process_remove_call_record},- channel::{process_channel_message_update, process_remove_channel_message},+use self::{+ call::process_call_event, channel::process_channel_event, project::process_project_event,};/// Consumer group used for live search-index event offsets.@@ -41,12 +47,15 @@ impl GroupName for SearchProcessingConsumerGroup {}type SearchProcessingKafkaAdapter =- KafkaConsumerAdapter<SearchProcessingConsumerGroup, SearchProcessingBrokerEvent>;+ KafkaConsumerAdapter<SearchProcessingConsumerGroup, DeclaredMacroEvent>;type SearchProcessingKafkaConsumer =- MacroEventConsumerService<SearchProcessingBrokerEvent, SearchProcessingKafkaAdapter>;+ MacroEventConsumerService<DeclaredMacroEvent, SearchProcessingKafkaAdapter>;macro_event_broker::declare_topics!(- SearchProcessingBrokerEvent: CallMacroEvent, ChannelMacroEvent+ DeclaredMacroEvent:+ CallMacroEvent,+ ChannelMacroEvent,+ ProjectMacroEvent,);/// Maximum number of decoded events waiting for the sequential worker.@@ -60,40 +69,11 @@ const PROCESSING_RETRY_BASE_DELAY: Duration = Duration::from_secs(1);/// A decoded search-processing event and its Kafka coordinates.struct ReceivedEvent {- event: SearchProcessingBrokerEvent,+ event: DeclaredMacroEvent,partition: i32,offset: i64,}-#[derive(Debug, Clone, Copy, PartialEq, Eq)]-enum CallIndexAction {- Upsert { call_id: Uuid },- Remove { call_id: Uuid, channel_id: Uuid },- Ignore,-}--#[derive(Debug, Clone, Copy, PartialEq, Eq)]-struct CallEventDescription {- action: CallIndexAction,- call_id: Uuid,- event_type: &'static str,-}--#[derive(Debug, Clone, Copy, PartialEq, Eq)]-enum ChannelIndexAction {- UpsertMessage { channel_id: Uuid, message_id: Uuid },- RemoveMessage { channel_id: Uuid, message_id: Uuid },- RemoveChannel { channel_id: Uuid },- Ignore,-}--#[derive(Debug, Clone, Copy, PartialEq, Eq)]-struct ChannelEventDescription {- action: ChannelIndexAction,- channel_id: Uuid,- event_type: &'static str,-}-/// Outcome after the worker handles one decoded broker event.#[derive(Debug, Clone, Copy, PartialEq, Eq)]enum EventOutcome {@@ -117,122 +97,6 @@ enum HandoffOutcome<E> {WorkerClosed,}-fn describe_call_event(event: &CallTopicEvent) -> CallEventDescription {- match event {- CallTopicEvent::Started(metadata) => CallEventDescription {- action: CallIndexAction::Ignore,- call_id: metadata.call_id,- event_type: "call.started",- },- CallTopicEvent::RecordArchived(metadata) => CallEventDescription {- action: CallIndexAction::Upsert {- call_id: metadata.call_id,- },- call_id: metadata.call_id,- event_type: "call.record_archived",- },- CallTopicEvent::RecordUpdated(metadata) => CallEventDescription {- action: CallIndexAction::Upsert {- call_id: metadata.call_id,- },- call_id: metadata.call_id,- event_type: "call.record_updated",- },- CallTopicEvent::RecordDeleted(metadata) => CallEventDescription {- action: CallIndexAction::Remove {- call_id: metadata.call_id,- channel_id: metadata.channel_id,- },- call_id: metadata.call_id,- event_type: "call.record_deleted",- },- CallTopicEvent::RecordSummarized(metadata) => CallEventDescription {- action: CallIndexAction::Upsert {- call_id: metadata.call_id,- },- call_id: metadata.call_id,- event_type: "call.record_summarized",- },- CallTopicEvent::RecordingReady(metadata) => CallEventDescription {- action: CallIndexAction::Ignore,- call_id: metadata.call_id,- event_type: "call.recording_ready",- },- }-}--fn describe_channel_event(event: &ChannelTopicEvent) -> ChannelEventDescription {- match event {- ChannelTopicEvent::Created(metadata) => ChannelEventDescription {- action: ChannelIndexAction::Ignore,- channel_id: metadata.channel_id,- event_type: "channel.created",- },- ChannelTopicEvent::Updated(metadata) => ChannelEventDescription {- action: ChannelIndexAction::Ignore,- channel_id: metadata.channel_id,- event_type: "channel.updated",- },- ChannelTopicEvent::Deleted(metadata) => ChannelEventDescription {- action: ChannelIndexAction::RemoveChannel {- channel_id: metadata.channel_id,- },- channel_id: metadata.channel_id,- event_type: "channel.deleted",- },- ChannelTopicEvent::MessagePosted(metadata) => ChannelEventDescription {- action: ChannelIndexAction::UpsertMessage {- channel_id: metadata.channel_id,- message_id: metadata.message_id,- },- channel_id: metadata.channel_id,- event_type: "channel.message_posted",- },- ChannelTopicEvent::MessagePatched(metadata) => ChannelEventDescription {- action: ChannelIndexAction::UpsertMessage {- channel_id: metadata.channel_id,- message_id: metadata.message_id,- },- channel_id: metadata.channel_id,- event_type: "channel.message_patched",- },- ChannelTopicEvent::MessageDeleted(metadata) => ChannelEventDescription {- action: ChannelIndexAction::RemoveMessage {- channel_id: metadata.channel_id,- message_id: metadata.message_id,- },- channel_id: metadata.channel_id,- event_type: "channel.message_deleted",- },- ChannelTopicEvent::MessageAttachmentCreated(metadata) => ChannelEventDescription {- action: ChannelIndexAction::UpsertMessage {- channel_id: metadata.channel_id,- message_id: metadata.message_id,- },- channel_id: metadata.channel_id,- event_type: "channel.message_attachment_created",- },- ChannelTopicEvent::MessageAttachmentRemoved(metadata) => ChannelEventDescription {- action: ChannelIndexAction::UpsertMessage {- channel_id: metadata.channel_id,- message_id: metadata.message_id,- },- channel_id: metadata.channel_id,- event_type: "channel.message_attachment_removed",- },- ChannelTopicEvent::ParticipantAdded(metadata) => ChannelEventDescription {- action: ChannelIndexAction::Ignore,- channel_id: metadata.channel_id,- event_type: "channel.participant_added",- },- ChannelTopicEvent::ParticipantRemoved(metadata) => ChannelEventDescription {- action: ChannelIndexAction::Ignore,- channel_id: metadata.channel_id,- event_type: "channel.participant_removed",- },- }-}-fn processing_retry_strategy() -> impl Iterator<Item = Duration> {ExponentialBackoff::from_millis(2).factor(500)@@ -280,7 +144,7 @@ async fn handoff_decoded<T, E>(}fn attach_event_coordinates<E>(- decoded: Result<SearchProcessingBrokerEvent, E>,+ decoded: Result<DeclaredMacroEvent, E>,partition: i32,offset: i64,) -> Result<ReceivedEvent, E> {@@ -291,197 +155,24 @@ fn attach_event_coordinates<E>(})}-async fn process_call_index_action(- db: &PgPool,- opensearch_client: &OpensearchClient,- action: CallIndexAction,-) -> anyhow::Result<()> {- match action {- CallIndexAction::Upsert { call_id } => {- process_call_record(opensearch_client, db, call_id, None).await- }- CallIndexAction::Remove {- call_id,- channel_id,- } => process_remove_call_record(opensearch_client, channel_id, Some(call_id), None).await,- CallIndexAction::Ignore => Ok(()),- }-}--async fn process_channel_index_action(- db: &PgPool,- opensearch_client: &OpensearchClient,- action: ChannelIndexAction,-) -> anyhow::Result<()> {- match action {- ChannelIndexAction::UpsertMessage {- channel_id,- message_id,- } => {- process_channel_message_update(opensearch_client, db, channel_id, message_id, None)- .await- }- ChannelIndexAction::RemoveMessage {- channel_id,- message_id,- } => {- process_remove_channel_message(opensearch_client, channel_id, Some(message_id), None)- .await- }- ChannelIndexAction::RemoveChannel { channel_id } => {- process_remove_channel_message(opensearch_client, channel_id, None, None).await- }- ChannelIndexAction::Ignore => Ok(()),- }-}--async fn process_call_event(- db: &PgPool,- opensearch_client: &OpensearchClient,- event: &CallMacroEvent,- partition: i32,- offset: i64,-) -> EventOutcome {- let description = describe_call_event(&event.event().event);- if description.action == CallIndexAction::Ignore {- tracing::trace!(- call_id = %description.call_id,- event_type = description.event_type,- partition,- offset,- "ignoring call event without a search-index action"- );- return EventOutcome::Ignored;- }-- let result = retry_processing(|attempt| async move {- tracing::trace!(- call_id = %description.call_id,- event_type = description.event_type,- partition,- offset,- attempt,- "processing call search-index event"- );- process_call_index_action(db, opensearch_client, description.action)- .await- .inspect_err(|error| {- if attempt < MAX_PROCESSING_ATTEMPTS {- let retry_delay =- PROCESSING_RETRY_BASE_DELAY * 2u32.pow(attempt.saturating_sub(1));- tracing::warn!(- error = ?error,- call_id = %description.call_id,- event_type = description.event_type,- partition,- offset,- attempt,- delay_secs = retry_delay.as_secs(),- "call search-index processing failed, retrying"- );- }- })- })- .await;-- match result {- Ok(()) => EventOutcome::Indexed,- Err(error) => {- tracing::error!(- error = ?error,- call_id = %description.call_id,- event_type = description.event_type,- partition,- offset,- attempts = MAX_PROCESSING_ATTEMPTS,- "dropping call event after processing retries were exhausted"- );- EventOutcome::Dropped- }- }-}--async fn process_channel_event(- db: &PgPool,- opensearch_client: &OpensearchClient,- event: &ChannelMacroEvent,- partition: i32,- offset: i64,-) -> EventOutcome {- let description = describe_channel_event(&event.event().event);- if description.action == ChannelIndexAction::Ignore {- tracing::trace!(- channel_id = %description.channel_id,- event_type = description.event_type,- partition,- offset,- "ignoring channel event without a search-index action"- );- return EventOutcome::Ignored;- }-- let result = retry_processing(|attempt| async move {- tracing::trace!(- channel_id = %description.channel_id,- event_type = description.event_type,- partition,- offset,- attempt,- "processing channel search-index event"- );- process_channel_index_action(db, opensearch_client, description.action)- .await- .inspect_err(|error| {- if attempt < MAX_PROCESSING_ATTEMPTS {- let retry_delay =- PROCESSING_RETRY_BASE_DELAY * 2u32.pow(attempt.saturating_sub(1));- tracing::warn!(- error = ?error,- channel_id = %description.channel_id,- event_type = description.event_type,- partition,- offset,- attempt,- delay_secs = retry_delay.as_secs(),- "channel search-index processing failed, retrying"- );- }- })- })- .await;-- match result {- Ok(()) => EventOutcome::Indexed,- Err(error) => {- tracing::error!(- error = ?error,- channel_id = %description.channel_id,- event_type = description.event_type,- partition,- offset,- attempts = MAX_PROCESSING_ATTEMPTS,- "dropping channel event after processing retries were exhausted"- );- EventOutcome::Dropped- }- }-}-#[tracing::instrument(skip(db, opensearch_client, event), fields(partition, offset))]async fn process_event(db: &PgPool,opensearch_client: &OpensearchClient,- event: &SearchProcessingBrokerEvent,+ event: &DeclaredMacroEvent,partition: i32,offset: i64,) -> EventOutcome {match event {- SearchProcessingBrokerEvent::CallMacroEvent(event) => {+ DeclaredMacroEvent::CallMacroEvent(event) => {process_call_event(db, opensearch_client, event, partition, offset).await}- SearchProcessingBrokerEvent::ChannelMacroEvent(event) => {+ DeclaredMacroEvent::ChannelMacroEvent(event) => {process_channel_event(db, opensearch_client, event, partition, offset).await}+ DeclaredMacroEvent::ProjectMacroEvent(event) => {+ process_project_event(db, opensearch_client, event, partition, offset).await+ }}}@@ -598,11 +289,11 @@ pub(crate) async fn run_event_consumer() -> Result<(), Report> {let consumer = KafkaEventConsumer::<SearchProcessingConsumerGroup>::from_env(brokers)?;let consumer = KafkaConsumerAdapter::<SearchProcessingConsumerGroup, ()>::new(consumer)- .subscribe::<SearchProcessingBrokerEvent>()+ .subscribe::<DeclaredMacroEvent>().context("failed to subscribe to search processing event topics")?;let consumer = SearchProcessingKafkaConsumer::new(consumer);tracing::info!(- topics = ?SearchProcessingBrokerEvent::topics(),+ topics = ?DeclaredMacroEvent::topics(),group = SearchProcessingConsumerGroup::GROUP_NAME,"search processing event consumer listening");diff --git a/services/search_processing_service/src/inbound/kafka_consumer/call.rs b/services/search_processing_service/src/inbound/kafka_consumer/call.rsnew file mode 100644index 0000000000..024d85af25--- /dev/null+++ b/services/search_processing_service/src/inbound/kafka_consumer/call.rs@@ -0,0 +1,151 @@+//! Maps call lifecycle events to search-index actions and processes them.++use ::call::domain::events::{CallMacroEvent, CallTopicEvent};+use macro_event_broker::MacroEvent as _;+use opensearch_client::OpensearchClient;+use sqlx::PgPool;+use uuid::Uuid;++use super::{EventOutcome, MAX_PROCESSING_ATTEMPTS, PROCESSING_RETRY_BASE_DELAY, retry_processing};+use crate::process::call::{process_call_record, process_remove_call_record};++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub(super) enum CallIndexAction {+ Upsert { call_id: Uuid },+ Remove { call_id: Uuid, channel_id: Uuid },+ Ignore,+}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub(super) struct CallEventDescription {+ pub(super) action: CallIndexAction,+ pub(super) call_id: Uuid,+ pub(super) event_type: &'static str,+}++pub(super) fn describe_call_event(event: &CallTopicEvent) -> CallEventDescription {+ match event {+ CallTopicEvent::Started(metadata) => CallEventDescription {+ action: CallIndexAction::Ignore,+ call_id: metadata.call_id,+ event_type: "call.started",+ },+ CallTopicEvent::RecordArchived(metadata) => CallEventDescription {+ action: CallIndexAction::Upsert {+ call_id: metadata.call_id,+ },+ call_id: metadata.call_id,+ event_type: "call.record_archived",+ },+ CallTopicEvent::RecordUpdated(metadata) => CallEventDescription {+ action: CallIndexAction::Upsert {+ call_id: metadata.call_id,+ },+ call_id: metadata.call_id,+ event_type: "call.record_updated",+ },+ CallTopicEvent::RecordDeleted(metadata) => CallEventDescription {+ action: CallIndexAction::Remove {+ call_id: metadata.call_id,+ channel_id: metadata.channel_id,+ },+ call_id: metadata.call_id,+ event_type: "call.record_deleted",+ },+ CallTopicEvent::RecordSummarized(metadata) => CallEventDescription {+ action: CallIndexAction::Upsert {+ call_id: metadata.call_id,+ },+ call_id: metadata.call_id,+ event_type: "call.record_summarized",+ },+ CallTopicEvent::RecordingReady(metadata) => CallEventDescription {+ action: CallIndexAction::Ignore,+ call_id: metadata.call_id,+ event_type: "call.recording_ready",+ },+ }+}++async fn process_call_index_action(+ db: &PgPool,+ opensearch_client: &OpensearchClient,+ action: CallIndexAction,+) -> anyhow::Result<()> {+ match action {+ CallIndexAction::Upsert { call_id } => {+ process_call_record(opensearch_client, db, call_id, None).await+ }+ CallIndexAction::Remove {+ call_id,+ channel_id,+ } => process_remove_call_record(opensearch_client, channel_id, Some(call_id), None).await,+ CallIndexAction::Ignore => Ok(()),+ }+}++pub(super) async fn process_call_event(+ db: &PgPool,+ opensearch_client: &OpensearchClient,+ event: &CallMacroEvent,+ partition: i32,+ offset: i64,+) -> EventOutcome {+ let description = describe_call_event(&event.event().event);+ if description.action == CallIndexAction::Ignore {+ tracing::trace!(+ call_id = %description.call_id,+ event_type = description.event_type,+ partition,+ offset,+ "ignoring call event without a search-index action"+ );+ return EventOutcome::Ignored;+ }++ let result = retry_processing(|attempt| async move {+ tracing::trace!(+ call_id = %description.call_id,+ event_type = description.event_type,+ partition,+ offset,+ attempt,+ "processing call search-index event"+ );+ process_call_index_action(db, opensearch_client, description.action)+ .await+ .inspect_err(|error| {+ if attempt < MAX_PROCESSING_ATTEMPTS {+ let retry_delay =+ PROCESSING_RETRY_BASE_DELAY * 2u32.pow(attempt.saturating_sub(1));+ tracing::warn!(+ error = ?error,+ call_id = %description.call_id,+ event_type = description.event_type,+ partition,+ offset,+ attempt,+ delay_secs = retry_delay.as_secs(),+ "call search-index processing failed, retrying"+ );+ }+ })+ })+ .await;++ match result {+ Ok(()) => EventOutcome::Indexed,+ Err(error) => {+ tracing::error!(+ error = ?error,+ call_id = %description.call_id,+ event_type = description.event_type,+ partition,+ offset,+ attempts = MAX_PROCESSING_ATTEMPTS,+ "dropping call event after processing retries were exhausted"+ );+ EventOutcome::Dropped+ }+ }+}diff --git a/services/search_processing_service/src/inbound/kafka_consumer/channel.rs b/services/search_processing_service/src/inbound/kafka_consumer/channel.rsnew file mode 100644index 0000000000..e811a36e65--- /dev/null+++ b/services/search_processing_service/src/inbound/kafka_consumer/channel.rs@@ -0,0 +1,190 @@+//! Maps channel lifecycle events to search-index actions and processes them.++use channels::domain::broker_events::{ChannelMacroEvent, ChannelTopicEvent};+use macro_event_broker::MacroEvent as _;+use opensearch_client::OpensearchClient;+use sqlx::PgPool;+use uuid::Uuid;++use super::{EventOutcome, MAX_PROCESSING_ATTEMPTS, PROCESSING_RETRY_BASE_DELAY, retry_processing};+use crate::process::channel::{process_channel_message_update, process_remove_channel_message};++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub(super) enum ChannelIndexAction {+ UpsertMessage { channel_id: Uuid, message_id: Uuid },+ RemoveMessage { channel_id: Uuid, message_id: Uuid },+ RemoveChannel { channel_id: Uuid },+ Ignore,+}++#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub(super) struct ChannelEventDescription {+ pub(super) action: ChannelIndexAction,+ pub(super) channel_id: Uuid,+ pub(super) event_type: &'static str,+}++pub(super) fn describe_channel_event(event: &ChannelTopicEvent) -> ChannelEventDescription {+ match event {+ ChannelTopicEvent::Created(metadata) => ChannelEventDescription {+ action: ChannelIndexAction::Ignore,+ channel_id: metadata.channel_id,+ event_type: "channel.created",+ },+ ChannelTopicEvent::Updated(metadata) => ChannelEventDescription {+ action: ChannelIndexAction::Ignore,+ channel_id: metadata.channel_id,+ event_type: "channel.updated",+ },+ ChannelTopicEvent::Deleted(metadata) => ChannelEventDescription {+ action: ChannelIndexAction::RemoveChannel {+ channel_id: metadata.channel_id,+ },+ channel_id: metadata.channel_id,+ event_type: "channel.deleted",+ },+ ChannelTopicEvent::MessagePosted(metadata) => ChannelEventDescription {+ action: ChannelIndexAction::UpsertMessage {+ channel_id: metadata.channel_id,+ message_id: metadata.message_id,+ },+ channel_id: metadata.channel_id,+ event_type: "channel.message_posted",+ },+ ChannelTopicEvent::MessagePatched(metadata) => ChannelEventDescription {+ action: ChannelIndexAction::UpsertMessage {+ channel_id: metadata.channel_id,+ message_id: metadata.message_id,+ },+ channel_id: metadata.channel_id,+ event_type: "channel.message_patched",+ },+ ChannelTopicEvent::MessageDeleted(metadata) => ChannelEventDescription {+ action: ChannelIndexAction::RemoveMessage {+ channel_id: metadata.channel_id,+ message_id: metadata.message_id,+ },+ channel_id: metadata.channel_id,+ event_type: "channel.message_deleted",+ },+ ChannelTopicEvent::MessageAttachmentCreated(metadata) => ChannelEventDescription {+ action: ChannelIndexAction::UpsertMessage {+ channel_id: metadata.channel_id,+ message_id: metadata.message_id,+ },+ channel_id: metadata.channel_id,+ event_type: "channel.message_attachment_created",+ },+ ChannelTopicEvent::MessageAttachmentRemoved(metadata) => ChannelEventDescription {+ action: ChannelIndexAction::UpsertMessage {+ channel_id: metadata.channel_id,+ message_id: metadata.message_id,+ },+ channel_id: metadata.channel_id,+ event_type: "channel.message_attachment_removed",+ },+ ChannelTopicEvent::ParticipantAdded(metadata) => ChannelEventDescription {+ action: ChannelIndexAction::Ignore,+ channel_id: metadata.channel_id,+ event_type: "channel.participant_added",+ },+ ChannelTopicEvent::ParticipantRemoved(metadata) => ChannelEventDescription {+ action: ChannelIndexAction::Ignore,+ channel_id: metadata.channel_id,+ event_type: "channel.participant_removed",+ },+ }+}++async fn process_channel_index_action(+ db: &PgPool,+ opensearch_client: &OpensearchClient,+ action: ChannelIndexAction,+) -> anyhow::Result<()> {+ match action {+ ChannelIndexAction::UpsertMessage {+ channel_id,+ message_id,+ } => {+ process_channel_message_update(opensearch_client, db, channel_id, message_id, None)+ .await+ }+ ChannelIndexAction::RemoveMessage {+ channel_id,+ message_id,+ } => {+ process_remove_channel_message(opensearch_client, channel_id, Some(message_id), None)+ .await+ }+ ChannelIndexAction::RemoveChannel { channel_id } => {+ process_remove_channel_message(opensearch_client, channel_id, None, None).await+ }+ ChannelIndexAction::Ignore => Ok(()),+ }+}++pub(super) async fn process_channel_event(+ db: &PgPool,+ opensearch_client: &OpensearchClient,+ event: &ChannelMacroEvent,+ partition: i32,+ offset: i64,+) -> EventOutcome {+ let description = describe_channel_event(&event.event().event);+ if description.action == ChannelIndexAction::Ignore {+ tracing::trace!(+ channel_id = %description.channel_id,+ event_type = description.event_type,+ partition,+ offset,+ "ignoring channel event without a search-index action"+ );+ return EventOutcome::Ignored;+ }++ let result = retry_processing(|attempt| async move {+ tracing::trace!(+ channel_id = %description.channel_id,+ event_type = description.event_type,+ partition,+ offset,+ attempt,+ "processing channel search-index event"+ );+ process_channel_index_action(db, opensearch_client, description.action)+ .await+ .inspect_err(|error| {+ if attempt < MAX_PROCESSING_ATTEMPTS {+ let retry_delay =+ PROCESSING_RETRY_BASE_DELAY * 2u32.pow(attempt.saturating_sub(1));+ tracing::warn!(+ error = ?error,+ channel_id = %description.channel_id,+ event_type = description.event_type,+ partition,+ offset,+ attempt,+ delay_secs = retry_delay.as_secs(),+ "channel search-index processing failed, retrying"+ );+ }+ })+ })+ .await;++ match result {+ Ok(()) => EventOutcome::Indexed,+ Err(error) => {+ tracing::error!(+ error = ?error,+ channel_id = %description.channel_id,+ event_type = description.event_type,+ partition,+ offset,+ attempts = MAX_PROCESSING_ATTEMPTS,+ "dropping channel event after processing retries were exhausted"+ );+ EventOutcome::Dropped+ }+ }+}diff --git a/services/search_processing_service/src/inbound/kafka_consumer/project.rs b/services/search_processing_service/src/inbound/kafka_consumer/project.rsnew file mode 100644index 0000000000..2619764745--- /dev/null+++ b/services/search_processing_service/src/inbound/kafka_consumer/project.rs@@ -0,0 +1,198 @@+//! Maps project lifecycle events to search-index reconciliations and processes them.++use std::collections::HashSet;++use macro_event_broker::MacroEvent as _;+use opensearch_client::OpensearchClient;+use projects::domain::events::{ProjectMacroEvent, ProjectTopicEvent};+use sqlx::PgPool;+use sqs_client::search::project::UpsertProject;++use super::{EventOutcome, MAX_PROCESSING_ATTEMPTS, PROCESSING_RETRY_BASE_DELAY, retry_processing};+use crate::process::project::upsert_project;++#[derive(Debug, Clone, PartialEq, Eq)]+pub(super) enum ProjectIndexAction {+ Reconcile { project_ids: Vec<String> },+}++#[derive(Debug, Clone, PartialEq, Eq)]+pub(super) struct ProjectEventDescription<'a> {+ pub(super) action: ProjectIndexAction,+ pub(super) project_id: &'a str,+ pub(super) event_type: &'static str,+}++pub(super) fn collect_project_ids<'a>(+ project_ids: impl IntoIterator<Item = Option<&'a str>>,+) -> Vec<String> {+ let mut seen = HashSet::new();+ let mut collected = Vec::new();++ for project_id in project_ids.into_iter().flatten() {+ if !project_id.is_empty() && seen.insert(project_id) {+ collected.push(project_id.to_string());+ }+ }++ collected+}++pub(super) fn describe_project_event(event: &ProjectTopicEvent) -> ProjectEventDescription<'_> {+ match event {+ ProjectTopicEvent::Created(metadata) => ProjectEventDescription {+ action: ProjectIndexAction::Reconcile {+ project_ids: collect_project_ids([+ Some(metadata.project_id.as_str()),+ metadata.parent_project_id.as_deref(),+ ]),+ },+ project_id: &metadata.project_id,+ event_type: "project.created",+ },+ ProjectTopicEvent::Updated(metadata) => ProjectEventDescription {+ action: ProjectIndexAction::Reconcile {+ project_ids: collect_project_ids([+ Some(metadata.project_id.as_str()),+ metadata.previous_parent_id.as_deref(),+ metadata.parent_id.as_deref(),+ ]),+ },+ project_id: &metadata.project_id,+ event_type: "project.updated",+ },+ ProjectTopicEvent::Deleted(metadata) => ProjectEventDescription {+ action: ProjectIndexAction::Reconcile {+ project_ids: collect_project_ids(+ metadata+ .deleted_project_ids+ .iter()+ .map(|project_id| Some(project_id.as_str()))+ .chain([metadata.parent_project_id.as_deref()]),+ ),+ },+ project_id: &metadata.project_id,+ event_type: "project.deleted",+ },+ ProjectTopicEvent::Restored(metadata) => ProjectEventDescription {+ action: ProjectIndexAction::Reconcile {+ project_ids: collect_project_ids(+ metadata+ .restored_project_ids+ .iter()+ .map(|project_id| Some(project_id.as_str()))+ .chain([metadata.parent_project_id.as_deref()]),+ ),+ },+ project_id: &metadata.project_id,+ event_type: "project.restored",+ },+ ProjectTopicEvent::PermanentlyDeleted(metadata) => ProjectEventDescription {+ action: ProjectIndexAction::Reconcile {+ project_ids: collect_project_ids(+ metadata+ .purged_project_ids+ .iter()+ .map(|project_id| Some(project_id.as_str()))+ .chain([metadata.parent_project_id.as_deref()]),+ ),+ },+ project_id: &metadata.project_id,+ event_type: "project.permanently_deleted",+ },+ ProjectTopicEvent::Uploaded(metadata) => ProjectEventDescription {+ action: ProjectIndexAction::Reconcile {+ project_ids: collect_project_ids(+ metadata+ .project_ids+ .iter()+ .map(|project_id| Some(project_id.as_str()))+ .chain([metadata.parent_project_id.as_deref()]),+ ),+ },+ project_id: &metadata.root_project_id,+ event_type: "project.uploaded",+ },+ }+}++async fn process_project_index_action(+ db: &PgPool,+ opensearch_client: &OpensearchClient,+ action: ProjectIndexAction,+) -> anyhow::Result<()> {+ match action {+ ProjectIndexAction::Reconcile { project_ids } => {+ for project_id in project_ids {+ upsert_project(+ opensearch_client,+ db,+ &UpsertProject {+ project_id,+ index_override: None,+ },+ )+ .await?;+ }+ Ok(())+ }+ }+}++pub(super) async fn process_project_event(+ db: &PgPool,+ opensearch_client: &OpensearchClient,+ event: &ProjectMacroEvent,+ partition: i32,+ offset: i64,+) -> EventOutcome {+ let description = describe_project_event(&event.event().event);+ let result = retry_processing(|attempt| {+ let action = description.action.clone();+ async move {+ tracing::trace!(+ project_id = description.project_id,+ event_type = description.event_type,+ partition,+ offset,+ attempt,+ "processing project search-index event"+ );+ process_project_index_action(db, opensearch_client, action)+ .await+ .inspect_err(|error| {+ if attempt < MAX_PROCESSING_ATTEMPTS {+ let retry_delay =+ PROCESSING_RETRY_BASE_DELAY * 2u32.pow(attempt.saturating_sub(1));+ tracing::warn!(+ error = ?error,+ project_id = description.project_id,+ event_type = description.event_type,+ partition,+ offset,+ attempt,+ delay_secs = retry_delay.as_secs(),+ "project search-index processing failed, retrying"+ );+ }+ })+ }+ })+ .await;++ match result {+ Ok(()) => EventOutcome::Indexed,+ Err(error) => {+ tracing::error!(+ error = ?error,+ project_id = description.project_id,+ event_type = description.event_type,+ partition,+ offset,+ attempts = MAX_PROCESSING_ATTEMPTS,+ "dropping project event after processing retries were exhausted"+ );+ EventOutcome::Dropped+ }+ }+}diff --git a/services/search_processing_service/src/inbound/kafka_consumer/test.rs b/services/search_processing_service/src/inbound/kafka_consumer/test.rsindex 421e5a0dae..dd55691cae 100644--- a/services/search_processing_service/src/inbound/kafka_consumer/test.rs+++ b/services/search_processing_service/src/inbound/kafka_consumer/test.rs@@ -1,10 +1,10 @@use std::sync::Arc;use std::sync::atomic::{AtomicU32, Ordering};-use call::domain::events::{+use ::call::domain::events::{CallArchiveReason, CallRecordArchivedMetadata, CallRecordDeletedMetadata,CallRecordSummarizedMetadata, CallRecordUpdatedMetadata, CallRecordingReadyMetadata,- CallStartedMetadata,+ CallStartedMetadata, CallTopicEvent,};use channels::domain::{broker_events::{@@ -18,14 +18,30 @@ use channels::domain::{};use chrono::Utc;use macro_event_broker::{Event, EventBrokerError, MacroEvent as _, MessageParts};-use macro_event_topics::{MacroCallsTopic, MacroChannelsTopic, Topic as _};+use macro_event_topics::{MacroCallsTopic, MacroChannelsTopic, MacroProjectsTopic, Topic as _};use macro_user_id::user_id::MacroUserIdStr;+use projects::domain::events::{+ ProjectCreatedMetadata, ProjectDeletedMetadata, ProjectPermanentlyDeletedMetadata,+ ProjectRestoredMetadata, ProjectTopicEvent, ProjectUpdatedMetadata, ProjectUploadedMetadata,+};+use uuid::Uuid;-use super::*;+use super::{+ call::{CallEventDescription, CallIndexAction, describe_call_event},+ channel::{ChannelEventDescription, ChannelIndexAction, describe_channel_event},+ project::{+ ProjectEventDescription, ProjectIndexAction, collect_project_ids, describe_project_event,+ },+ *,+};const CALL_ID: Uuid = Uuid::from_u128(1);const CHANNEL_ID: Uuid = Uuid::from_u128(2);const MESSAGE_ID: Uuid = Uuid::from_u128(3);+const PROJECT_ID: &str = "project-root";+const CHILD_PROJECT_ID: &str = "project-child";+const PARENT_PROJECT_ID: &str = "project-parent";+const NEW_PARENT_PROJECT_ID: &str = "project-new-parent";struct TestMessage {topic: &'static str,@@ -114,7 +130,7 @@ fn recording_ready_event() -> CallTopicEvent {fn encoded_message<E: serde::Serialize>(topic: &'static str,- key: Uuid,+ key: impl ToString,event: Event<E>,) -> TestMessage {TestMessage {@@ -287,15 +303,168 @@ fn channel_event_cases() -> Vec<(ChannelTopicEvent, ChannelEventDescription)> {]}+fn project_event_cases() -> Vec<(ProjectTopicEvent, ProjectEventDescription<'static>)> {+ let owner = user_id();++ vec![+ (+ ProjectTopicEvent::Created(ProjectCreatedMetadata {+ project_id: PROJECT_ID.to_string(),+ owner: owner.clone(),+ name: "Project".to_string(),+ parent_project_id: Some(PARENT_PROJECT_ID.to_string()),+ created_at: Some(Utc::now()),+ }),+ ProjectEventDescription {+ action: ProjectIndexAction::Reconcile {+ project_ids: vec![PROJECT_ID.to_string(), PARENT_PROJECT_ID.to_string()],+ },+ project_id: PROJECT_ID,+ event_type: "project.created",+ },+ ),+ (+ ProjectTopicEvent::Updated(ProjectUpdatedMetadata {+ project_id: PROJECT_ID.to_string(),+ owner: owner.clone(),+ actor_user_id: Some(user_id()),+ name: Some("Renamed project".to_string()),+ previous_parent_id: Some(PARENT_PROJECT_ID.to_string()),+ parent_id: Some(NEW_PARENT_PROJECT_ID.to_string()),+ share_permission_updated: false,+ }),+ ProjectEventDescription {+ action: ProjectIndexAction::Reconcile {+ project_ids: vec![+ PROJECT_ID.to_string(),+ PARENT_PROJECT_ID.to_string(),+ NEW_PARENT_PROJECT_ID.to_string(),+ ],+ },+ project_id: PROJECT_ID,+ event_type: "project.updated",+ },+ ),+ (+ ProjectTopicEvent::Updated(ProjectUpdatedMetadata {+ project_id: PROJECT_ID.to_string(),+ owner: owner.clone(),+ actor_user_id: Some(user_id()),+ name: None,+ previous_parent_id: Some(PARENT_PROJECT_ID.to_string()),+ parent_id: Some(String::new()),+ share_permission_updated: true,+ }),+ ProjectEventDescription {+ action: ProjectIndexAction::Reconcile {+ project_ids: vec![PROJECT_ID.to_string(), PARENT_PROJECT_ID.to_string()],+ },+ project_id: PROJECT_ID,+ event_type: "project.updated",+ },+ ),+ (+ ProjectTopicEvent::Deleted(ProjectDeletedMetadata {+ project_id: PROJECT_ID.to_string(),+ owner: owner.clone(),+ actor_user_id: Some(user_id()),+ parent_project_id: Some(PARENT_PROJECT_ID.to_string()),+ deleted_project_ids: vec![+ PROJECT_ID.to_string(),+ CHILD_PROJECT_ID.to_string(),+ CHILD_PROJECT_ID.to_string(),+ PARENT_PROJECT_ID.to_string(),+ ],+ deleted_document_ids: vec!["document-id".to_string()],+ deleted_chat_ids: vec!["chat-id".to_string()],+ }),+ ProjectEventDescription {+ action: ProjectIndexAction::Reconcile {+ project_ids: vec![+ PROJECT_ID.to_string(),+ CHILD_PROJECT_ID.to_string(),+ PARENT_PROJECT_ID.to_string(),+ ],+ },+ project_id: PROJECT_ID,+ event_type: "project.deleted",+ },+ ),+ (+ ProjectTopicEvent::Restored(ProjectRestoredMetadata {+ project_id: PROJECT_ID.to_string(),+ owner: owner.clone(),+ actor_user_id: Some(user_id()),+ parent_project_id: Some(PARENT_PROJECT_ID.to_string()),+ restored_project_ids: vec![PROJECT_ID.to_string(), CHILD_PROJECT_ID.to_string()],+ }),+ ProjectEventDescription {+ action: ProjectIndexAction::Reconcile {+ project_ids: vec![+ PROJECT_ID.to_string(),+ CHILD_PROJECT_ID.to_string(),+ PARENT_PROJECT_ID.to_string(),+ ],+ },+ project_id: PROJECT_ID,+ event_type: "project.restored",+ },+ ),+ (+ ProjectTopicEvent::PermanentlyDeleted(ProjectPermanentlyDeletedMetadata {+ project_id: PROJECT_ID.to_string(),+ owner: owner.clone(),+ actor_user_id: Some(user_id()),+ parent_project_id: Some(PARENT_PROJECT_ID.to_string()),+ purged_project_ids: vec![PROJECT_ID.to_string(), CHILD_PROJECT_ID.to_string()],+ purged_document_ids: vec!["document-id".to_string()],+ purged_chat_ids: vec!["chat-id".to_string()],+ }),+ ProjectEventDescription {+ action: ProjectIndexAction::Reconcile {+ project_ids: vec![+ PROJECT_ID.to_string(),+ CHILD_PROJECT_ID.to_string(),+ PARENT_PROJECT_ID.to_string(),+ ],+ },+ project_id: PROJECT_ID,+ event_type: "project.permanently_deleted",+ },+ ),+ (+ ProjectTopicEvent::Uploaded(ProjectUploadedMetadata {+ root_project_id: PROJECT_ID.to_string(),+ owner,+ name: "Uploaded project".to_string(),+ parent_project_id: Some(PARENT_PROJECT_ID.to_string()),+ project_ids: vec![PROJECT_ID.to_string(), CHILD_PROJECT_ID.to_string()],+ }),+ ProjectEventDescription {+ action: ProjectIndexAction::Reconcile {+ project_ids: vec![+ PROJECT_ID.to_string(),+ CHILD_PROJECT_ID.to_string(),+ PARENT_PROJECT_ID.to_string(),+ ],+ },+ project_id: PROJECT_ID,+ event_type: "project.uploaded",+ },+ ),+ ]+}+#[test]fn subscribes_to_declared_search_processing_topics_with_durable_group() {assert_eq!(SearchProcessingConsumerGroup::GROUP_NAME,"search-processing-service");- let topics = SearchProcessingBrokerEvent::topics();+ let topics = DeclaredMacroEvent::topics();assert!(topics.contains(&MacroCallsTopic::TOPIC_STR));assert!(topics.contains(&MacroChannelsTopic::TOPIC_STR));+ assert!(topics.contains(&MacroProjectsTopic::TOPIC_STR));}#[test]@@ -365,6 +534,32 @@ fn maps_all_channel_lifecycle_events_to_index_actions() {}}+#[test]+fn maps_all_project_lifecycle_events_to_reconciliation_actions() {+ let cases = project_event_cases();+ assert_eq!(cases.len(), 7);++ for (event, expected) in cases {+ let serialized = serde_json::to_value(&event).expect("serializable project event");+ assert_eq!(serialized["event_type"], expected.event_type);+ assert_eq!(describe_project_event(&event), expected);+ }+}++#[test]+fn project_id_collection_is_stable_and_drops_missing_or_empty_ids() {+ assert_eq!(+ collect_project_ids([+ Some(PROJECT_ID),+ None,+ Some(""),+ Some(CHILD_PROJECT_ID),+ Some(PROJECT_ID),+ ]),+ vec![PROJECT_ID.to_string(), CHILD_PROJECT_ID.to_string()]+ );+}+#[test]fn channel_envelope_decodes_round_trip() {let event = ChannelTopicEvent::Deleted(ChannelDeletedMetadata {@@ -377,14 +572,72 @@ fn channel_envelope_decodes_round_trip() {Event::new(event.clone()),);- let decoded = SearchProcessingBrokerEvent::decode(&message).expect("decodable channel event");- let SearchProcessingBrokerEvent::ChannelMacroEvent(decoded_event) = decoded else {+ let decoded = DeclaredMacroEvent::decode(&message).expect("decodable channel event");+ let DeclaredMacroEvent::ChannelMacroEvent(decoded_event) = decoded else {panic!("expected channel event");};assert_eq!(decoded_event.key(), CHANNEL_ID.to_string());assert_eq!(decoded_event.event().event, event);}+#[test]+fn project_envelope_decodes_round_trip_with_string_key() {+ let event = ProjectTopicEvent::Restored(ProjectRestoredMetadata {+ project_id: PROJECT_ID.to_string(),+ owner: user_id(),+ actor_user_id: Some(user_id()),+ parent_project_id: Some(PARENT_PROJECT_ID.to_string()),+ restored_project_ids: vec![PROJECT_ID.to_string(), CHILD_PROJECT_ID.to_string()],+ });+ let message = encoded_message(+ MacroProjectsTopic::TOPIC_STR,+ PROJECT_ID,+ Event::new(event.clone()),+ );++ let decoded = DeclaredMacroEvent::decode(&message).expect("decodable project event");+ let DeclaredMacroEvent::ProjectMacroEvent(decoded_event) = decoded else {+ panic!("expected project event");+ };+ assert_eq!(decoded_event.key(), PROJECT_ID);+ assert_eq!(decoded_event.event().event, event);+}++#[tokio::test]+async fn unsupported_project_schema_message_is_commit_safe() {+ let event = ProjectTopicEvent::Restored(ProjectRestoredMetadata {+ project_id: PROJECT_ID.to_string(),+ owner: user_id(),+ actor_user_id: Some(user_id()),+ parent_project_id: None,+ restored_project_ids: vec![PROJECT_ID.to_string()],+ });+ let message = encoded_message(+ MacroProjectsTopic::TOPIC_STR,+ PROJECT_ID,+ Event::with_schema_version(event, 2),+ );+ let decoded = attach_event_coordinates(DeclaredMacroEvent::decode(&message), 4, 30);+ let (sender, mut receiver) = mpsc::channel(1);++ match handoff_decoded(&sender, decoded).await {+ HandoffOutcome::MalformedRecord(EventBrokerError::UnsupportedSchemaVersion {+ topic,+ expected,+ actual,+ }) => {+ assert_eq!(topic, MacroProjectsTopic::TOPIC_STR);+ assert_eq!(expected, 1);+ assert_eq!(actual, 2);+ }+ outcome => panic!("expected malformed project record, got {outcome:?}"),+ }+ assert!(matches!(+ receiver.try_recv(),+ Err(mpsc::error::TryRecvError::Empty)+ ));+}+#[tokio::test]async fn unsupported_channel_schema_message_is_commit_safe() {let event = ChannelTopicEvent::Deleted(ChannelDeletedMetadata {@@ -396,7 +649,7 @@ async fn unsupported_channel_schema_message_is_commit_safe() {CHANNEL_ID,Event::with_schema_version(event, 2),);- let decoded = attach_event_coordinates(SearchProcessingBrokerEvent::decode(&message), 2, 20);+ let decoded = attach_event_coordinates(DeclaredMacroEvent::decode(&message), 2, 20);let (sender, mut receiver) = mpsc::channel(1);match handoff_decoded(&sender, decoded).await {@@ -424,8 +677,7 @@ async fn malformed_missing_key_and_unsupported_schema_messages_are_commit_safe()key: Some(CALL_ID.to_string()),payload: Some(b"not json".to_vec()),};- let malformed =- attach_event_coordinates(SearchProcessingBrokerEvent::decode(&malformed), 1, 10);+ let malformed = attach_event_coordinates(DeclaredMacroEvent::decode(&malformed), 1, 10);assert!(matches!(malformed, Err(EventBrokerError::Serialization(_))));let missing_key = TestMessage {@@ -438,8 +690,7 @@ async fn malformed_missing_key_and_unsupported_schema_messages_are_commit_safe()).payload,};- let missing_key =- attach_event_coordinates(SearchProcessingBrokerEvent::decode(&missing_key), 1, 11);+ let missing_key = attach_event_coordinates(DeclaredMacroEvent::decode(&missing_key), 1, 11);assert!(matches!(missing_key,Err(EventBrokerError::MissingMessageKey)@@ -450,8 +701,7 @@ async fn malformed_missing_key_and_unsupported_schema_messages_are_commit_safe()CALL_ID,Event::with_schema_version(archived_event(), 2),);- let unsupported =- attach_event_coordinates(SearchProcessingBrokerEvent::decode(&unsupported), 1, 12);+ let unsupported = attach_event_coordinates(DeclaredMacroEvent::decode(&unsupported), 1, 12);assert!(matches!(unsupported,Err(EventBrokerError::UnsupportedSchemaVersion {@@ -482,7 +732,7 @@ async fn successful_handoff_carries_event_partition_and_offset() {CALL_ID,Event::new(event.clone()),);- let decoded = attach_event_coordinates(SearchProcessingBrokerEvent::decode(&message), 3, 42);+ let decoded = attach_event_coordinates(DeclaredMacroEvent::decode(&message), 3, 42);let (sender, mut receiver) = mpsc::channel(1);assert!(matches!(@@ -493,7 +743,7 @@ async fn successful_handoff_carries_event_partition_and_offset() {let received = receiver.recv().await.expect("handed-off event");assert_eq!(received.partition, 3);assert_eq!(received.offset, 42);- let SearchProcessingBrokerEvent::CallMacroEvent(received_event) = received.event else {+ let DeclaredMacroEvent::CallMacroEvent(received_event) = received.event else {panic!("expected call event");};assert_eq!(received_event.event().event, event);@@ -506,7 +756,7 @@ async fn closed_worker_channel_leaves_the_current_message_uncommitted() {CALL_ID,Event::new(archived_event()),);- let decoded = attach_event_coordinates(SearchProcessingBrokerEvent::decode(&message), 3, 42);+ let decoded = attach_event_coordinates(DeclaredMacroEvent::decode(&message), 3, 42);let (sender, receiver) = mpsc::channel(1);drop(receiver);diff --git a/services/search_processing_service/src/outbound/source.rs b/services/search_processing_service/src/outbound/source.rsindex 50d960ef91..694906bbae 100644--- a/services/search_processing_service/src/outbound/source.rs+++ b/services/search_processing_service/src/outbound/source.rs@@ -396,6 +396,8 @@ impl BackfillSource for PgBackfillSource {})});let rows_consumed = batch.len();+ // This SQS message is intentionally limited to backfills, which may+ // target an alternate OpenSearch index through `index_override`.let messages: Vec<SearchQueueMessage> = batch.into_iter().map(|p| {diff --git a/services/search_processing_service/src/process/mod.rs b/services/search_processing_service/src/process/mod.rsindex 6702c24158..7a21f79c7e 100644--- a/services/search_processing_service/src/process/mod.rs+++ b/services/search_processing_service/src/process/mod.rs@@ -4,7 +4,7 @@ mod chat;pub mod context;mod document;mod email;-mod project;+pub(crate) mod project;mod properties;mod user;pub mod worker;@@ -153,9 +153,6 @@ pub async fn process_message(SearchQueueMessage::UpsertProject(message) => {project::upsert_project(&ctx.opensearch_client, &ctx.db, &message).await?;}- SearchQueueMessage::RemoveProject(message) => {- project::remove_project(&ctx.opensearch_client, &message).await?;- }}ctx.worker.cleanup_message(message).await?;diff --git a/services/search_processing_service/src/process/project.rs b/services/search_processing_service/src/process/project.rsindex 8bc0e23a34..28ed839d17 100644--- a/services/search_processing_service/src/process/project.rs+++ b/services/search_processing_service/src/process/project.rs@@ -6,7 +6,7 @@ use opensearch_client::{upsert::{project::UpsertProjectArgs, properties::IndexedProperty},};use properties::outbound::entity_properties_get_query::get_entity_properties_for_index;-use sqs_client::search::project::{RemoveProject, UpsertProject};+use sqs_client::search::project::UpsertProject;/// Fetch the project's indexed properties, flattened for the search index.async fn fetch_indexed_properties(@@ -86,19 +86,3 @@ pub async fn upsert_project(Ok(())}--/// Handles the removal of a project from the opensearch index-#[tracing::instrument(skip(opensearch_client), err)]-pub async fn remove_project(- opensearch_client: &OpensearchClient,- message: &RemoveProject,-) -> anyhow::Result<()> {- opensearch_client- .delete_project(- message.project_id.as_str(),- message.index_override.as_deref(),- )- .await?;-- Ok(())-}diff --git a/tooling/just/rust.just b/tooling/just/rust.justindex 3648b4bbce..7c0df453a2 100644--- a/tooling/just/rust.just+++ b/tooling/just/rust.just@@ -97,7 +97,6 @@ setup_test_envs:echo "DATABASE_URL={{ DATABASE_URL }}" >> ./crates/projects/.envecho "DATABASE_URL={{ DATABASE_URL }}" >> ./crates/foreign_entity/.envecho "DATABASE_URL={{ DATABASE_URL }}" >> ./crates/stream/.env- echo "DATABASE_URL={{ DATABASE_URL }}" >> ./crates/macro_project_utils/.envecho "DATABASE_URL={{ DATABASE_URL }}" >> ./crates/entity_access/.envecho "DATABASE_URL={{ DATABASE_URL }}" >> ./crates/github/.envecho "DATABASE_URL={{ DATABASE_URL }}" >> ./crates/referral/.env