Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(search_processing_service): projects kafka (#5295)

Will Hutchinsonabf9de570bb1
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.json
index 4a5b2a25ae..09e2587f83 100644
--- a/.github/workspace-dep-closures.json
+++ b/.github/workspace-dep-closures.json
@@ -1307,6 +1307,7 @@
"crates/cowlike",
"crates/crm",
"crates/document_sub_type",
+ "crates/dynamodb_client",
"crates/email",
"crates/email_db_client",
"crates/email_utils",
@@ -1331,6 +1332,7 @@
"crates/macro_event_topics",
"crates/macro_queues",
"crates/macro_service_urls",
+ "crates/macro_sha_count_client",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -1352,6 +1354,7 @@
"crates/models_team",
"crates/non_empty",
"crates/notification",
+ "crates/projects",
"crates/prompt",
"crates/properties",
"crates/rate_limit",
@@ -4929,6 +4932,7 @@
"crates/cowlike",
"crates/crm",
"crates/document_sub_type",
+ "crates/dynamodb_client",
"crates/email",
"crates/email_db_client",
"crates/email_utils",
@@ -4960,6 +4964,7 @@
"crates/macro_middleware",
"crates/macro_queues",
"crates/macro_service_urls",
+ "crates/macro_sha_count_client",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -4986,6 +4991,7 @@
"crates/notification",
"crates/opensearch_client",
"crates/opensearch_query_builder",
+ "crates/projects",
"crates/prompt",
"crates/properties",
"crates/rate_limit",
diff --git a/.sqlx/query-306950934a036e99471a1646c6f3b97a9ca9da460ba754db95f8772ab18e9f54.json b/.sqlx/query-306950934a036e99471a1646c6f3b97a9ca9da460ba754db95f8772ab18e9f54.json
new file mode 100644
index 0000000000..8ddafbf80f
--- /dev/null
+++ b/.sqlx/query-306950934a036e99471a1646c6f3b97a9ca9da460ba754db95f8772ab18e9f54.json
@@ -0,0 +1,28 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n SELECT p.id as project_id, p.\"userId\" as user_id\n FROM \"Project\" p\n WHERE p.\"deletedAt\" IS NOT NULL AND p.\"deletedAt\" <= $1\n ",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "project_id",
+ "type_info": "Text"
+ },
+ {
+ "ordinal": 1,
+ "name": "user_id",
+ "type_info": "Text"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Timestamp"
+ ]
+ },
+ "nullable": [
+ false,
+ false
+ ]
+ },
+ "hash": "306950934a036e99471a1646c6f3b97a9ca9da460ba754db95f8772ab18e9f54"
+}
diff --git a/.sqlx/query-9f595d4ae9aa9111c316efa5d4be2b8e4a6cc69374b90600457a32b14a91ae61.json b/.sqlx/query-9f595d4ae9aa9111c316efa5d4be2b8e4a6cc69374b90600457a32b14a91ae61.json
deleted file mode 100644
index 2620ca0739..0000000000
--- a/.sqlx/query-9f595d4ae9aa9111c316efa5d4be2b8e4a6cc69374b90600457a32b14a91ae61.json
+++ /dev/null
@@ -1,22 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "\n SELECT p.id\n FROM \"Project\" p\n WHERE p.\"deletedAt\" IS NOT NULL AND p.\"deletedAt\" <= $1\n ",
- "describe": {
- "columns": [
- {
- "ordinal": 0,
- "name": "id",
- "type_info": "Text"
- }
- ],
- "parameters": {
- "Left": [
- "Timestamp"
- ]
- },
- "nullable": [
- false
- ]
- },
- "hash": "9f595d4ae9aa9111c316efa5d4be2b8e4a6cc69374b90600457a32b14a91ae61"
-}
diff --git a/Cargo.lock b/Cargo.lock
index 267d784166..5fb66cbeda 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4003,9 +4003,12 @@ dependencies = [
"macro_entrypoint",
"macro_env",
"macro_env_var",
+ "macro_event_broker",
"macro_queues",
+ "macro_user_id",
"macro_uuid",
"model",
+ "projects",
"serde",
"serde_json",
"sqlx",
@@ -12920,6 +12923,7 @@ dependencies = [
"nom 8.0.0",
"opensearch_client",
"pdfium-render",
+ "projects",
"properties",
"rdkafka",
"rootcause",
diff --git a/crates/macro_db_client/src/projects/get_projects.rs b/crates/macro_db_client/src/projects/get_projects.rs
index 7a4dddce09..a5a177e128 100644
--- a/crates/macro_db_client/src/projects/get_projects.rs
+++ b/crates/macro_db_client/src/projects/get_projects.rs
@@ -1,6 +1,9 @@
use chrono::{DateTime, Utc};
use sqlx::{Pool, Postgres};
+#[cfg(test)]
+mod test;
+
/// One row of the search-backfill scan: just enough to build an upsert
/// message and the keyset cursor.
pub struct ProjectSearchBackfillRow {
@@ -73,21 +76,30 @@ pub async fn get_sub_project_ids(
Ok(result)
}
-/// Gets all projects that are older than the provided date
+/// A deleted project and the user who owns it.
+#[derive(Debug, Eq, PartialEq)]
+pub struct ProjectToDelete {
+ /// The deleted project's ID.
+ pub project_id: String,
+ /// The owning user's ID.
+ pub user_id: String,
+}
+
+/// Gets all projects and their owners that are older than the provided date.
#[tracing::instrument(skip(db))]
pub async fn get_projects_to_delete(
db: &Pool<Postgres>,
date: &chrono::NaiveDateTime,
-) -> anyhow::Result<Vec<String>> {
- let result = sqlx::query!(
+) -> anyhow::Result<Vec<ProjectToDelete>> {
+ let result = sqlx::query_as!(
+ ProjectToDelete,
r#"
- SELECT p.id
+ SELECT p.id as project_id, p."userId" as user_id
FROM "Project" p
WHERE p."deletedAt" IS NOT NULL AND p."deletedAt" <= $1
"#,
date
)
- .map(|row| row.id)
.fetch_all(db)
.await?;
diff --git a/crates/macro_db_client/src/projects/get_projects/test.rs b/crates/macro_db_client/src/projects/get_projects/test.rs
new file mode 100644
index 0000000000..9ad8711019
--- /dev/null
+++ b/crates/macro_db_client/src/projects/get_projects/test.rs
@@ -0,0 +1,29 @@
+use super::*;
+use chrono::NaiveDateTime;
+
+#[sqlx::test(fixtures(path = "../../../fixtures", scripts("recently_deleted")))]
+async fn returns_old_deleted_projects_with_owners(pool: Pool<Postgres>) -> anyhow::Result<()> {
+ let cutoff = NaiveDateTime::parse_from_str("2020-01-01 00:00:00", "%Y-%m-%d %H:%M:%S")?;
+ let mut projects = get_projects_to_delete(&pool, &cutoff).await?;
+ projects.sort_unstable_by(|left, right| left.project_id.cmp(&right.project_id));
+
+ assert_eq!(
+ projects,
+ vec![
+ ProjectToDelete {
+ project_id: "p1".to_owned(),
+ user_id: "macro|[email protected]".to_owned(),
+ },
+ ProjectToDelete {
+ project_id: "p2".to_owned(),
+ user_id: "macro|[email protected]".to_owned(),
+ },
+ ProjectToDelete {
+ project_id: "p3".to_owned(),
+ user_id: "macro|[email protected]".to_owned(),
+ },
+ ]
+ );
+
+ Ok(())
+}
diff --git a/crates/macro_db_client/src/projects/mod.rs b/crates/macro_db_client/src/projects/mod.rs
index dec5da97e6..68c0073d60 100644
--- a/crates/macro_db_client/src/projects/mod.rs
+++ b/crates/macro_db_client/src/projects/mod.rs
@@ -2,7 +2,7 @@ pub mod get_project;
mod get_projects;
pub use get_projects::{
- ProjectSearchBackfillRow, get_all_project_ids_with_users_paginated,
+ ProjectSearchBackfillRow, ProjectToDelete, get_all_project_ids_with_users_paginated,
get_projects_for_search_backfill, get_projects_to_delete, get_sub_project_ids,
};
pub mod delete;
diff --git a/crates/macro_project_utils/Cargo.toml b/crates/macro_project_utils/Cargo.toml
deleted file mode 100644
index 971ad5567d..0000000000
--- a/crates/macro_project_utils/Cargo.toml
+++ /dev/null
@@ -1,15 +0,0 @@
-[package]
-edition = "2024"
-name = "macro_project_utils"
-publish = false
-version = "0.1.0"
-
-[dependencies]
-anyhow = { workspace = true }
-sqlx = { workspace = true }
-sqs_client = { path = "../sqs_client", default-features = false, features = [
- "search",
-] }
-tokio = { workspace = true }
-tracing = { workspace = true }
-workspace-hack = { version = "0.1", path = "../workspace-hack" }
diff --git a/crates/macro_project_utils/justfile b/crates/macro_project_utils/justfile
deleted file mode 100644
index 1849fc42e3..0000000000
--- a/crates/macro_project_utils/justfile
+++ /dev/null
@@ -1,6 +0,0 @@
-import '../../tooling/just/database.just'
-
-mod sqlx '../../tooling/just/sqlx.just'
-
-prepare_db:
- just sqlx::prepare_db {{ DATABASE_URL }}
diff --git a/crates/macro_project_utils/src/lib.rs b/crates/macro_project_utils/src/lib.rs
deleted file mode 100644
index ddfa74c46a..0000000000
--- a/crates/macro_project_utils/src/lib.rs
+++ /dev/null
@@ -1,91 +0,0 @@
-use sqs_client::{
- SQS,
- search::{SearchQueueMessage, project::UpsertProject},
-};
-
-#[derive(Clone, Debug)]
-pub struct ProjectModifiedArgs<T>
-where
- T: Clone + std::fmt::Debug + ToString + std::marker::Send + Sync,
-{
- /// The new project id of the item or the project that was modified
- pub project_id: Option<T>,
- /// The old project id if the item was moved
- pub old_project_id: Option<T>,
- /// The user who performed the action
- pub user_id: String,
-}
-
-#[tracing::instrument(skip(db, sqs))]
-pub async fn update_project_modified<T>(
- db: &sqlx::Pool<sqlx::Postgres>,
- sqs: Option<&SQS>,
- project_modified_args: ProjectModifiedArgs<T>,
-) where
- T: Clone + std::fmt::Debug + ToString + std::marker::Send + Sync,
-{
- tracing::trace!("updating project modified date");
-
- let project_id = project_modified_args
- .project_id
- .as_ref()
- .map(|s| s.to_string());
- let old_project_id = project_modified_args
- .old_project_id
- .as_ref()
- .map(|s| s.to_string());
-
- tokio::spawn({
- let db = db.clone();
- let sqs = sqs.cloned();
- async move {
- if let Some(old_project_id) = old_project_id
- && !old_project_id.is_empty()
- {
- tracing::trace!(project_id=?old_project_id, "updating project modified date");
- let _ = update_project_modified_date(&db, sqs.as_ref(), &old_project_id).await.inspect_err(|e| {
- tracing::error!(error=?e, project_id=?old_project_id, "unable to update project modified date");
- });
- }
-
- if let Some(project_id) = project_id
- && !project_id.is_empty()
- {
- tracing::trace!(project_id=?project_id, "updating project modified date");
-
- let _ = update_project_modified_date(&db, sqs.as_ref(), &project_id).await.inspect_err(|e| {
- tracing::error!(error=?e, project_id=?project_id, "unable to update project modified date");
- });
- }
- }
- });
-}
-
-async fn update_project_modified_date(
- db: &sqlx::Pool<sqlx::Postgres>,
- sqs: Option<&SQS>,
- project_id: &str,
-) -> anyhow::Result<()> {
- sqlx::query!(
- r#"UPDATE "Project" SET "updatedAt" = NOW() WHERE id = $1"#,
- project_id,
- )
- .execute(db)
- .await?;
-
- // Refresh the project's search doc so ranking follows the bumped
- // updatedAt. Best-effort like the bump itself.
- if let Some(sqs) = sqs {
- let _ = sqs
- .send_message_to_search_event_queue(SearchQueueMessage::UpsertProject(UpsertProject {
- project_id: project_id.to_string(),
- index_override: None,
- }))
- .await
- .inspect_err(|e| {
- tracing::error!(error=?e, project_id=?project_id, "unable to enqueue project search upsert");
- });
- }
-
- Ok(())
-}
diff --git a/crates/projects/src/domain.rs b/crates/projects/src/domain.rs
index 4ed26cf913..9c75d16302 100644
--- a/crates/projects/src/domain.rs
+++ b/crates/projects/src/domain.rs
@@ -1,6 +1,7 @@
//! Domain models, ports, and project service definitions.
/// Unified entity-mutation capability impls.
+#[cfg(feature = "service")]
pub mod entity_mutation;
pub mod events;
pub mod models;
diff --git a/crates/projects/src/domain/ports.rs b/crates/projects/src/domain/ports.rs
index 5677ec672c..5ec4e2e3cb 100644
--- a/crates/projects/src/domain/ports.rs
+++ b/crates/projects/src/domain/ports.rs
@@ -206,20 +206,9 @@ pub trait ShaCounterPort: Send + Sync + 'static {
) -> impl Future<Output = anyhow::Result<()>> + Send;
}
-/// Port for publishing project search and document-deletion work.
+/// Port for publishing cross-family search and document-deletion work.
+// TODO: Remove this port and its SQS adapter after the chats and documents migrations.
pub trait ProjectSearchIndexer: Send + Sync + 'static {
- /// Enqueue projects for indexing.
- fn upsert_projects(
- &self,
- project_ids: Vec<String>,
- ) -> impl Future<Output = anyhow::Result<()>> + Send;
-
- /// Remove projects from the search index.
- fn remove_projects(
- &self,
- project_ids: Vec<String>,
- ) -> impl Future<Output = anyhow::Result<()>> + Send;
-
/// Remove chats from the search index.
fn remove_chats(
&self,
diff --git a/crates/projects/src/domain/service.rs b/crates/projects/src/domain/service.rs
index 4d82776eb6..e63707d81f 100644
--- a/crates/projects/src/domain/service.rs
+++ b/crates/projects/src/domain/service.rs
@@ -10,7 +10,7 @@ use entity_access_management::domain::ports::EntityAccessManagementService;
use futures::stream::{FuturesUnordered, StreamExt};
use macro_event_broker::MacroEventBroker;
use macro_user_id::user_id::MacroUserIdStr;
-use model::folder::{UploadFolderRequest, UploadFolderResponseData};
+use model::folder::{FileSystemNodeWithIds, UploadFolderRequest, UploadFolderResponseData};
use model::item::{Item, ItemWithUserAccessLevel};
use model::project::request::{CreateProjectRequest, PatchProjectRequestV2};
use model::project::response::GetProjectResponseData;
@@ -141,17 +141,6 @@ where
"unable to update project modified date"
);
});
- let _ = self
- .search_indexer
- .upsert_projects(vec![project_id.to_string()])
- .await
- .inspect_err(|error| {
- tracing::error!(
- error = ?error,
- project_id,
- "unable to enqueue project search upsert"
- );
- });
}
}
@@ -341,17 +330,6 @@ where
.project_parent_id
.map(|_| parse_internal_uuid(&project.id, "created project ID"))
.transpose()?;
- let _ = self
- .search_indexer
- .upsert_projects(vec![project.id.clone()])
- .await
- .inspect_err(|error| {
- tracing::error!(
- error = ?error,
- project_id = %project.id,
- "unable to enqueue project search upsert"
- );
- });
if let Some((parent_id, entity_id)) = args.project_parent_id.zip(entity_id) {
let _ = self
@@ -508,15 +486,6 @@ where
.await
.map_err(|error| internal_error(error, "unable to delete project"))?;
- if !deleted.project_ids.is_empty() {
- let _ = self
- .search_indexer
- .remove_projects(deleted.project_ids.clone())
- .await
- .inspect_err(|error| {
- tracing::error!(error = ?error, "unable to enqueue deleted projects for search");
- });
- }
if let Some(parent_id) = project.parent_id.as_deref() {
self.bump_project_modified(parent_id).await;
}
@@ -565,13 +534,6 @@ where
.map(|(document_id, _)| document_id.clone())
.collect::<Vec<_>>();
- if !purged.project_ids.is_empty() {
- let _ = self
- .search_indexer
- .remove_projects(purged.project_ids)
- .await
- .inspect_err(|error| tracing::error!(error = ?error, "unable to enqueue purged projects for search"));
- }
if !purged.chat_ids.is_empty() {
let _ = self
.search_indexer
@@ -624,16 +586,6 @@ where
.map_err(|error| internal_error(error, "unable to revert project"))?;
let result = restored.clone();
- if !restored.project_ids.is_empty() {
- let _ = self
- .search_indexer
- .upsert_projects(restored.project_ids.clone())
- .await
- .inspect_err(|error| {
- tracing::error!(error = ?error, "unable to enqueue restored projects for search");
- });
- }
-
let project_id = receipt.entity().entity_id.clone();
self.publish_project_event(&ProjectMacroEvent::restored(
project_id.clone(),
@@ -655,6 +607,9 @@ where
internal: bool,
args: UploadFolderRequest,
) -> Result<UploadFolderResponseData, ProjectError> {
+ let event_owner = actor.clone();
+ let event_name = args.root_folder_name.clone();
+ let event_parent_project_id = args.parent_id.clone();
let root_folder = build_root_folder(&args.root_folder_name, args.content)
.map_err(|error| internal_error(error, "unable to prepare folder upload"))?;
let uploaded = self
@@ -699,13 +654,26 @@ where
};
if !uploaded.project_ids.is_empty() {
- let _ = self
- .search_indexer
- .upsert_projects(uploaded.project_ids)
- .await
- .inspect_err(|error| {
- tracing::error!(error = ?error, "unable to enqueue uploaded projects for search");
- });
+ match &uploaded.file_system {
+ FileSystemNodeWithIds::Folder {
+ project_id: root_project_id,
+ ..
+ } => {
+ self.publish_project_event(&ProjectMacroEvent::uploaded(
+ root_project_id.clone(),
+ ProjectUploadedMetadata {
+ root_project_id: root_project_id.clone(),
+ owner: event_owner,
+ name: event_name,
+ parent_project_id: event_parent_project_id,
+ project_ids: uploaded.project_ids,
+ },
+ ));
+ }
+ FileSystemNodeWithIds::File { .. } => {
+ tracing::error!("unable to publish folder upload event: root node is a file");
+ }
+ }
}
Ok(UploadFolderResponseData {
diff --git a/crates/projects/src/domain/service/tests.rs b/crates/projects/src/domain/service/tests.rs
index 6f4f3d04be..b986ddb57c 100644
--- a/crates/projects/src/domain/service/tests.rs
+++ b/crates/projects/src/domain/service/tests.rs
@@ -153,14 +153,6 @@ impl EntityAccessManagementService for NullPort {
}
impl ProjectSearchIndexer for NullPort {
- async fn upsert_projects(&self, _project_ids: Vec<String>) -> anyhow::Result<()> {
- unreachable!()
- }
-
- async fn remove_projects(&self, _project_ids: Vec<String>) -> anyhow::Result<()> {
- unreachable!()
- }
-
async fn remove_chats(&self, _chat_ids: Vec<String>) -> anyhow::Result<()> {
unreachable!()
}
@@ -226,34 +218,12 @@ impl EntityAccessManagementService for RecordingEam {
}
}
-#[derive(Clone, Debug, PartialEq, Eq)]
-enum IndexCall {
- Upsert(Vec<String>),
- Remove(Vec<String>),
-}
-
-#[derive(Clone, Default)]
+#[derive(Clone, Copy, Default)]
struct RecordingIndexer {
- calls: Arc<Mutex<Vec<IndexCall>>>,
+ _unused: (),
}
impl ProjectSearchIndexer for RecordingIndexer {
- async fn upsert_projects(&self, project_ids: Vec<String>) -> anyhow::Result<()> {
- self.calls
- .lock()
- .unwrap()
- .push(IndexCall::Upsert(project_ids));
- Ok(())
- }
-
- async fn remove_projects(&self, project_ids: Vec<String>) -> anyhow::Result<()> {
- self.calls
- .lock()
- .unwrap()
- .push(IndexCall::Remove(project_ids));
- Ok(())
- }
-
async fn remove_chats(&self, _chat_ids: Vec<String>) -> anyhow::Result<()> {
unreachable!()
}
@@ -705,9 +675,7 @@ async fn create_uses_grapheme_limit_and_orchestrates_parent_side_effects() {
.return_once(|_| Box::pin(async { Ok(()) }));
let eam = RecordingEam::default();
let eam_calls = eam.calls.clone();
- let indexer = RecordingIndexer::default();
- let index_calls = indexer.calls.clone();
- let service = mutation_service(repo, eam, indexer);
+ let service = mutation_service(repo, eam, RecordingIndexer::default());
service
.create_project(
@@ -735,13 +703,6 @@ async fn create_uses_grapheme_limit_and_orchestrates_parent_side_effects() {
*eam_calls.lock().unwrap(),
vec![EamCall::Add(project_id, parent_id)]
);
- assert_eq!(
- *index_calls.lock().unwrap(),
- vec![
- IndexCall::Upsert(vec![project_id.to_string()]),
- IndexCall::Upsert(vec![parent_id.to_string()]),
- ]
- );
}
#[tokio::test]
@@ -1168,7 +1129,7 @@ async fn edit_project_succeeds_when_event_publication_fails() {
}
#[tokio::test]
-async fn owner_edit_propagates_move_and_pairs_every_bump_with_an_upsert() {
+async fn owner_edit_propagates_move_and_updates_all_modified_timestamps() {
let project_id = Uuid::new_v4();
let old_parent_id = Uuid::new_v4();
let new_parent_id = Uuid::new_v4();
@@ -1200,9 +1161,7 @@ async fn owner_edit_propagates_move_and_pairs_every_bump_with_an_upsert() {
});
let eam = RecordingEam::default();
let eam_calls = eam.calls.clone();
- let indexer = RecordingIndexer::default();
- let index_calls = indexer.calls.clone();
- let service = mutation_service(repo, eam, indexer);
+ let service = mutation_service(repo, eam, RecordingIndexer::default());
service
.edit_project(
@@ -1221,17 +1180,14 @@ async fn owner_edit_propagates_move_and_pairs_every_bump_with_an_upsert() {
Some(new_parent_id)
)]
);
- let bumps = bumped.lock().unwrap().clone();
- let indexed = index_calls
- .lock()
- .unwrap()
- .iter()
- .filter_map(|call| match call {
- IndexCall::Upsert(ids) if ids.len() == 1 => Some(ids[0].clone()),
- _ => None,
- })
- .collect::<Vec<_>>();
- assert_eq!(bumps, indexed);
+ assert_eq!(
+ *bumped.lock().unwrap(),
+ vec![
+ project_id.to_string(),
+ old_parent_id.to_string(),
+ new_parent_id.to_string(),
+ ]
+ );
}
#[tokio::test]
@@ -1314,25 +1270,19 @@ async fn edit_name_limit_counts_unicode_graphemes() {
}
#[tokio::test]
-async fn failed_database_bump_still_runs_the_paired_index_upsert() {
+async fn failed_project_modified_update_is_non_fatal() {
let project_id = Uuid::new_v4();
let mut repo = MockProjectRepo::new();
repo.expect_update_project_modified()
+ .withf(move |id| id == project_id.to_string())
.return_once(|_| Box::pin(async { Err(anyhow::anyhow!("database unavailable")) }));
- let indexer = RecordingIndexer::default();
- let calls = indexer.calls.clone();
- let service = mutation_service(repo, RecordingEam::default(), indexer);
+ let service = mutation_service(repo, RecordingEam::default(), RecordingIndexer::default());
service.bump_project_modified(&project_id.to_string()).await;
-
- assert_eq!(
- *calls.lock().unwrap(),
- vec![IndexCall::Upsert(vec![project_id.to_string()])]
- );
}
#[tokio::test]
-async fn soft_delete_removes_index_entries_and_bumps_parent_with_upsert() {
+async fn soft_delete_updates_parent_timestamp_and_publishes_event() {
let project_id = Uuid::new_v4();
let parent_id = Uuid::new_v4();
let deleted_ids = vec![project_id.to_string(), Uuid::new_v4().to_string()];
@@ -1350,12 +1300,14 @@ async fn soft_delete_removes_index_entries_and_bumps_parent_with_upsert() {
repo.expect_update_project_modified()
.withf(move |id| id == parent_id.to_string())
.return_once(|_| Box::pin(async { Ok(()) }));
- let indexer = RecordingIndexer::default();
- let calls = indexer.calls.clone();
let event_broker = TestEventBroker::default();
let published = event_broker.published();
- let service =
- mutation_service_with_event_broker(repo, RecordingEam::default(), indexer, event_broker);
+ let service = mutation_service_with_event_broker(
+ repo,
+ RecordingEam::default(),
+ RecordingIndexer::default(),
+ event_broker,
+ );
service
.soft_delete_project(
@@ -1366,13 +1318,6 @@ async fn soft_delete_removes_index_entries_and_bumps_parent_with_upsert() {
.await
.unwrap();
- assert_eq!(
- *calls.lock().unwrap(),
- vec![
- IndexCall::Remove(expected_deleted_ids.clone()),
- IndexCall::Upsert(vec![parent_id.to_string()]),
- ]
- );
let published = published.lock().unwrap();
assert_eq!(published.len(), 1);
assert_project_event(&published[0], project_id, "project.deleted");
@@ -1393,7 +1338,7 @@ async fn soft_delete_removes_index_entries_and_bumps_parent_with_upsert() {
}
#[tokio::test]
-async fn revert_upserts_every_restored_project() {
+async fn revert_publishes_every_restored_project_in_event() {
let project_id = Uuid::new_v4();
let parent_id = Uuid::new_v4();
let restored_ids = vec![project_id.to_string(), Uuid::new_v4().to_string()];
@@ -1413,12 +1358,14 @@ async fn revert_upserts_every_restored_project() {
})
})
});
- let indexer = RecordingIndexer::default();
- let calls = indexer.calls.clone();
let event_broker = TestEventBroker::default();
let published = event_broker.published();
- let service =
- mutation_service_with_event_broker(repo, RecordingEam::default(), indexer, event_broker);
+ let service = mutation_service_with_event_broker(
+ repo,
+ RecordingEam::default(),
+ RecordingIndexer::default(),
+ event_broker,
+ );
service
.revert_delete_project(
@@ -1432,10 +1379,6 @@ async fn revert_upserts_every_restored_project() {
.await
.unwrap();
- assert_eq!(
- *calls.lock().unwrap(),
- vec![IndexCall::Upsert(expected_ids.clone())]
- );
let published = published.lock().unwrap();
assert_eq!(published.len(), 1);
assert_project_event(&published[0], project_id, "project.restored");
@@ -1477,9 +1420,7 @@ async fn restore_entity_returns_document_and_chat_effects() {
})
})
});
- let indexer = RecordingIndexer::default();
- let index_calls = indexer.calls.clone();
- let service = mutation_service(repo, RecordingEam::default(), indexer);
+ let service = mutation_service(repo, RecordingEam::default(), RecordingIndexer::default());
let entity = EntityType::Project.with_entity_string(project_id.to_string());
let effects = service
@@ -1508,10 +1449,6 @@ async fn restore_entity_returns_document_and_chat_effects() {
),
]
);
- assert_eq!(
- *index_calls.lock().unwrap(),
- vec![IndexCall::Upsert(restored_project_ids)]
- );
}
#[tokio::test]
@@ -1637,14 +1574,6 @@ impl OrderedIndexer {
}
impl ProjectSearchIndexer for OrderedIndexer {
- async fn upsert_projects(&self, _project_ids: Vec<String>) -> anyhow::Result<()> {
- self.record("upsert")
- }
-
- async fn remove_projects(&self, _project_ids: Vec<String>) -> anyhow::Result<()> {
- self.record("projects")
- }
-
async fn remove_chats(&self, _chat_ids: Vec<String>) -> anyhow::Result<()> {
self.record("chats")
}
@@ -1811,14 +1740,7 @@ async fn permanent_delete_runs_external_work_after_committed_purge() {
assert_eq!(
*events.lock().unwrap(),
- vec![
- "repo",
- "sha",
- "projects",
- "chats",
- "documents",
- "document_deletes"
- ]
+ vec!["repo", "sha", "chats", "documents", "document_deletes"]
);
let published = published.lock().unwrap();
assert_eq!(published.len(), 1);
@@ -2070,6 +1992,123 @@ async fn mark_projects_uploaded_publication_failure_is_non_fatal() {
assert_eq!(result, vec!["root-project", "child-project"]);
}
+#[tokio::test]
+async fn upload_folder_publishes_uploaded_event() {
+ let project_ids = vec!["root-project".to_string(), "child-project".to_string()];
+ let expected_project_ids = project_ids.clone();
+ let mut repo = MockProjectRepo::new();
+ repo.expect_upload_folder()
+ .withf(|args| {
+ args.user_id.as_ref() == "macro|[email protected]"
+ && args.root_folder_name == "Uploaded tree"
+ && args.upload_request_id == "request"
+ && args.parent_id.as_deref() == Some("parent-project")
+ })
+ .return_once(move |_| {
+ Box::pin(async move {
+ Ok(UploadFolderWithIdsResponse {
+ file_system: FileSystemNodeWithIds::Folder {
+ content: Default::default(),
+ project_id: "root-project".to_string(),
+ },
+ project_ids,
+ documents: vec![upload_document("document", FileType::Pdf)],
+ })
+ })
+ });
+ let upload_urls = RecordingUploadUrls {
+ calls: Arc::new(Mutex::new(Vec::new())),
+ fail_document: false,
+ };
+ let upload_url_calls = upload_urls.calls.clone();
+ let event_broker = TestEventBroker::default();
+ let published = event_broker.published();
+ let service = ProjectServiceImpl::new(
+ repo,
+ upload_urls,
+ RecordingBulkUpload::default(),
+ NullPort,
+ NullPort,
+ RecordingIndexer::default(),
+ None,
+ event_broker,
+ );
+
+ service
+ .upload_folder(
+ user_id("macro|[email protected]"),
+ false,
+ UploadFolderRequest {
+ content: vec![FolderItem {
+ name: "document".to_string(),
+ full_name: "document.pdf".to_string(),
+ file_type: Some(FileType::Pdf),
+ relative_path: "Uploaded tree".to_string(),
+ sha: "0123456789abcdef".to_string(),
+ }],
+ root_folder_name: "Uploaded tree".to_string(),
+ upload_request_id: "request".to_string(),
+ parent_id: Some("parent-project".to_string()),
+ },
+ )
+ .await
+ .unwrap();
+
+ assert_eq!(upload_url_calls.lock().unwrap().len(), 1);
+ let published = published.lock().unwrap();
+ assert_eq!(published.len(), 1);
+ assert_eq!(published[0].topic, "macro.projects");
+ assert_eq!(published[0].key, "root-project");
+ assert_eq!(published[0].payload["schema_version"], 1);
+ assert_eq!(published[0].payload["event_type"], "project.uploaded");
+ assert_eq!(
+ published[0].payload["metadata"],
+ serde_json::json!({
+ "root_project_id": "root-project",
+ "owner": "macro|[email protected]",
+ "name": "Uploaded tree",
+ "parent_project_id": "parent-project",
+ "project_ids": expected_project_ids,
+ })
+ );
+}
+
+#[tokio::test]
+async fn upload_folder_with_no_project_ids_publishes_no_event() {
+ let mut repo = MockProjectRepo::new();
+ repo.expect_upload_folder().return_once(|_| {
+ Box::pin(async {
+ Ok(UploadFolderWithIdsResponse {
+ file_system: FileSystemNodeWithIds::Folder {
+ content: Default::default(),
+ project_id: "root-project".to_string(),
+ },
+ project_ids: Vec::new(),
+ documents: Vec::new(),
+ })
+ })
+ });
+ let event_broker = TestEventBroker::default();
+ let published = event_broker.published();
+ let service = service_with_event_broker(repo, RecordingBulkUpload::default(), event_broker);
+
+ service
+ .upload_folder(
+ user_id("macro|[email protected]"),
+ false,
+ UploadFolderRequest {
+ content: Vec::new(),
+ root_folder_name: "Uploaded tree".to_string(),
+ upload_request_id: "request".to_string(),
+ parent_id: None,
+ },
+ )
+ .await
+ .unwrap();
+
+ assert!(published.lock().unwrap().is_empty());
+}
+
#[tokio::test]
async fn upload_folder_compensates_after_destination_failure() {
let mut repo = MockProjectRepo::new();
@@ -2088,6 +2127,8 @@ async fn upload_folder_compensates_after_destination_failure() {
repo.expect_delete_uploaded_tree()
.withf(|projects, documents| projects == ["project"] && documents == ["document"])
.return_once(|_, _| Box::pin(async { Ok(()) }));
+ let event_broker = TestEventBroker::default();
+ let published = event_broker.published();
let service = ProjectServiceImpl::new(
repo,
RecordingUploadUrls {
@@ -2099,7 +2140,7 @@ async fn upload_folder_compensates_after_destination_failure() {
NullPort,
NullPort,
None,
- TestEventBroker::default(),
+ event_broker,
);
let result = service
@@ -2122,6 +2163,7 @@ async fn upload_folder_compensates_after_destination_failure() {
.await;
assert!(matches!(result, Err(ProjectError::Internal(_))));
+ assert!(published.lock().unwrap().is_empty());
}
#[tokio::test]
diff --git a/crates/projects/src/outbound/sqs_search_indexer.rs b/crates/projects/src/outbound/sqs_search_indexer.rs
index eb89ae672f..447429bc2c 100644
--- a/crates/projects/src/outbound/sqs_search_indexer.rs
+++ b/crates/projects/src/outbound/sqs_search_indexer.rs
@@ -1,17 +1,12 @@
-//! SQS adapter for project search indexing and document deletion.
+//! SQS adapter for cross-family search cleanup and document deletion.
use std::sync::Arc;
-use sqs_client::search::{
- SearchQueueMessage,
- chat::RemoveChatMessage,
- document::DocumentId,
- project::{RemoveProject, UpsertProject},
-};
+use sqs_client::search::{SearchQueueMessage, chat::RemoveChatMessage, document::DocumentId};
use crate::domain::ports::ProjectSearchIndexer;
-/// SQS-backed project search-index and document-deletion adapter.
+/// SQS-backed cross-family search cleanup and document-deletion adapter.
#[derive(Clone)]
pub struct SqsProjectSearchIndexer {
sqs: Arc<sqs_client::SQS>,
@@ -25,40 +20,6 @@ impl SqsProjectSearchIndexer {
}
impl ProjectSearchIndexer for SqsProjectSearchIndexer {
- #[tracing::instrument(skip(self), err)]
- async fn upsert_projects(&self, project_ids: Vec<String>) -> anyhow::Result<()> {
- let messages = project_ids
- .into_iter()
- .map(|project_id| {
- SearchQueueMessage::UpsertProject(UpsertProject {
- project_id,
- index_override: None,
- })
- })
- .collect();
-
- self.sqs
- .bulk_send_message_to_search_event_queue(messages)
- .await
- }
-
- #[tracing::instrument(skip(self), err)]
- async fn remove_projects(&self, project_ids: Vec<String>) -> anyhow::Result<()> {
- let messages = project_ids
- .into_iter()
- .map(|project_id| {
- SearchQueueMessage::RemoveProject(RemoveProject {
- project_id,
- index_override: None,
- })
- })
- .collect();
-
- self.sqs
- .bulk_send_message_to_search_event_queue(messages)
- .await
- }
-
#[tracing::instrument(skip(self), err)]
async fn remove_chats(&self, chat_ids: Vec<String>) -> anyhow::Result<()> {
let messages = chat_ids
diff --git a/crates/sqs_client/src/search/mod.rs b/crates/sqs_client/src/search/mod.rs
index 0f0f0c82a4..e1bd1f8bfc 100644
--- a/crates/sqs_client/src/search/mod.rs
+++ b/crates/sqs_client/src/search/mod.rs
@@ -6,7 +6,7 @@ use crate::{
chat::{ChatMessage, RemoveChatMessage},
document::{DocumentId, DocumentPropertiesUpdate, SearchExtractorMessage},
email::{EmailLinkMessage, EmailMessage, EmailThreadBatchMessage, EmailThreadMessage},
- project::{RemoveProject, UpsertProject},
+ project::UpsertProject,
},
};
use anyhow::Context;
@@ -90,7 +90,6 @@ pub enum SearchQueueMessage {
RemoveCallRecord(RemoveCallRecord),
// Project
UpsertProject(UpsertProject),
- RemoveProject(RemoveProject),
// User
RemoveUserProfile(String),
@@ -122,7 +121,6 @@ impl PrimaryId for SearchQueueMessage {
message.call_id.clone().unwrap_or_default()
),
SearchQueueMessage::UpsertProject(message) => message.project_id.clone(),
- SearchQueueMessage::RemoveProject(message) => message.project_id.clone(),
SearchQueueMessage::RemoveUserProfile(message) => message.clone(),
}
@@ -154,7 +152,6 @@ impl SearchQueueMessage {
SearchQueueMessage::RemoveCallRecord(_) => Operation::Remove,
// Projects
SearchQueueMessage::UpsertProject(_) => Operation::UpdateMetadata,
- SearchQueueMessage::RemoveProject(_) => Operation::Remove,
// Users
SearchQueueMessage::RemoveUserProfile(_) => Operation::Remove,
}
diff --git a/crates/sqs_client/src/search/project.rs b/crates/sqs_client/src/search/project.rs
index 5c45b9c420..8e2af56ab5 100644
--- a/crates/sqs_client/src/search/project.rs
+++ b/crates/sqs_client/src/search/project.rs
@@ -1,3 +1,7 @@
+/// Requests project reconciliation through the SQS backfill path.
+///
+/// Project lifecycle updates use Kafka. This payload remains available only so
+/// search backfills can use `index_override` to target an alternate index.
#[derive(serde::Serialize, serde::Deserialize, Debug, PartialEq, Eq)]
pub struct UpsertProject {
/// The project id
@@ -6,12 +10,3 @@ pub struct UpsertProject {
#[serde(skip_serializing_if = "Option::is_none", default)]
pub index_override: Option<String>,
}
-
-#[derive(serde::Serialize, serde::Deserialize, Debug, PartialEq, Eq)]
-pub struct RemoveProject {
- /// The project id to remove
- pub project_id: String,
- /// Optional override for the target OpenSearch index
- #[serde(skip_serializing_if = "Option::is_none", default)]
- pub index_override: Option<String>,
-}
diff --git a/infra/stacks/deleted-item-poller/index.ts b/infra/stacks/deleted-item-poller/index.ts
index 485bc0dd79..90ff7e1e06 100644
--- a/infra/stacks/deleted-item-poller/index.ts
+++ b/infra/stacks/deleted-item-poller/index.ts
@@ -23,6 +23,17 @@ const cloudStorageServiceStack = new pulumi.StackReference(
}
);
+const kafkaClusterStack = new pulumi.StackReference(
+ 'kafka-cluster-brokers-stack',
+ {
+ name: `macro-inc/kafka-cluster/${stack}`,
+ }
+);
+
+const kafkaBrokers = kafkaClusterStack
+ .getOutput('bootstrapBrokersSaslIam')
+ .apply((brokers) => brokers as string);
+
const deleteDocumentQueueArn: pulumi.Output<string> = cloudStorageServiceStack
.getOutput('deleteDocumentQueueArn')
.apply((arn) => arn as string);
@@ -41,6 +52,7 @@ const deletedItemPoller = new DeleteItemPoller('deleted-item-poller', {
envVars: {
DATABASE_URL: pulumi.interpolate`${DATABASE_URL}`,
ENVIRONMENT: stack,
+ KAFKA_BROKERS: kafkaBrokers,
RUST_LOG: 'deleted_item_poller=info,macro_http_request=info',
},
tags,
diff --git a/infra/stacks/deleted-item-poller/lambda.ts b/infra/stacks/deleted-item-poller/lambda.ts
index d6426ffcaa..5a554aeb55 100644
--- a/infra/stacks/deleted-item-poller/lambda.ts
+++ b/infra/stacks/deleted-item-poller/lambda.ts
@@ -1,7 +1,11 @@
import * as aws from '@pulumi/aws';
import * as pulumi from '@pulumi/pulumi';
import { Lambda } from '../../packages/lambda';
-import { CLOUD_TRAIL_SNS_TOPIC_ARN, stack } from '../../packages/shared';
+import {
+ CLOUD_TRAIL_SNS_TOPIC_ARN,
+ getKafkaClusterPolicy,
+ stack,
+} from '../../packages/shared';
const LAMBDA_BASE_NAME = 'deleted_item_poller';
const REPO_ROOT = '../../..';
@@ -11,6 +15,7 @@ const ZIP_LOCATION = `${REPO_ROOT}/target/lambda/${LAMBDA_BASE_NAME}/bootstrap.z
export type EnvVars = {
DATABASE_URL: pulumi.Output<string> | string;
ENVIRONMENT: pulumi.Output<string> | string;
+ KAFKA_BROKERS: pulumi.Output<string> | string;
RUST_LOG: pulumi.Output<string> | string;
};
@@ -80,6 +85,7 @@ export class DeleteItemPoller extends pulumi.ComponentResource {
aws.iam.ManagedPolicy.AWSLambdaVPCAccessExecutionRole,
aws.iam.ManagedPolicy.CloudWatchLogsFullAccess,
sqsPolicy.arn,
+ getKafkaClusterPolicy(),
],
tags: this.tags,
},
diff --git a/services/deleted_item_poller/Cargo.toml b/services/deleted_item_poller/Cargo.toml
index 9b8ce72308..eba4186fcf 100644
--- a/services/deleted_item_poller/Cargo.toml
+++ b/services/deleted_item_poller/Cargo.toml
@@ -17,9 +17,12 @@ macro_db_client = { path = "../../crates/macro_db_client" }
macro_entrypoint = { path = "../../crates/macro_entrypoint" }
macro_env = { path = "../../crates/macro_env" }
macro_env_var = { path = "../../crates/macro_env_var" }
+macro_event_broker = { path = "../../crates/macro_event_broker" }
macro_queues = { path = "../../crates/macro_queues" }
+macro_user_id = { path = "../../crates/macro_user_id", default-features = false }
macro_uuid = { path = "../../crates/macro_uuid" }
model = { path = "../../crates/model" }
+projects = { path = "../../crates/projects", default-features = false }
serde = { workspace = true }
serde_json = { workspace = true }
sqlx = { workspace = true }
diff --git a/services/deleted_item_poller/src/config.rs b/services/deleted_item_poller/src/config.rs
index 3bf0211428..5780c8e957 100644
--- a/services/deleted_item_poller/src/config.rs
+++ b/services/deleted_item_poller/src/config.rs
@@ -4,6 +4,8 @@ use macro_env_var::env_vars;
env_vars! {
struct DatabaseUrl;
+ #[derive(Debug, Clone)]
+ pub struct KafkaBrokers;
}
#[derive(Debug, Clone)]
@@ -11,6 +13,9 @@ pub struct Config {
/// The connection URL for the Postgres database this application should use.
pub database_url: String,
+ /// Comma-separated Kafka bootstrap servers for project lifecycle events.
+ pub kafka_brokers: KafkaBrokers,
+
/// The environment we are in
#[allow(dead_code)]
pub environment: Environment,
@@ -21,9 +26,11 @@ impl Config {
let database_url = DatabaseUrl::new()
.context("DATABASE_URL must be provided")?
.to_string();
+ let kafka_brokers = KafkaBrokers::new().context("KAFKA_BROKERS must be provided")?;
Ok(Config {
database_url,
+ kafka_brokers,
environment: Environment::new_or_prod(),
})
}
diff --git a/services/deleted_item_poller/src/context.rs b/services/deleted_item_poller/src/context.rs
index 7b9c0db64e..a92c89477a 100644
--- a/services/deleted_item_poller/src/context.rs
+++ b/services/deleted_item_poller/src/context.rs
@@ -1,7 +1,12 @@
use std::sync::Arc;
+use macro_event_broker::{GlobalSpawner, KafkaEventPublisher, MacroEventBrokerService};
+
+pub type PollerEventBroker = MacroEventBrokerService<KafkaEventPublisher, GlobalSpawner>;
+
#[derive(Clone)]
pub struct Context {
pub db: sqlx::Pool<sqlx::Postgres>,
+ pub macro_event_broker: PollerEventBroker,
pub sqs_client: Arc<sqs_client::SQS>,
}
diff --git a/services/deleted_item_poller/src/handler.rs b/services/deleted_item_poller/src/handler.rs
index 590a2bffa6..4599839b54 100644
--- a/services/deleted_item_poller/src/handler.rs
+++ b/services/deleted_item_poller/src/handler.rs
@@ -1,13 +1,19 @@
+#[cfg(test)]
+mod test;
+
use crate::context::{self};
use anyhow::Context;
use aws_lambda_events::eventbridge::EventBridgeEvent;
+use futures::future::join_all;
use lambda_runtime::{
Error, LambdaEvent,
tracing::{self},
};
-use sqs_client::search::{
- SearchQueueMessage, chat::RemoveChatMessage, document::DocumentId, project::RemoveProject,
-};
+use macro_db_client::projects::ProjectToDelete;
+use macro_event_broker::MacroEventBroker;
+use macro_user_id::user_id::MacroUserIdStr;
+use projects::domain::events::{ProjectMacroEvent, ProjectPermanentlyDeletedMetadata};
+use sqs_client::search::{SearchQueueMessage, chat::RemoveChatMessage, document::DocumentId};
#[tracing::instrument(skip(ctx, _event), err)]
pub async fn handler(
@@ -23,6 +29,53 @@ pub async fn handler(
Ok(())
}
+#[tracing::instrument(skip(event_broker, projects_to_delete), err)]
+async fn publish_project_purge_events<B: MacroEventBroker>(
+ event_broker: &B,
+ projects_to_delete: &[ProjectToDelete],
+) -> anyhow::Result<()> {
+ let events = projects_to_delete
+ .iter()
+ .map(|project| {
+ let project_id = project.project_id.clone();
+ let owner: MacroUserIdStr<'static> = MacroUserIdStr::try_from(project.user_id.clone())
+ .with_context(|| format!("invalid owner for project {}", project.project_id))?;
+
+ Ok(ProjectMacroEvent::permanently_deleted(
+ project_id.clone(),
+ ProjectPermanentlyDeletedMetadata {
+ project_id: project_id.clone(),
+ owner,
+ actor_user_id: None,
+ parent_project_id: None,
+ purged_project_ids: vec![project_id],
+ purged_document_ids: Vec::new(),
+ purged_chat_ids: Vec::new(),
+ },
+ ))
+ })
+ .collect::<anyhow::Result<Vec<_>>>()?;
+
+ let publications = events
+ .iter()
+ .map(|event| event_broker.send_event(event))
+ .collect::<Vec<_>>();
+ let publication_results = join_all(publications.into_iter().map(|publication| async move {
+ let handle = publication.context("failed to enqueue project purge event")?;
+ handle
+ .await
+ .context("project purge event publication task failed")?
+ .context("failed to publish project purge event")
+ }))
+ .await;
+
+ for result in publication_results {
+ result?;
+ }
+
+ Ok(())
+}
+
#[tracing::instrument(skip(ctx), err)]
async fn handle_projects(ctx: &context::Context) -> anyhow::Result<()> {
let date = chrono::Utc::now().naive_utc() - chrono::Duration::days(30);
@@ -37,25 +90,18 @@ async fn handle_projects(ctx: &context::Context) -> anyhow::Result<()> {
tracing::debug!(projects_to_delete=?projects_to_delete, "projects to delete");
- // Idempotent safety net — the soft delete already removed these from the
- // search index.
- ctx.sqs_client
- .bulk_send_message_to_search_event_queue(
- projects_to_delete
- .iter()
- .map(|id| {
- SearchQueueMessage::RemoveProject(RemoveProject {
- project_id: id.to_string(),
- index_override: None,
- })
- })
- .collect(),
- )
- .await?;
+ publish_project_purge_events(&ctx.macro_event_broker, &projects_to_delete)
+ .await
+ .context("unable to publish project purge events")?;
+
+ let project_ids = projects_to_delete
+ .into_iter()
+ .map(|project| project.project_id)
+ .collect::<Vec<String>>();
// We can actually perform the project deletion here as we will automatically be queuing all
// the items in the project for deletion as well
- macro_db_client::projects::delete::delete_projects_bulk(&ctx.db, &projects_to_delete)
+ macro_db_client::projects::delete::delete_projects_bulk(&ctx.db, &project_ids)
.await
.context("unable to delete projects")?;
diff --git a/services/deleted_item_poller/src/handler/test.rs b/services/deleted_item_poller/src/handler/test.rs
new file mode 100644
index 0000000000..c342399143
--- /dev/null
+++ b/services/deleted_item_poller/src/handler/test.rs
@@ -0,0 +1,128 @@
+use std::sync::{Arc, Mutex};
+
+use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker};
+use serde_json::{Value, json};
+
+use super::*;
+
+#[derive(Clone, Debug, PartialEq)]
+struct PublishedEvent {
+ topic: String,
+ key: String,
+ envelope: Value,
+}
+
+#[derive(Clone, Default)]
+struct FakeEventBroker {
+ published: Arc<Mutex<Vec<PublishedEvent>>>,
+ fail_publication: bool,
+}
+
+impl FakeEventBroker {
+ fn failing() -> Self {
+ Self {
+ fail_publication: true,
+ ..Self::default()
+ }
+ }
+
+ fn published(&self) -> Vec<PublishedEvent> {
+ self.published.lock().unwrap().clone()
+ }
+}
+
+impl MacroEventBroker for FakeEventBroker {
+ fn send_event<E: MacroEvent + ?Sized>(
+ &self,
+ event: &E,
+ ) -> Result<tokio::task::JoinHandle<Result<(), EventBrokerError>>, EventBrokerError> {
+ self.published.lock().unwrap().push(PublishedEvent {
+ topic: event.topic().to_string(),
+ key: event.key().to_string(),
+ envelope: serde_json::to_value(event.event())?,
+ });
+
+ let fail_publication = self.fail_publication;
+ Ok(tokio::spawn(async move {
+ if fail_publication {
+ Err(EventBrokerError::Publish(
+ "publisher unavailable".to_string(),
+ ))
+ } else {
+ Ok(())
+ }
+ }))
+ }
+}
+
+#[tokio::test]
+async fn publish_project_purge_events_publishes_separately_keyed_events() {
+ let event_broker = FakeEventBroker::default();
+ let projects = vec![
+ ProjectToDelete {
+ project_id: "project-one".to_string(),
+ user_id: "macro|[email protected]".to_string(),
+ },
+ ProjectToDelete {
+ project_id: "project-two".to_string(),
+ user_id: "macro|[email protected]".to_string(),
+ },
+ ];
+
+ publish_project_purge_events(&event_broker, &projects)
+ .await
+ .unwrap();
+
+ let published = event_broker.published();
+ assert_eq!(published.len(), projects.len());
+
+ for (event, project) in published.iter().zip(&projects) {
+ assert_eq!(event.topic, "macro.projects");
+ assert_eq!(event.key, project.project_id);
+ assert_eq!(event.envelope["schema_version"], json!(1));
+ assert_eq!(
+ event.envelope["event_type"],
+ json!("project.permanently_deleted")
+ );
+
+ let metadata = &event.envelope["metadata"];
+ assert_eq!(metadata["project_id"], json!(project.project_id));
+ assert_eq!(metadata["owner"], json!(project.user_id));
+ assert_eq!(metadata["actor_user_id"], Value::Null);
+ assert_eq!(metadata["parent_project_id"], Value::Null);
+ assert_eq!(metadata["purged_project_ids"], json!([project.project_id]));
+ assert_eq!(metadata["purged_document_ids"], json!([]));
+ assert_eq!(metadata["purged_chat_ids"], json!([]));
+ }
+}
+
+#[tokio::test]
+async fn publish_project_purge_events_returns_publication_failures() {
+ let event_broker = FakeEventBroker::failing();
+ let projects = vec![ProjectToDelete {
+ project_id: "project-one".to_string(),
+ user_id: "macro|[email protected]".to_string(),
+ }];
+
+ let error = publish_project_purge_events(&event_broker, &projects)
+ .await
+ .expect_err("publication failure should be returned");
+
+ assert!(format!("{error:#}").contains("publisher unavailable"));
+}
+
+#[tokio::test]
+async fn publish_project_purge_events_rejects_malformed_owners() {
+ let event_broker = FakeEventBroker::default();
+ let projects = vec![ProjectToDelete {
+ project_id: "project-one".to_string(),
+ user_id: "not-a-macro-user".to_string(),
+ }];
+
+ let error = publish_project_purge_events(&event_broker, &projects)
+ .await
+ .expect_err("malformed owners should be returned");
+
+ assert!(format!("{error:#}").contains("invalid owner for project project-one"));
+ assert!(event_broker.published().is_empty());
+}
diff --git a/services/deleted_item_poller/src/main.rs b/services/deleted_item_poller/src/main.rs
index 7033137cee..e4022b7175 100644
--- a/services/deleted_item_poller/src/main.rs
+++ b/services/deleted_item_poller/src/main.rs
@@ -12,6 +12,7 @@ use lambda_runtime::{
tracing::{self},
};
use macro_entrypoint::MacroEntrypoint;
+use macro_event_broker::{GlobalSpawner, KafkaEventPublisher};
use sqlx::postgres::PgPoolOptions;
use std::sync::Arc;
@@ -24,6 +25,12 @@ async fn main() -> Result<(), Error> {
tracing::trace!("initialized config");
+ let macro_event_broker = context::PollerEventBroker::new(
+ KafkaEventPublisher::new(config.kafka_brokers.as_ref())
+ .context("failed to create kafka event publisher")?,
+ GlobalSpawner,
+ );
+
// We should only ever need 1 connection
let db = PgPoolOptions::new()
.min_connections(3)
@@ -44,6 +51,7 @@ async fn main() -> Result<(), Error> {
let ctx = context::Context {
db,
+ macro_event_broker,
sqs_client: Arc::new(sqs_client),
};
diff --git a/services/document_storage_service/src/api/context.rs b/services/document_storage_service/src/api/context.rs
index 7a94cdc840..9f2ab31b72 100644
--- a/services/document_storage_service/src/api/context.rs
+++ b/services/document_storage_service/src/api/context.rs
@@ -451,6 +451,7 @@ pub(crate) struct ApiContext {
pub favorites_state: DssFavoritesState,
pub favorites_service: Arc<FavoritesServiceType>,
pub foreign_entity_state: DssForeignEntityState,
+ pub macro_event_broker: DssEventBroker,
pub sqs_client: Arc<sqs_client::SQS>,
pub contacts_ingress: Arc<SqsContactsIngress<SqsContactsQueue>>,
pub notification_ingress_service: Arc<NotificationIngressType>,
diff --git a/services/document_storage_service/src/api/documents/initialize_user_documents.rs b/services/document_storage_service/src/api/documents/initialize_user_documents.rs
index 74f7819804..21547b92a3 100644
--- a/services/document_storage_service/src/api/documents/initialize_user_documents.rs
+++ b/services/document_storage_service/src/api/documents/initialize_user_documents.rs
@@ -5,15 +5,16 @@ use axum::{
};
use futures::StreamExt;
use macro_authorization::{MacroAuthorizationExtractor, UserOrInternal};
+use macro_event_broker::MacroEventBroker;
use macro_user_id::cowlike::CowLike;
use model::{
document::BasicDocument,
response::{ErrorResponse, GenericErrorResponse, GenericSuccessResponse},
};
use models_permissions::share_permission::SharePermissionV2;
+use projects_hex::domain::events::{ProjectCreatedMetadata, ProjectMacroEvent};
use reqwest::StatusCode;
use s3_key::build_cloud_storage_bucket_document_key;
-use sqs_client::search::{SearchQueueMessage, project::UpsertProject};
const ONBOARDING_FOLDER_NAME: &str = "ONBOARDING_DOCUMENTS";
const PROJECT_NAME: &str = "Starter Docs";
@@ -246,23 +247,26 @@ pub async fn handler(
.into_response()
})?;
- tokio::spawn({
- let sqs_client = state.sqs_client.clone();
- let project_id = project.id.clone();
- async move {
- let _ = sqs_client
- .send_message_to_search_event_queue(SearchQueueMessage::UpsertProject(
- UpsertProject {
- project_id: project_id.clone(),
- index_override: None,
- },
- ))
- .await
- .inspect_err(
- |e| tracing::error!(error=?e, project_id=?project_id, "unable to enqueue project search upsert"),
- );
- }
- });
+ let project_event = ProjectMacroEvent::created(
+ project.id.clone(),
+ ProjectCreatedMetadata {
+ project_id: project.id.clone(),
+ owner: user_context.authorization.user.macro_user_id.clone(),
+ name: PROJECT_NAME.to_string(),
+ parent_project_id: None,
+ created_at: project.created_at,
+ },
+ );
+ let _ = state
+ .macro_event_broker
+ .send_event(&project_event)
+ .inspect_err(|error| {
+ tracing::error!(
+ error=?error,
+ project_id = %project.id,
+ "unable to enqueue project created event"
+ );
+ });
Ok((StatusCode::OK, Json(GenericSuccessResponse::default())).into_response())
}
diff --git a/services/document_storage_service/src/main.rs b/services/document_storage_service/src/main.rs
index 49f7abd9cc..13cdbf1773 100644
--- a/services/document_storage_service/src/main.rs
+++ b/services/document_storage_service/src/main.rs
@@ -1044,6 +1044,7 @@ async fn main() -> anyhow::Result<()> {
s3_client: s3,
dynamodb_client: Arc::new(dynamodb_client),
dynamo_db,
+ macro_event_broker: macro_event_broker.clone(),
sqs_client: sqs_client.clone(),
notification_ingress_service: notification_ingress_service.clone(),
conn_gateway_client: conn_gateway_client.clone(),
diff --git a/services/search_processing_service/Cargo.toml b/services/search_processing_service/Cargo.toml
index 23cb6e3773..bfc46f9bab 100644
--- a/services/search_processing_service/Cargo.toml
+++ b/services/search_processing_service/Cargo.toml
@@ -13,6 +13,7 @@ processing = [
"dep:kafka_util",
"dep:macro_event_broker",
"dep:macro_event_topics",
+ "dep:projects",
"dep:rdkafka",
"dep:rootcause",
"dep:tokio-retry",
@@ -78,6 +79,7 @@ pdfium-render = { workspace = true, optional = true }
properties = { path = "../../crates/properties", default-features = false, features = [
"outbound",
], optional = true }
+projects = { path = "../../crates/projects", default-features = false, optional = true }
rdkafka = { workspace = true, optional = true }
rootcause = { workspace = true, optional = true }
rust-embed = "8.6.0"
diff --git a/services/search_processing_service/src/inbound/kafka_consumer.rs b/services/search_processing_service/src/inbound/kafka_consumer.rs
index 391070dee2..298924a806 100644
--- a/services/search_processing_service/src/inbound/kafka_consumer.rs
+++ b/services/search_processing_service/src/inbound/kafka_consumer.rs
@@ -5,19 +5,27 @@
//! committed without a handoff so they cannot wedge a partition. Processing is
//! retried in-process; exhausted events are logged and dropped because their
//! offsets are already committed.
+//!
+//! Per-entity event mapping and processing live in the [`call`], [`channel`],
+//! and [`project`] submodules; this module owns the poll loop, worker, retry
+//! policy, and commit semantics.
+
+#![allow(clippy::enum_variant_names)]
+mod call;
+mod channel;
+mod project;
#[cfg(test)]
mod test;
use std::{future::Future, time::Duration};
-use call::domain::events::{CallMacroEvent, CallTopicEvent};
-use channels::domain::broker_events::{ChannelMacroEvent, ChannelTopicEvent};
+use ::call::domain::events::CallMacroEvent;
+use channels::domain::broker_events::ChannelMacroEvent;
use kafka_util::{GroupName, KafkaEventConsumer};
-use macro_event_broker::{
- KafkaConsumerAdapter, MacroEvent as _, MacroEventCollection as _, MacroEventConsumerService,
-};
+use macro_event_broker::{KafkaConsumerAdapter, MacroEventCollection, MacroEventConsumerService};
use opensearch_client::OpensearchClient;
+use projects::domain::events::ProjectMacroEvent;
use rdkafka::{
consumer::CommitMode,
message::{BorrowedMessage, Message as _},
@@ -26,11 +34,9 @@ use rootcause::prelude::{Report, ResultExt as _};
use sqlx::PgPool;
use tokio::sync::mpsc;
use tokio_retry::{Retry, strategy::ExponentialBackoff};
-use uuid::Uuid;
-use crate::process::{
- call::{process_call_record, process_remove_call_record},
- channel::{process_channel_message_update, process_remove_channel_message},
+use self::{
+ call::process_call_event, channel::process_channel_event, project::process_project_event,
};
/// Consumer group used for live search-index event offsets.
@@ -41,12 +47,15 @@ impl GroupName for SearchProcessingConsumerGroup {
}
type SearchProcessingKafkaAdapter =
- KafkaConsumerAdapter<SearchProcessingConsumerGroup, SearchProcessingBrokerEvent>;
+ KafkaConsumerAdapter<SearchProcessingConsumerGroup, DeclaredMacroEvent>;
type SearchProcessingKafkaConsumer =
- MacroEventConsumerService<SearchProcessingBrokerEvent, SearchProcessingKafkaAdapter>;
+ MacroEventConsumerService<DeclaredMacroEvent, SearchProcessingKafkaAdapter>;
macro_event_broker::declare_topics!(
- SearchProcessingBrokerEvent: CallMacroEvent, ChannelMacroEvent
+ DeclaredMacroEvent:
+ CallMacroEvent,
+ ChannelMacroEvent,
+ ProjectMacroEvent,
);
/// Maximum number of decoded events waiting for the sequential worker.
@@ -60,40 +69,11 @@ const PROCESSING_RETRY_BASE_DELAY: Duration = Duration::from_secs(1);
/// A decoded search-processing event and its Kafka coordinates.
struct ReceivedEvent {
- event: SearchProcessingBrokerEvent,
+ event: DeclaredMacroEvent,
partition: i32,
offset: i64,
}
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-enum CallIndexAction {
- Upsert { call_id: Uuid },
- Remove { call_id: Uuid, channel_id: Uuid },
- Ignore,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-struct CallEventDescription {
- action: CallIndexAction,
- call_id: Uuid,
- event_type: &'static str,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-enum ChannelIndexAction {
- UpsertMessage { channel_id: Uuid, message_id: Uuid },
- RemoveMessage { channel_id: Uuid, message_id: Uuid },
- RemoveChannel { channel_id: Uuid },
- Ignore,
-}
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-struct ChannelEventDescription {
- action: ChannelIndexAction,
- channel_id: Uuid,
- event_type: &'static str,
-}
-
/// Outcome after the worker handles one decoded broker event.
#[derive(Debug, Clone, Copy, PartialEq, Eq)]
enum EventOutcome {
@@ -117,122 +97,6 @@ enum HandoffOutcome<E> {
WorkerClosed,
}
-fn describe_call_event(event: &CallTopicEvent) -> CallEventDescription {
- match event {
- CallTopicEvent::Started(metadata) => CallEventDescription {
- action: CallIndexAction::Ignore,
- call_id: metadata.call_id,
- event_type: "call.started",
- },
- CallTopicEvent::RecordArchived(metadata) => CallEventDescription {
- action: CallIndexAction::Upsert {
- call_id: metadata.call_id,
- },
- call_id: metadata.call_id,
- event_type: "call.record_archived",
- },
- CallTopicEvent::RecordUpdated(metadata) => CallEventDescription {
- action: CallIndexAction::Upsert {
- call_id: metadata.call_id,
- },
- call_id: metadata.call_id,
- event_type: "call.record_updated",
- },
- CallTopicEvent::RecordDeleted(metadata) => CallEventDescription {
- action: CallIndexAction::Remove {
- call_id: metadata.call_id,
- channel_id: metadata.channel_id,
- },
- call_id: metadata.call_id,
- event_type: "call.record_deleted",
- },
- CallTopicEvent::RecordSummarized(metadata) => CallEventDescription {
- action: CallIndexAction::Upsert {
- call_id: metadata.call_id,
- },
- call_id: metadata.call_id,
- event_type: "call.record_summarized",
- },
- CallTopicEvent::RecordingReady(metadata) => CallEventDescription {
- action: CallIndexAction::Ignore,
- call_id: metadata.call_id,
- event_type: "call.recording_ready",
- },
- }
-}
-
-fn describe_channel_event(event: &ChannelTopicEvent) -> ChannelEventDescription {
- match event {
- ChannelTopicEvent::Created(metadata) => ChannelEventDescription {
- action: ChannelIndexAction::Ignore,
- channel_id: metadata.channel_id,
- event_type: "channel.created",
- },
- ChannelTopicEvent::Updated(metadata) => ChannelEventDescription {
- action: ChannelIndexAction::Ignore,
- channel_id: metadata.channel_id,
- event_type: "channel.updated",
- },
- ChannelTopicEvent::Deleted(metadata) => ChannelEventDescription {
- action: ChannelIndexAction::RemoveChannel {
- channel_id: metadata.channel_id,
- },
- channel_id: metadata.channel_id,
- event_type: "channel.deleted",
- },
- ChannelTopicEvent::MessagePosted(metadata) => ChannelEventDescription {
- action: ChannelIndexAction::UpsertMessage {
- channel_id: metadata.channel_id,
- message_id: metadata.message_id,
- },
- channel_id: metadata.channel_id,
- event_type: "channel.message_posted",
- },
- ChannelTopicEvent::MessagePatched(metadata) => ChannelEventDescription {
- action: ChannelIndexAction::UpsertMessage {
- channel_id: metadata.channel_id,
- message_id: metadata.message_id,
- },
- channel_id: metadata.channel_id,
- event_type: "channel.message_patched",
- },
- ChannelTopicEvent::MessageDeleted(metadata) => ChannelEventDescription {
- action: ChannelIndexAction::RemoveMessage {
- channel_id: metadata.channel_id,
- message_id: metadata.message_id,
- },
- channel_id: metadata.channel_id,
- event_type: "channel.message_deleted",
- },
- ChannelTopicEvent::MessageAttachmentCreated(metadata) => ChannelEventDescription {
- action: ChannelIndexAction::UpsertMessage {
- channel_id: metadata.channel_id,
- message_id: metadata.message_id,
- },
- channel_id: metadata.channel_id,
- event_type: "channel.message_attachment_created",
- },
- ChannelTopicEvent::MessageAttachmentRemoved(metadata) => ChannelEventDescription {
- action: ChannelIndexAction::UpsertMessage {
- channel_id: metadata.channel_id,
- message_id: metadata.message_id,
- },
- channel_id: metadata.channel_id,
- event_type: "channel.message_attachment_removed",
- },
- ChannelTopicEvent::ParticipantAdded(metadata) => ChannelEventDescription {
- action: ChannelIndexAction::Ignore,
- channel_id: metadata.channel_id,
- event_type: "channel.participant_added",
- },
- ChannelTopicEvent::ParticipantRemoved(metadata) => ChannelEventDescription {
- action: ChannelIndexAction::Ignore,
- channel_id: metadata.channel_id,
- event_type: "channel.participant_removed",
- },
- }
-}
-
fn processing_retry_strategy() -> impl Iterator<Item = Duration> {
ExponentialBackoff::from_millis(2)
.factor(500)
@@ -280,7 +144,7 @@ async fn handoff_decoded<T, E>(
}
fn attach_event_coordinates<E>(
- decoded: Result<SearchProcessingBrokerEvent, E>,
+ decoded: Result<DeclaredMacroEvent, E>,
partition: i32,
offset: i64,
) -> Result<ReceivedEvent, E> {
@@ -291,197 +155,24 @@ fn attach_event_coordinates<E>(
})
}
-async fn process_call_index_action(
- db: &PgPool,
- opensearch_client: &OpensearchClient,
- action: CallIndexAction,
-) -> anyhow::Result<()> {
- match action {
- CallIndexAction::Upsert { call_id } => {
- process_call_record(opensearch_client, db, call_id, None).await
- }
- CallIndexAction::Remove {
- call_id,
- channel_id,
- } => process_remove_call_record(opensearch_client, channel_id, Some(call_id), None).await,
- CallIndexAction::Ignore => Ok(()),
- }
-}
-
-async fn process_channel_index_action(
- db: &PgPool,
- opensearch_client: &OpensearchClient,
- action: ChannelIndexAction,
-) -> anyhow::Result<()> {
- match action {
- ChannelIndexAction::UpsertMessage {
- channel_id,
- message_id,
- } => {
- process_channel_message_update(opensearch_client, db, channel_id, message_id, None)
- .await
- }
- ChannelIndexAction::RemoveMessage {
- channel_id,
- message_id,
- } => {
- process_remove_channel_message(opensearch_client, channel_id, Some(message_id), None)
- .await
- }
- ChannelIndexAction::RemoveChannel { channel_id } => {
- process_remove_channel_message(opensearch_client, channel_id, None, None).await
- }
- ChannelIndexAction::Ignore => Ok(()),
- }
-}
-
-async fn process_call_event(
- db: &PgPool,
- opensearch_client: &OpensearchClient,
- event: &CallMacroEvent,
- partition: i32,
- offset: i64,
-) -> EventOutcome {
- let description = describe_call_event(&event.event().event);
- if description.action == CallIndexAction::Ignore {
- tracing::trace!(
- call_id = %description.call_id,
- event_type = description.event_type,
- partition,
- offset,
- "ignoring call event without a search-index action"
- );
- return EventOutcome::Ignored;
- }
-
- let result = retry_processing(|attempt| async move {
- tracing::trace!(
- call_id = %description.call_id,
- event_type = description.event_type,
- partition,
- offset,
- attempt,
- "processing call search-index event"
- );
- process_call_index_action(db, opensearch_client, description.action)
- .await
- .inspect_err(|error| {
- if attempt < MAX_PROCESSING_ATTEMPTS {
- let retry_delay =
- PROCESSING_RETRY_BASE_DELAY * 2u32.pow(attempt.saturating_sub(1));
- tracing::warn!(
- error = ?error,
- call_id = %description.call_id,
- event_type = description.event_type,
- partition,
- offset,
- attempt,
- delay_secs = retry_delay.as_secs(),
- "call search-index processing failed, retrying"
- );
- }
- })
- })
- .await;
-
- match result {
- Ok(()) => EventOutcome::Indexed,
- Err(error) => {
- tracing::error!(
- error = ?error,
- call_id = %description.call_id,
- event_type = description.event_type,
- partition,
- offset,
- attempts = MAX_PROCESSING_ATTEMPTS,
- "dropping call event after processing retries were exhausted"
- );
- EventOutcome::Dropped
- }
- }
-}
-
-async fn process_channel_event(
- db: &PgPool,
- opensearch_client: &OpensearchClient,
- event: &ChannelMacroEvent,
- partition: i32,
- offset: i64,
-) -> EventOutcome {
- let description = describe_channel_event(&event.event().event);
- if description.action == ChannelIndexAction::Ignore {
- tracing::trace!(
- channel_id = %description.channel_id,
- event_type = description.event_type,
- partition,
- offset,
- "ignoring channel event without a search-index action"
- );
- return EventOutcome::Ignored;
- }
-
- let result = retry_processing(|attempt| async move {
- tracing::trace!(
- channel_id = %description.channel_id,
- event_type = description.event_type,
- partition,
- offset,
- attempt,
- "processing channel search-index event"
- );
- process_channel_index_action(db, opensearch_client, description.action)
- .await
- .inspect_err(|error| {
- if attempt < MAX_PROCESSING_ATTEMPTS {
- let retry_delay =
- PROCESSING_RETRY_BASE_DELAY * 2u32.pow(attempt.saturating_sub(1));
- tracing::warn!(
- error = ?error,
- channel_id = %description.channel_id,
- event_type = description.event_type,
- partition,
- offset,
- attempt,
- delay_secs = retry_delay.as_secs(),
- "channel search-index processing failed, retrying"
- );
- }
- })
- })
- .await;
-
- match result {
- Ok(()) => EventOutcome::Indexed,
- Err(error) => {
- tracing::error!(
- error = ?error,
- channel_id = %description.channel_id,
- event_type = description.event_type,
- partition,
- offset,
- attempts = MAX_PROCESSING_ATTEMPTS,
- "dropping channel event after processing retries were exhausted"
- );
- EventOutcome::Dropped
- }
- }
-}
-
#[tracing::instrument(skip(db, opensearch_client, event), fields(partition, offset))]
async fn process_event(
db: &PgPool,
opensearch_client: &OpensearchClient,
- event: &SearchProcessingBrokerEvent,
+ event: &DeclaredMacroEvent,
partition: i32,
offset: i64,
) -> EventOutcome {
match event {
- SearchProcessingBrokerEvent::CallMacroEvent(event) => {
+ DeclaredMacroEvent::CallMacroEvent(event) => {
process_call_event(db, opensearch_client, event, partition, offset).await
}
- SearchProcessingBrokerEvent::ChannelMacroEvent(event) => {
+ DeclaredMacroEvent::ChannelMacroEvent(event) => {
process_channel_event(db, opensearch_client, event, partition, offset).await
}
+ DeclaredMacroEvent::ProjectMacroEvent(event) => {
+ process_project_event(db, opensearch_client, event, partition, offset).await
+ }
}
}
@@ -598,11 +289,11 @@ pub(crate) async fn run_event_consumer(
) -> Result<(), Report> {
let consumer = KafkaEventConsumer::<SearchProcessingConsumerGroup>::from_env(brokers)?;
let consumer = KafkaConsumerAdapter::<SearchProcessingConsumerGroup, ()>::new(consumer)
- .subscribe::<SearchProcessingBrokerEvent>()
+ .subscribe::<DeclaredMacroEvent>()
.context("failed to subscribe to search processing event topics")?;
let consumer = SearchProcessingKafkaConsumer::new(consumer);
tracing::info!(
- topics = ?SearchProcessingBrokerEvent::topics(),
+ topics = ?DeclaredMacroEvent::topics(),
group = SearchProcessingConsumerGroup::GROUP_NAME,
"search processing event consumer listening"
);
diff --git a/services/search_processing_service/src/inbound/kafka_consumer/call.rs b/services/search_processing_service/src/inbound/kafka_consumer/call.rs
new file mode 100644
index 0000000000..024d85af25
--- /dev/null
+++ b/services/search_processing_service/src/inbound/kafka_consumer/call.rs
@@ -0,0 +1,151 @@
+//! Maps call lifecycle events to search-index actions and processes them.
+
+use ::call::domain::events::{CallMacroEvent, CallTopicEvent};
+use macro_event_broker::MacroEvent as _;
+use opensearch_client::OpensearchClient;
+use sqlx::PgPool;
+use uuid::Uuid;
+
+use super::{EventOutcome, MAX_PROCESSING_ATTEMPTS, PROCESSING_RETRY_BASE_DELAY, retry_processing};
+use crate::process::call::{process_call_record, process_remove_call_record};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(super) enum CallIndexAction {
+ Upsert { call_id: Uuid },
+ Remove { call_id: Uuid, channel_id: Uuid },
+ Ignore,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(super) struct CallEventDescription {
+ pub(super) action: CallIndexAction,
+ pub(super) call_id: Uuid,
+ pub(super) event_type: &'static str,
+}
+
+pub(super) fn describe_call_event(event: &CallTopicEvent) -> CallEventDescription {
+ match event {
+ CallTopicEvent::Started(metadata) => CallEventDescription {
+ action: CallIndexAction::Ignore,
+ call_id: metadata.call_id,
+ event_type: "call.started",
+ },
+ CallTopicEvent::RecordArchived(metadata) => CallEventDescription {
+ action: CallIndexAction::Upsert {
+ call_id: metadata.call_id,
+ },
+ call_id: metadata.call_id,
+ event_type: "call.record_archived",
+ },
+ CallTopicEvent::RecordUpdated(metadata) => CallEventDescription {
+ action: CallIndexAction::Upsert {
+ call_id: metadata.call_id,
+ },
+ call_id: metadata.call_id,
+ event_type: "call.record_updated",
+ },
+ CallTopicEvent::RecordDeleted(metadata) => CallEventDescription {
+ action: CallIndexAction::Remove {
+ call_id: metadata.call_id,
+ channel_id: metadata.channel_id,
+ },
+ call_id: metadata.call_id,
+ event_type: "call.record_deleted",
+ },
+ CallTopicEvent::RecordSummarized(metadata) => CallEventDescription {
+ action: CallIndexAction::Upsert {
+ call_id: metadata.call_id,
+ },
+ call_id: metadata.call_id,
+ event_type: "call.record_summarized",
+ },
+ CallTopicEvent::RecordingReady(metadata) => CallEventDescription {
+ action: CallIndexAction::Ignore,
+ call_id: metadata.call_id,
+ event_type: "call.recording_ready",
+ },
+ }
+}
+
+async fn process_call_index_action(
+ db: &PgPool,
+ opensearch_client: &OpensearchClient,
+ action: CallIndexAction,
+) -> anyhow::Result<()> {
+ match action {
+ CallIndexAction::Upsert { call_id } => {
+ process_call_record(opensearch_client, db, call_id, None).await
+ }
+ CallIndexAction::Remove {
+ call_id,
+ channel_id,
+ } => process_remove_call_record(opensearch_client, channel_id, Some(call_id), None).await,
+ CallIndexAction::Ignore => Ok(()),
+ }
+}
+
+pub(super) async fn process_call_event(
+ db: &PgPool,
+ opensearch_client: &OpensearchClient,
+ event: &CallMacroEvent,
+ partition: i32,
+ offset: i64,
+) -> EventOutcome {
+ let description = describe_call_event(&event.event().event);
+ if description.action == CallIndexAction::Ignore {
+ tracing::trace!(
+ call_id = %description.call_id,
+ event_type = description.event_type,
+ partition,
+ offset,
+ "ignoring call event without a search-index action"
+ );
+ return EventOutcome::Ignored;
+ }
+
+ let result = retry_processing(|attempt| async move {
+ tracing::trace!(
+ call_id = %description.call_id,
+ event_type = description.event_type,
+ partition,
+ offset,
+ attempt,
+ "processing call search-index event"
+ );
+ process_call_index_action(db, opensearch_client, description.action)
+ .await
+ .inspect_err(|error| {
+ if attempt < MAX_PROCESSING_ATTEMPTS {
+ let retry_delay =
+ PROCESSING_RETRY_BASE_DELAY * 2u32.pow(attempt.saturating_sub(1));
+ tracing::warn!(
+ error = ?error,
+ call_id = %description.call_id,
+ event_type = description.event_type,
+ partition,
+ offset,
+ attempt,
+ delay_secs = retry_delay.as_secs(),
+ "call search-index processing failed, retrying"
+ );
+ }
+ })
+ })
+ .await;
+
+ match result {
+ Ok(()) => EventOutcome::Indexed,
+ Err(error) => {
+ tracing::error!(
+ error = ?error,
+ call_id = %description.call_id,
+ event_type = description.event_type,
+ partition,
+ offset,
+ attempts = MAX_PROCESSING_ATTEMPTS,
+ "dropping call event after processing retries were exhausted"
+ );
+ EventOutcome::Dropped
+ }
+ }
+}
diff --git a/services/search_processing_service/src/inbound/kafka_consumer/channel.rs b/services/search_processing_service/src/inbound/kafka_consumer/channel.rs
new file mode 100644
index 0000000000..e811a36e65
--- /dev/null
+++ b/services/search_processing_service/src/inbound/kafka_consumer/channel.rs
@@ -0,0 +1,190 @@
+//! Maps channel lifecycle events to search-index actions and processes them.
+
+use channels::domain::broker_events::{ChannelMacroEvent, ChannelTopicEvent};
+use macro_event_broker::MacroEvent as _;
+use opensearch_client::OpensearchClient;
+use sqlx::PgPool;
+use uuid::Uuid;
+
+use super::{EventOutcome, MAX_PROCESSING_ATTEMPTS, PROCESSING_RETRY_BASE_DELAY, retry_processing};
+use crate::process::channel::{process_channel_message_update, process_remove_channel_message};
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(super) enum ChannelIndexAction {
+ UpsertMessage { channel_id: Uuid, message_id: Uuid },
+ RemoveMessage { channel_id: Uuid, message_id: Uuid },
+ RemoveChannel { channel_id: Uuid },
+ Ignore,
+}
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(super) struct ChannelEventDescription {
+ pub(super) action: ChannelIndexAction,
+ pub(super) channel_id: Uuid,
+ pub(super) event_type: &'static str,
+}
+
+pub(super) fn describe_channel_event(event: &ChannelTopicEvent) -> ChannelEventDescription {
+ match event {
+ ChannelTopicEvent::Created(metadata) => ChannelEventDescription {
+ action: ChannelIndexAction::Ignore,
+ channel_id: metadata.channel_id,
+ event_type: "channel.created",
+ },
+ ChannelTopicEvent::Updated(metadata) => ChannelEventDescription {
+ action: ChannelIndexAction::Ignore,
+ channel_id: metadata.channel_id,
+ event_type: "channel.updated",
+ },
+ ChannelTopicEvent::Deleted(metadata) => ChannelEventDescription {
+ action: ChannelIndexAction::RemoveChannel {
+ channel_id: metadata.channel_id,
+ },
+ channel_id: metadata.channel_id,
+ event_type: "channel.deleted",
+ },
+ ChannelTopicEvent::MessagePosted(metadata) => ChannelEventDescription {
+ action: ChannelIndexAction::UpsertMessage {
+ channel_id: metadata.channel_id,
+ message_id: metadata.message_id,
+ },
+ channel_id: metadata.channel_id,
+ event_type: "channel.message_posted",
+ },
+ ChannelTopicEvent::MessagePatched(metadata) => ChannelEventDescription {
+ action: ChannelIndexAction::UpsertMessage {
+ channel_id: metadata.channel_id,
+ message_id: metadata.message_id,
+ },
+ channel_id: metadata.channel_id,
+ event_type: "channel.message_patched",
+ },
+ ChannelTopicEvent::MessageDeleted(metadata) => ChannelEventDescription {
+ action: ChannelIndexAction::RemoveMessage {
+ channel_id: metadata.channel_id,
+ message_id: metadata.message_id,
+ },
+ channel_id: metadata.channel_id,
+ event_type: "channel.message_deleted",
+ },
+ ChannelTopicEvent::MessageAttachmentCreated(metadata) => ChannelEventDescription {
+ action: ChannelIndexAction::UpsertMessage {
+ channel_id: metadata.channel_id,
+ message_id: metadata.message_id,
+ },
+ channel_id: metadata.channel_id,
+ event_type: "channel.message_attachment_created",
+ },
+ ChannelTopicEvent::MessageAttachmentRemoved(metadata) => ChannelEventDescription {
+ action: ChannelIndexAction::UpsertMessage {
+ channel_id: metadata.channel_id,
+ message_id: metadata.message_id,
+ },
+ channel_id: metadata.channel_id,
+ event_type: "channel.message_attachment_removed",
+ },
+ ChannelTopicEvent::ParticipantAdded(metadata) => ChannelEventDescription {
+ action: ChannelIndexAction::Ignore,
+ channel_id: metadata.channel_id,
+ event_type: "channel.participant_added",
+ },
+ ChannelTopicEvent::ParticipantRemoved(metadata) => ChannelEventDescription {
+ action: ChannelIndexAction::Ignore,
+ channel_id: metadata.channel_id,
+ event_type: "channel.participant_removed",
+ },
+ }
+}
+
+async fn process_channel_index_action(
+ db: &PgPool,
+ opensearch_client: &OpensearchClient,
+ action: ChannelIndexAction,
+) -> anyhow::Result<()> {
+ match action {
+ ChannelIndexAction::UpsertMessage {
+ channel_id,
+ message_id,
+ } => {
+ process_channel_message_update(opensearch_client, db, channel_id, message_id, None)
+ .await
+ }
+ ChannelIndexAction::RemoveMessage {
+ channel_id,
+ message_id,
+ } => {
+ process_remove_channel_message(opensearch_client, channel_id, Some(message_id), None)
+ .await
+ }
+ ChannelIndexAction::RemoveChannel { channel_id } => {
+ process_remove_channel_message(opensearch_client, channel_id, None, None).await
+ }
+ ChannelIndexAction::Ignore => Ok(()),
+ }
+}
+
+pub(super) async fn process_channel_event(
+ db: &PgPool,
+ opensearch_client: &OpensearchClient,
+ event: &ChannelMacroEvent,
+ partition: i32,
+ offset: i64,
+) -> EventOutcome {
+ let description = describe_channel_event(&event.event().event);
+ if description.action == ChannelIndexAction::Ignore {
+ tracing::trace!(
+ channel_id = %description.channel_id,
+ event_type = description.event_type,
+ partition,
+ offset,
+ "ignoring channel event without a search-index action"
+ );
+ return EventOutcome::Ignored;
+ }
+
+ let result = retry_processing(|attempt| async move {
+ tracing::trace!(
+ channel_id = %description.channel_id,
+ event_type = description.event_type,
+ partition,
+ offset,
+ attempt,
+ "processing channel search-index event"
+ );
+ process_channel_index_action(db, opensearch_client, description.action)
+ .await
+ .inspect_err(|error| {
+ if attempt < MAX_PROCESSING_ATTEMPTS {
+ let retry_delay =
+ PROCESSING_RETRY_BASE_DELAY * 2u32.pow(attempt.saturating_sub(1));
+ tracing::warn!(
+ error = ?error,
+ channel_id = %description.channel_id,
+ event_type = description.event_type,
+ partition,
+ offset,
+ attempt,
+ delay_secs = retry_delay.as_secs(),
+ "channel search-index processing failed, retrying"
+ );
+ }
+ })
+ })
+ .await;
+
+ match result {
+ Ok(()) => EventOutcome::Indexed,
+ Err(error) => {
+ tracing::error!(
+ error = ?error,
+ channel_id = %description.channel_id,
+ event_type = description.event_type,
+ partition,
+ offset,
+ attempts = MAX_PROCESSING_ATTEMPTS,
+ "dropping channel event after processing retries were exhausted"
+ );
+ EventOutcome::Dropped
+ }
+ }
+}
diff --git a/services/search_processing_service/src/inbound/kafka_consumer/project.rs b/services/search_processing_service/src/inbound/kafka_consumer/project.rs
new file mode 100644
index 0000000000..2619764745
--- /dev/null
+++ b/services/search_processing_service/src/inbound/kafka_consumer/project.rs
@@ -0,0 +1,198 @@
+//! Maps project lifecycle events to search-index reconciliations and processes them.
+
+use std::collections::HashSet;
+
+use macro_event_broker::MacroEvent as _;
+use opensearch_client::OpensearchClient;
+use projects::domain::events::{ProjectMacroEvent, ProjectTopicEvent};
+use sqlx::PgPool;
+use sqs_client::search::project::UpsertProject;
+
+use super::{EventOutcome, MAX_PROCESSING_ATTEMPTS, PROCESSING_RETRY_BASE_DELAY, retry_processing};
+use crate::process::project::upsert_project;
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(super) enum ProjectIndexAction {
+ Reconcile { project_ids: Vec<String> },
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(super) struct ProjectEventDescription<'a> {
+ pub(super) action: ProjectIndexAction,
+ pub(super) project_id: &'a str,
+ pub(super) event_type: &'static str,
+}
+
+pub(super) fn collect_project_ids<'a>(
+ project_ids: impl IntoIterator<Item = Option<&'a str>>,
+) -> Vec<String> {
+ let mut seen = HashSet::new();
+ let mut collected = Vec::new();
+
+ for project_id in project_ids.into_iter().flatten() {
+ if !project_id.is_empty() && seen.insert(project_id) {
+ collected.push(project_id.to_string());
+ }
+ }
+
+ collected
+}
+
+pub(super) fn describe_project_event(event: &ProjectTopicEvent) -> ProjectEventDescription<'_> {
+ match event {
+ ProjectTopicEvent::Created(metadata) => ProjectEventDescription {
+ action: ProjectIndexAction::Reconcile {
+ project_ids: collect_project_ids([
+ Some(metadata.project_id.as_str()),
+ metadata.parent_project_id.as_deref(),
+ ]),
+ },
+ project_id: &metadata.project_id,
+ event_type: "project.created",
+ },
+ ProjectTopicEvent::Updated(metadata) => ProjectEventDescription {
+ action: ProjectIndexAction::Reconcile {
+ project_ids: collect_project_ids([
+ Some(metadata.project_id.as_str()),
+ metadata.previous_parent_id.as_deref(),
+ metadata.parent_id.as_deref(),
+ ]),
+ },
+ project_id: &metadata.project_id,
+ event_type: "project.updated",
+ },
+ ProjectTopicEvent::Deleted(metadata) => ProjectEventDescription {
+ action: ProjectIndexAction::Reconcile {
+ project_ids: collect_project_ids(
+ metadata
+ .deleted_project_ids
+ .iter()
+ .map(|project_id| Some(project_id.as_str()))
+ .chain([metadata.parent_project_id.as_deref()]),
+ ),
+ },
+ project_id: &metadata.project_id,
+ event_type: "project.deleted",
+ },
+ ProjectTopicEvent::Restored(metadata) => ProjectEventDescription {
+ action: ProjectIndexAction::Reconcile {
+ project_ids: collect_project_ids(
+ metadata
+ .restored_project_ids
+ .iter()
+ .map(|project_id| Some(project_id.as_str()))
+ .chain([metadata.parent_project_id.as_deref()]),
+ ),
+ },
+ project_id: &metadata.project_id,
+ event_type: "project.restored",
+ },
+ ProjectTopicEvent::PermanentlyDeleted(metadata) => ProjectEventDescription {
+ action: ProjectIndexAction::Reconcile {
+ project_ids: collect_project_ids(
+ metadata
+ .purged_project_ids
+ .iter()
+ .map(|project_id| Some(project_id.as_str()))
+ .chain([metadata.parent_project_id.as_deref()]),
+ ),
+ },
+ project_id: &metadata.project_id,
+ event_type: "project.permanently_deleted",
+ },
+ ProjectTopicEvent::Uploaded(metadata) => ProjectEventDescription {
+ action: ProjectIndexAction::Reconcile {
+ project_ids: collect_project_ids(
+ metadata
+ .project_ids
+ .iter()
+ .map(|project_id| Some(project_id.as_str()))
+ .chain([metadata.parent_project_id.as_deref()]),
+ ),
+ },
+ project_id: &metadata.root_project_id,
+ event_type: "project.uploaded",
+ },
+ }
+}
+
+async fn process_project_index_action(
+ db: &PgPool,
+ opensearch_client: &OpensearchClient,
+ action: ProjectIndexAction,
+) -> anyhow::Result<()> {
+ match action {
+ ProjectIndexAction::Reconcile { project_ids } => {
+ for project_id in project_ids {
+ upsert_project(
+ opensearch_client,
+ db,
+ &UpsertProject {
+ project_id,
+ index_override: None,
+ },
+ )
+ .await?;
+ }
+ Ok(())
+ }
+ }
+}
+
+pub(super) async fn process_project_event(
+ db: &PgPool,
+ opensearch_client: &OpensearchClient,
+ event: &ProjectMacroEvent,
+ partition: i32,
+ offset: i64,
+) -> EventOutcome {
+ let description = describe_project_event(&event.event().event);
+ let result = retry_processing(|attempt| {
+ let action = description.action.clone();
+ async move {
+ tracing::trace!(
+ project_id = description.project_id,
+ event_type = description.event_type,
+ partition,
+ offset,
+ attempt,
+ "processing project search-index event"
+ );
+ process_project_index_action(db, opensearch_client, action)
+ .await
+ .inspect_err(|error| {
+ if attempt < MAX_PROCESSING_ATTEMPTS {
+ let retry_delay =
+ PROCESSING_RETRY_BASE_DELAY * 2u32.pow(attempt.saturating_sub(1));
+ tracing::warn!(
+ error = ?error,
+ project_id = description.project_id,
+ event_type = description.event_type,
+ partition,
+ offset,
+ attempt,
+ delay_secs = retry_delay.as_secs(),
+ "project search-index processing failed, retrying"
+ );
+ }
+ })
+ }
+ })
+ .await;
+
+ match result {
+ Ok(()) => EventOutcome::Indexed,
+ Err(error) => {
+ tracing::error!(
+ error = ?error,
+ project_id = description.project_id,
+ event_type = description.event_type,
+ partition,
+ offset,
+ attempts = MAX_PROCESSING_ATTEMPTS,
+ "dropping project event after processing retries were exhausted"
+ );
+ EventOutcome::Dropped
+ }
+ }
+}
diff --git a/services/search_processing_service/src/inbound/kafka_consumer/test.rs b/services/search_processing_service/src/inbound/kafka_consumer/test.rs
index 421e5a0dae..dd55691cae 100644
--- a/services/search_processing_service/src/inbound/kafka_consumer/test.rs
+++ b/services/search_processing_service/src/inbound/kafka_consumer/test.rs
@@ -1,10 +1,10 @@
use std::sync::Arc;
use std::sync::atomic::{AtomicU32, Ordering};
-use call::domain::events::{
+use ::call::domain::events::{
CallArchiveReason, CallRecordArchivedMetadata, CallRecordDeletedMetadata,
CallRecordSummarizedMetadata, CallRecordUpdatedMetadata, CallRecordingReadyMetadata,
- CallStartedMetadata,
+ CallStartedMetadata, CallTopicEvent,
};
use channels::domain::{
broker_events::{
@@ -18,14 +18,30 @@ use channels::domain::{
};
use chrono::Utc;
use macro_event_broker::{Event, EventBrokerError, MacroEvent as _, MessageParts};
-use macro_event_topics::{MacroCallsTopic, MacroChannelsTopic, Topic as _};
+use macro_event_topics::{MacroCallsTopic, MacroChannelsTopic, MacroProjectsTopic, Topic as _};
use macro_user_id::user_id::MacroUserIdStr;
+use projects::domain::events::{
+ ProjectCreatedMetadata, ProjectDeletedMetadata, ProjectPermanentlyDeletedMetadata,
+ ProjectRestoredMetadata, ProjectTopicEvent, ProjectUpdatedMetadata, ProjectUploadedMetadata,
+};
+use uuid::Uuid;
-use super::*;
+use super::{
+ call::{CallEventDescription, CallIndexAction, describe_call_event},
+ channel::{ChannelEventDescription, ChannelIndexAction, describe_channel_event},
+ project::{
+ ProjectEventDescription, ProjectIndexAction, collect_project_ids, describe_project_event,
+ },
+ *,
+};
const CALL_ID: Uuid = Uuid::from_u128(1);
const CHANNEL_ID: Uuid = Uuid::from_u128(2);
const MESSAGE_ID: Uuid = Uuid::from_u128(3);
+const PROJECT_ID: &str = "project-root";
+const CHILD_PROJECT_ID: &str = "project-child";
+const PARENT_PROJECT_ID: &str = "project-parent";
+const NEW_PARENT_PROJECT_ID: &str = "project-new-parent";
struct TestMessage {
topic: &'static str,
@@ -114,7 +130,7 @@ fn recording_ready_event() -> CallTopicEvent {
fn encoded_message<E: serde::Serialize>(
topic: &'static str,
- key: Uuid,
+ key: impl ToString,
event: Event<E>,
) -> TestMessage {
TestMessage {
@@ -287,15 +303,168 @@ fn channel_event_cases() -> Vec<(ChannelTopicEvent, ChannelEventDescription)> {
]
}
+fn project_event_cases() -> Vec<(ProjectTopicEvent, ProjectEventDescription<'static>)> {
+ let owner = user_id();
+
+ vec![
+ (
+ ProjectTopicEvent::Created(ProjectCreatedMetadata {
+ project_id: PROJECT_ID.to_string(),
+ owner: owner.clone(),
+ name: "Project".to_string(),
+ parent_project_id: Some(PARENT_PROJECT_ID.to_string()),
+ created_at: Some(Utc::now()),
+ }),
+ ProjectEventDescription {
+ action: ProjectIndexAction::Reconcile {
+ project_ids: vec![PROJECT_ID.to_string(), PARENT_PROJECT_ID.to_string()],
+ },
+ project_id: PROJECT_ID,
+ event_type: "project.created",
+ },
+ ),
+ (
+ ProjectTopicEvent::Updated(ProjectUpdatedMetadata {
+ project_id: PROJECT_ID.to_string(),
+ owner: owner.clone(),
+ actor_user_id: Some(user_id()),
+ name: Some("Renamed project".to_string()),
+ previous_parent_id: Some(PARENT_PROJECT_ID.to_string()),
+ parent_id: Some(NEW_PARENT_PROJECT_ID.to_string()),
+ share_permission_updated: false,
+ }),
+ ProjectEventDescription {
+ action: ProjectIndexAction::Reconcile {
+ project_ids: vec![
+ PROJECT_ID.to_string(),
+ PARENT_PROJECT_ID.to_string(),
+ NEW_PARENT_PROJECT_ID.to_string(),
+ ],
+ },
+ project_id: PROJECT_ID,
+ event_type: "project.updated",
+ },
+ ),
+ (
+ ProjectTopicEvent::Updated(ProjectUpdatedMetadata {
+ project_id: PROJECT_ID.to_string(),
+ owner: owner.clone(),
+ actor_user_id: Some(user_id()),
+ name: None,
+ previous_parent_id: Some(PARENT_PROJECT_ID.to_string()),
+ parent_id: Some(String::new()),
+ share_permission_updated: true,
+ }),
+ ProjectEventDescription {
+ action: ProjectIndexAction::Reconcile {
+ project_ids: vec![PROJECT_ID.to_string(), PARENT_PROJECT_ID.to_string()],
+ },
+ project_id: PROJECT_ID,
+ event_type: "project.updated",
+ },
+ ),
+ (
+ ProjectTopicEvent::Deleted(ProjectDeletedMetadata {
+ project_id: PROJECT_ID.to_string(),
+ owner: owner.clone(),
+ actor_user_id: Some(user_id()),
+ parent_project_id: Some(PARENT_PROJECT_ID.to_string()),
+ deleted_project_ids: vec![
+ PROJECT_ID.to_string(),
+ CHILD_PROJECT_ID.to_string(),
+ CHILD_PROJECT_ID.to_string(),
+ PARENT_PROJECT_ID.to_string(),
+ ],
+ deleted_document_ids: vec!["document-id".to_string()],
+ deleted_chat_ids: vec!["chat-id".to_string()],
+ }),
+ ProjectEventDescription {
+ action: ProjectIndexAction::Reconcile {
+ project_ids: vec![
+ PROJECT_ID.to_string(),
+ CHILD_PROJECT_ID.to_string(),
+ PARENT_PROJECT_ID.to_string(),
+ ],
+ },
+ project_id: PROJECT_ID,
+ event_type: "project.deleted",
+ },
+ ),
+ (
+ ProjectTopicEvent::Restored(ProjectRestoredMetadata {
+ project_id: PROJECT_ID.to_string(),
+ owner: owner.clone(),
+ actor_user_id: Some(user_id()),
+ parent_project_id: Some(PARENT_PROJECT_ID.to_string()),
+ restored_project_ids: vec![PROJECT_ID.to_string(), CHILD_PROJECT_ID.to_string()],
+ }),
+ ProjectEventDescription {
+ action: ProjectIndexAction::Reconcile {
+ project_ids: vec![
+ PROJECT_ID.to_string(),
+ CHILD_PROJECT_ID.to_string(),
+ PARENT_PROJECT_ID.to_string(),
+ ],
+ },
+ project_id: PROJECT_ID,
+ event_type: "project.restored",
+ },
+ ),
+ (
+ ProjectTopicEvent::PermanentlyDeleted(ProjectPermanentlyDeletedMetadata {
+ project_id: PROJECT_ID.to_string(),
+ owner: owner.clone(),
+ actor_user_id: Some(user_id()),
+ parent_project_id: Some(PARENT_PROJECT_ID.to_string()),
+ purged_project_ids: vec![PROJECT_ID.to_string(), CHILD_PROJECT_ID.to_string()],
+ purged_document_ids: vec!["document-id".to_string()],
+ purged_chat_ids: vec!["chat-id".to_string()],
+ }),
+ ProjectEventDescription {
+ action: ProjectIndexAction::Reconcile {
+ project_ids: vec![
+ PROJECT_ID.to_string(),
+ CHILD_PROJECT_ID.to_string(),
+ PARENT_PROJECT_ID.to_string(),
+ ],
+ },
+ project_id: PROJECT_ID,
+ event_type: "project.permanently_deleted",
+ },
+ ),
+ (
+ ProjectTopicEvent::Uploaded(ProjectUploadedMetadata {
+ root_project_id: PROJECT_ID.to_string(),
+ owner,
+ name: "Uploaded project".to_string(),
+ parent_project_id: Some(PARENT_PROJECT_ID.to_string()),
+ project_ids: vec![PROJECT_ID.to_string(), CHILD_PROJECT_ID.to_string()],
+ }),
+ ProjectEventDescription {
+ action: ProjectIndexAction::Reconcile {
+ project_ids: vec![
+ PROJECT_ID.to_string(),
+ CHILD_PROJECT_ID.to_string(),
+ PARENT_PROJECT_ID.to_string(),
+ ],
+ },
+ project_id: PROJECT_ID,
+ event_type: "project.uploaded",
+ },
+ ),
+ ]
+}
+
#[test]
fn subscribes_to_declared_search_processing_topics_with_durable_group() {
assert_eq!(
SearchProcessingConsumerGroup::GROUP_NAME,
"search-processing-service"
);
- let topics = SearchProcessingBrokerEvent::topics();
+ let topics = DeclaredMacroEvent::topics();
assert!(topics.contains(&MacroCallsTopic::TOPIC_STR));
assert!(topics.contains(&MacroChannelsTopic::TOPIC_STR));
+ assert!(topics.contains(&MacroProjectsTopic::TOPIC_STR));
}
#[test]
@@ -365,6 +534,32 @@ fn maps_all_channel_lifecycle_events_to_index_actions() {
}
}
+#[test]
+fn maps_all_project_lifecycle_events_to_reconciliation_actions() {
+ let cases = project_event_cases();
+ assert_eq!(cases.len(), 7);
+
+ for (event, expected) in cases {
+ let serialized = serde_json::to_value(&event).expect("serializable project event");
+ assert_eq!(serialized["event_type"], expected.event_type);
+ assert_eq!(describe_project_event(&event), expected);
+ }
+}
+
+#[test]
+fn project_id_collection_is_stable_and_drops_missing_or_empty_ids() {
+ assert_eq!(
+ collect_project_ids([
+ Some(PROJECT_ID),
+ None,
+ Some(""),
+ Some(CHILD_PROJECT_ID),
+ Some(PROJECT_ID),
+ ]),
+ vec![PROJECT_ID.to_string(), CHILD_PROJECT_ID.to_string()]
+ );
+}
+
#[test]
fn channel_envelope_decodes_round_trip() {
let event = ChannelTopicEvent::Deleted(ChannelDeletedMetadata {
@@ -377,14 +572,72 @@ fn channel_envelope_decodes_round_trip() {
Event::new(event.clone()),
);
- let decoded = SearchProcessingBrokerEvent::decode(&message).expect("decodable channel event");
- let SearchProcessingBrokerEvent::ChannelMacroEvent(decoded_event) = decoded else {
+ let decoded = DeclaredMacroEvent::decode(&message).expect("decodable channel event");
+ let DeclaredMacroEvent::ChannelMacroEvent(decoded_event) = decoded else {
panic!("expected channel event");
};
assert_eq!(decoded_event.key(), CHANNEL_ID.to_string());
assert_eq!(decoded_event.event().event, event);
}
+#[test]
+fn project_envelope_decodes_round_trip_with_string_key() {
+ let event = ProjectTopicEvent::Restored(ProjectRestoredMetadata {
+ project_id: PROJECT_ID.to_string(),
+ owner: user_id(),
+ actor_user_id: Some(user_id()),
+ parent_project_id: Some(PARENT_PROJECT_ID.to_string()),
+ restored_project_ids: vec![PROJECT_ID.to_string(), CHILD_PROJECT_ID.to_string()],
+ });
+ let message = encoded_message(
+ MacroProjectsTopic::TOPIC_STR,
+ PROJECT_ID,
+ Event::new(event.clone()),
+ );
+
+ let decoded = DeclaredMacroEvent::decode(&message).expect("decodable project event");
+ let DeclaredMacroEvent::ProjectMacroEvent(decoded_event) = decoded else {
+ panic!("expected project event");
+ };
+ assert_eq!(decoded_event.key(), PROJECT_ID);
+ assert_eq!(decoded_event.event().event, event);
+}
+
+#[tokio::test]
+async fn unsupported_project_schema_message_is_commit_safe() {
+ let event = ProjectTopicEvent::Restored(ProjectRestoredMetadata {
+ project_id: PROJECT_ID.to_string(),
+ owner: user_id(),
+ actor_user_id: Some(user_id()),
+ parent_project_id: None,
+ restored_project_ids: vec![PROJECT_ID.to_string()],
+ });
+ let message = encoded_message(
+ MacroProjectsTopic::TOPIC_STR,
+ PROJECT_ID,
+ Event::with_schema_version(event, 2),
+ );
+ let decoded = attach_event_coordinates(DeclaredMacroEvent::decode(&message), 4, 30);
+ let (sender, mut receiver) = mpsc::channel(1);
+
+ match handoff_decoded(&sender, decoded).await {
+ HandoffOutcome::MalformedRecord(EventBrokerError::UnsupportedSchemaVersion {
+ topic,
+ expected,
+ actual,
+ }) => {
+ assert_eq!(topic, MacroProjectsTopic::TOPIC_STR);
+ assert_eq!(expected, 1);
+ assert_eq!(actual, 2);
+ }
+ outcome => panic!("expected malformed project record, got {outcome:?}"),
+ }
+ assert!(matches!(
+ receiver.try_recv(),
+ Err(mpsc::error::TryRecvError::Empty)
+ ));
+}
+
#[tokio::test]
async fn unsupported_channel_schema_message_is_commit_safe() {
let event = ChannelTopicEvent::Deleted(ChannelDeletedMetadata {
@@ -396,7 +649,7 @@ async fn unsupported_channel_schema_message_is_commit_safe() {
CHANNEL_ID,
Event::with_schema_version(event, 2),
);
- let decoded = attach_event_coordinates(SearchProcessingBrokerEvent::decode(&message), 2, 20);
+ let decoded = attach_event_coordinates(DeclaredMacroEvent::decode(&message), 2, 20);
let (sender, mut receiver) = mpsc::channel(1);
match handoff_decoded(&sender, decoded).await {
@@ -424,8 +677,7 @@ async fn malformed_missing_key_and_unsupported_schema_messages_are_commit_safe()
key: Some(CALL_ID.to_string()),
payload: Some(b"not json".to_vec()),
};
- let malformed =
- attach_event_coordinates(SearchProcessingBrokerEvent::decode(&malformed), 1, 10);
+ let malformed = attach_event_coordinates(DeclaredMacroEvent::decode(&malformed), 1, 10);
assert!(matches!(malformed, Err(EventBrokerError::Serialization(_))));
let missing_key = TestMessage {
@@ -438,8 +690,7 @@ async fn malformed_missing_key_and_unsupported_schema_messages_are_commit_safe()
)
.payload,
};
- let missing_key =
- attach_event_coordinates(SearchProcessingBrokerEvent::decode(&missing_key), 1, 11);
+ let missing_key = attach_event_coordinates(DeclaredMacroEvent::decode(&missing_key), 1, 11);
assert!(matches!(
missing_key,
Err(EventBrokerError::MissingMessageKey)
@@ -450,8 +701,7 @@ async fn malformed_missing_key_and_unsupported_schema_messages_are_commit_safe()
CALL_ID,
Event::with_schema_version(archived_event(), 2),
);
- let unsupported =
- attach_event_coordinates(SearchProcessingBrokerEvent::decode(&unsupported), 1, 12);
+ let unsupported = attach_event_coordinates(DeclaredMacroEvent::decode(&unsupported), 1, 12);
assert!(matches!(
unsupported,
Err(EventBrokerError::UnsupportedSchemaVersion {
@@ -482,7 +732,7 @@ async fn successful_handoff_carries_event_partition_and_offset() {
CALL_ID,
Event::new(event.clone()),
);
- let decoded = attach_event_coordinates(SearchProcessingBrokerEvent::decode(&message), 3, 42);
+ let decoded = attach_event_coordinates(DeclaredMacroEvent::decode(&message), 3, 42);
let (sender, mut receiver) = mpsc::channel(1);
assert!(matches!(
@@ -493,7 +743,7 @@ async fn successful_handoff_carries_event_partition_and_offset() {
let received = receiver.recv().await.expect("handed-off event");
assert_eq!(received.partition, 3);
assert_eq!(received.offset, 42);
- let SearchProcessingBrokerEvent::CallMacroEvent(received_event) = received.event else {
+ let DeclaredMacroEvent::CallMacroEvent(received_event) = received.event else {
panic!("expected call event");
};
assert_eq!(received_event.event().event, event);
@@ -506,7 +756,7 @@ async fn closed_worker_channel_leaves_the_current_message_uncommitted() {
CALL_ID,
Event::new(archived_event()),
);
- let decoded = attach_event_coordinates(SearchProcessingBrokerEvent::decode(&message), 3, 42);
+ let decoded = attach_event_coordinates(DeclaredMacroEvent::decode(&message), 3, 42);
let (sender, receiver) = mpsc::channel(1);
drop(receiver);
diff --git a/services/search_processing_service/src/outbound/source.rs b/services/search_processing_service/src/outbound/source.rs
index 50d960ef91..694906bbae 100644
--- a/services/search_processing_service/src/outbound/source.rs
+++ b/services/search_processing_service/src/outbound/source.rs
@@ -396,6 +396,8 @@ impl BackfillSource for PgBackfillSource {
})
});
let rows_consumed = batch.len();
+ // This SQS message is intentionally limited to backfills, which may
+ // target an alternate OpenSearch index through `index_override`.
let messages: Vec<SearchQueueMessage> = batch
.into_iter()
.map(|p| {
diff --git a/services/search_processing_service/src/process/mod.rs b/services/search_processing_service/src/process/mod.rs
index 6702c24158..7a21f79c7e 100644
--- a/services/search_processing_service/src/process/mod.rs
+++ b/services/search_processing_service/src/process/mod.rs
@@ -4,7 +4,7 @@ mod chat;
pub mod context;
mod document;
mod email;
-mod project;
+pub(crate) mod project;
mod properties;
mod user;
pub mod worker;
@@ -153,9 +153,6 @@ pub async fn process_message(
SearchQueueMessage::UpsertProject(message) => {
project::upsert_project(&ctx.opensearch_client, &ctx.db, &message).await?;
}
- SearchQueueMessage::RemoveProject(message) => {
- project::remove_project(&ctx.opensearch_client, &message).await?;
- }
}
ctx.worker.cleanup_message(message).await?;
diff --git a/services/search_processing_service/src/process/project.rs b/services/search_processing_service/src/process/project.rs
index 8bc0e23a34..28ed839d17 100644
--- a/services/search_processing_service/src/process/project.rs
+++ b/services/search_processing_service/src/process/project.rs
@@ -6,7 +6,7 @@ use opensearch_client::{
upsert::{project::UpsertProjectArgs, properties::IndexedProperty},
};
use properties::outbound::entity_properties_get_query::get_entity_properties_for_index;
-use sqs_client::search::project::{RemoveProject, UpsertProject};
+use sqs_client::search::project::UpsertProject;
/// Fetch the project's indexed properties, flattened for the search index.
async fn fetch_indexed_properties(
@@ -86,19 +86,3 @@ pub async fn upsert_project(
Ok(())
}
-
-/// Handles the removal of a project from the opensearch index
-#[tracing::instrument(skip(opensearch_client), err)]
-pub async fn remove_project(
- opensearch_client: &OpensearchClient,
- message: &RemoveProject,
-) -> anyhow::Result<()> {
- opensearch_client
- .delete_project(
- message.project_id.as_str(),
- message.index_override.as_deref(),
- )
- .await?;
-
- Ok(())
-}
diff --git a/tooling/just/rust.just b/tooling/just/rust.just
index 3648b4bbce..7c0df453a2 100644
--- a/tooling/just/rust.just
+++ b/tooling/just/rust.just
@@ -97,7 +97,6 @@ setup_test_envs:
echo "DATABASE_URL={{ DATABASE_URL }}" >> ./crates/projects/.env
echo "DATABASE_URL={{ DATABASE_URL }}" >> ./crates/foreign_entity/.env
echo "DATABASE_URL={{ DATABASE_URL }}" >> ./crates/stream/.env
- echo "DATABASE_URL={{ DATABASE_URL }}" >> ./crates/macro_project_utils/.env
echo "DATABASE_URL={{ DATABASE_URL }}" >> ./crates/entity_access/.env
echo "DATABASE_URL={{ DATABASE_URL }}" >> ./crates/github/.env
echo "DATABASE_URL={{ DATABASE_URL }}" >> ./crates/referral/.env