Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(search_processing_service): documents kafka (#5313)
Will Hutchinsond0295c70156c
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.jsonindex b8c11e1a33..e3f6adedb9 100644--- a/.github/workspace-dep-closures.json+++ b/.github/workspace-dep-closures.json@@ -1297,24 +1297,34 @@"crates/attachment","crates/authentication_service_client","crates/bot_id",+ "crates/call","crates/channel_sender",+ "crates/channels","crates/chat",+ "crates/connection",+ "crates/connection_gateway_client",+ "crates/connection_gateway_models",+ "crates/contacts","crates/cowlike","crates/crm","crates/document_sub_type",+ "crates/documents","crates/dynamodb_client","crates/email","crates/email_utils",+ "crates/embedding","crates/entity_access","crates/entity_access_db_utils","crates/entity_access_management","crates/entity_mutation","crates/filter_ast",+ "crates/foreign_entity","crates/frecency","crates/invite_email","crates/item_filters","crates/kafka_util","crates/last_online_tracker",+ "crates/lexical_client","crates/macro_auth","crates/macro_authorization","crates/macro_aws_config",@@ -1327,6 +1337,7 @@"crates/macro_queues","crates/macro_service_urls","crates/macro_sha_count_client",+ "crates/macro_sync_service_jwt","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -1345,9 +1356,14 @@"crates/models_pagination","crates/models_permissions","crates/models_properties",+ "crates/models_search",+ "crates/models_search_cursor",+ "crates/models_soup","crates/models_team","crates/non_empty","crates/notification",+ "crates/opensearch_client",+ "crates/opensearch_query_builder","crates/projects","crates/prompt","crates/properties",@@ -1355,8 +1371,14 @@"crates/remote_env_var","crates/roles_and_permissions","crates/s3_key",+ "crates/secretsmanager_client",+ "crates/share_permission_db_utils","crates/sqs_client",+ "crates/sqs_worker",+ "crates/sync_service",+ "crates/sync_service_client","crates/system_properties",+ "crates/task_dedup","crates/unfurl","crates/user_quota","crates/workspace-hack",@@ -4564,32 +4586,44 @@"crates/attachment","crates/authentication_service_client","crates/bot_id",+ "crates/call","crates/channel_sender",+ "crates/channels",+ "crates/connection",+ "crates/connection_gateway_client",+ "crates/connection_gateway_models",+ "crates/contacts","crates/cowlike","crates/crm","crates/document_sub_type",+ "crates/documents","crates/dynamodb_client","crates/email","crates/email_utils",+ "crates/embedding","crates/entity_access","crates/entity_access_db_utils","crates/entity_access_management","crates/entity_mutation","crates/filter_ast",+ "crates/foreign_entity","crates/frecency","crates/invite_email","crates/item_filters","crates/kafka_util","crates/last_online_tracker",+ "crates/lexical_client","crates/macro_auth","crates/macro_authorization","crates/macro_aws_config",+ "crates/macro_entrypoint","crates/macro_env","crates/macro_env_var","crates/macro_event_broker","crates/macro_event_topics","crates/macro_service_urls","crates/macro_sha_count_client",+ "crates/macro_sync_service_jwt","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -4603,19 +4637,31 @@"crates/models_bulk_upload","crates/models_email","crates/models_entity_access_management",+ "crates/models_opensearch","crates/models_pagination","crates/models_permissions","crates/models_properties",+ "crates/models_search",+ "crates/models_search_cursor",+ "crates/models_soup","crates/non_empty","crates/notification",+ "crates/opensearch_client",+ "crates/opensearch_query_builder","crates/projects","crates/prompt","crates/properties","crates/rate_limit","crates/remote_env_var","crates/s3_key",+ "crates/secretsmanager_client",+ "crates/share_permission_db_utils","crates/sqs_client",+ "crates/sqs_worker",+ "crates/sync_service",+ "crates/sync_service_client","crates/system_properties",+ "crates/task_dedup","crates/unfurl","crates/workspace-hack"],@@ -4898,16 +4944,19 @@"crates/cowlike","crates/crm","crates/document_sub_type",+ "crates/documents","crates/dynamodb_client","crates/email","crates/email_db_client","crates/email_utils",+ "crates/embedding","crates/ensure_exists","crates/entity_access","crates/entity_access_db_utils","crates/entity_access_management","crates/entity_mutation","crates/filter_ast",+ "crates/foreign_entity","crates/frecency","crates/invite_email","crates/ip_extractor",@@ -4931,6 +4980,7 @@"crates/macro_queues","crates/macro_service_urls","crates/macro_sha_count_client",+ "crates/macro_sync_service_jwt","crates/macro_user_id","crates/macro_uuid","crates/mention_utils",@@ -4965,10 +5015,14 @@"crates/roles_and_permissions","crates/s3_client","crates/s3_key",+ "crates/secretsmanager_client","crates/share_permission_db_utils","crates/sqs_client","crates/sqs_worker",+ "crates/sync_service",+ "crates/sync_service_client","crates/system_properties",+ "crates/task_dedup","crates/unfurl","crates/user_quota","crates/workspace-hack",@@ -5129,61 +5183,30 @@"crates/ai_toolset","crates/ai_usage","crates/attachment",- "crates/authentication_service_client","crates/bot_id",- "crates/channel_sender","crates/cowlike",- "crates/crm","crates/document_storage_service_client","crates/document_sub_type",- "crates/email",- "crates/email_utils",- "crates/entity_access",- "crates/entity_access_db_utils",- "crates/entity_access_management",- "crates/entity_mutation",- "crates/filter_ast",- "crates/frecency",- "crates/invite_email",- "crates/item_filters",- "crates/kafka_util",- "crates/last_online_tracker","crates/macro_auth","crates/macro_authorization",- "crates/macro_aws_config","crates/macro_entrypoint","crates/macro_env","crates/macro_env_var",- "crates/macro_event_broker",- "crates/macro_event_topics",- "crates/macro_queues","crates/macro_service_urls","crates/macro_user_id","crates/macro_uuid",- "crates/mention_utils","crates/model","crates/model-entity","crates/model-error-response","crates/model_file_type",- "crates/model_notifications","crates/model_user",- "crates/models_ai_projection","crates/models_bulk_upload",- "crates/models_email",- "crates/models_entity_access_management","crates/models_pagination","crates/models_permissions",- "crates/models_properties","crates/non_empty",- "crates/notification","crates/prompt",- "crates/properties",- "crates/rate_limit","crates/remote_env_var","crates/s3_key",- "crates/sqs_client",- "crates/system_properties",- "crates/unfurl","crates/workspace-hack","services/search_upload_handler"],diff --git a/Cargo.lock b/Cargo.lockindex 02686eeb9a..a67c1b90a3 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -3997,6 +3997,7 @@ dependencies = ["aws_lambda_events","chat","chrono",+ "documents","futures","lambda_runtime","macro_aws_config",@@ -11417,11 +11418,13 @@ dependencies = ["anyhow","async-recursion","aws-sdk-s3",+ "aws-sdk-sqs","axum","axum-extra","base64 0.22.1","chrono","document_sub_type",+ "documents","dotenvy","dynamodb_client","entity_access",@@ -12893,6 +12896,7 @@ dependencies = ["chrono","comms_db_client","document_sub_type",+ "documents","doppler-config","email_db_client","ensure_exists",@@ -13020,17 +13024,13 @@ name = "search_upload_handler"version = "0.1.0"dependencies = ["anyhow",- "aws-sdk-sqs","aws_lambda_events","document_storage_service_client","lambda_runtime",- "macro_aws_config","macro_entrypoint","macro_env_var",- "macro_queues","macro_service_urls","s3_key",- "sqs_client","tokio","tracing","workspace-hack",diff --git a/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentContentUploadedMetadata.ts b/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentContentUploadedMetadata.tsnew file mode 100644index 0000000000..0561091f1a--- /dev/null+++ b/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentContentUploadedMetadata.ts@@ -0,0 +1,25 @@+/**+ * Generated by orval v7.21.0 🍺+ * Do not edit manually.+ * document_storage_service+ * OpenAPI spec version: 0.1.0+ */+import type { DocumentContentUploadedMetadataDocumentVersionId } from './documentContentUploadedMetadataDocumentVersionId';+import type { FileType } from './fileType';+import type { MacroUserIdStr } from './macroUserIdStr';++/**+ * Metadata for [`DocumentTopicEvent::ContentUploaded`].+ */+export interface DocumentContentUploadedMetadata {+ /** The id of the document whose stored bytes changed. */+ document_id: string;+ /** Version written, or the converted-file marker; `None` for unversioned+writes (mirrors `SearchExtractorMessage::document_version_id`). */+ document_version_id?: DocumentContentUploadedMetadataDocumentVersionId;+ /** File type of the uploaded object (may differ from the document's own+type, e.g. `pdf` for the converted rendition of a docx). */+ file_type: FileType;+ /** The owner of the document (used by the extractor to resolve S3 keys). */+ owner: MacroUserIdStr;+}diff --git a/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentContentUploadedMetadataDocumentVersionId.ts b/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentContentUploadedMetadataDocumentVersionId.tsnew file mode 100644index 0000000000..48cd42890c--- /dev/null+++ b/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentContentUploadedMetadataDocumentVersionId.ts@@ -0,0 +1,12 @@+/**+ * Generated by orval v7.21.0 🍺+ * Do not edit manually.+ * document_storage_service+ * OpenAPI spec version: 0.1.0+ */++/**+ * Version written, or the converted-file marker; `None` for unversioned+writes (mirrors `SearchExtractorMessage::document_version_id`).+ */+export type DocumentContentUploadedMetadataDocumentVersionId = string | null;diff --git a/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentPurgedMetadata.ts b/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentPurgedMetadata.tsnew file mode 100644index 0000000000..f2c92c3bd7--- /dev/null+++ b/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentPurgedMetadata.ts@@ -0,0 +1,14 @@+/**+ * Generated by orval v7.21.0 🍺+ * Do not edit manually.+ * document_storage_service+ * OpenAPI spec version: 0.1.0+ */++/**+ * Metadata for [`DocumentTopicEvent::Purged`].+ */+export interface DocumentPurgedMetadata {+ /** The id of the hard-deleted document. */+ document_id: string;+}diff --git a/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentSyncContentUpdatedMetadata.ts b/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentSyncContentUpdatedMetadata.tsnew file mode 100644index 0000000000..42bd7a7400--- /dev/null+++ b/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentSyncContentUpdatedMetadata.ts@@ -0,0 +1,20 @@+/**+ * Generated by orval v7.21.0 🍺+ * Do not edit manually.+ * document_storage_service+ * OpenAPI spec version: 0.1.0+ */+import type { DocumentSyncContentUpdatedMetadataDocumentVersionId } from './documentSyncContentUpdatedMetadataDocumentVersionId';+import type { FileType } from './fileType';++/**+ * Metadata for [`DocumentTopicEvent::SyncContentUpdated`].+ */+export interface DocumentSyncContentUpdatedMetadata {+ /** The id of the live-collab document whose content changed. */+ document_id: string;+ /** Version marker for the sync snapshot, when the caller supplies one. */+ document_version_id?: DocumentSyncContentUpdatedMetadataDocumentVersionId;+ /** File type of the sync document (markdown today). */+ file_type: FileType;+}diff --git a/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentSyncContentUpdatedMetadataDocumentVersionId.ts b/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentSyncContentUpdatedMetadataDocumentVersionId.tsnew file mode 100644index 0000000000..f241d7be3c--- /dev/null+++ b/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentSyncContentUpdatedMetadataDocumentVersionId.ts@@ -0,0 +1,11 @@+/**+ * Generated by orval v7.21.0 🍺+ * Do not edit manually.+ * document_storage_service+ * OpenAPI spec version: 0.1.0+ */++/**+ * Version marker for the sync snapshot, when the caller supplies one.+ */+export type DocumentSyncContentUpdatedMetadataDocumentVersionId = string | null;diff --git a/apps/web/src/lib/service-clients/service-storage/generated/schemas/index.ts b/apps/web/src/lib/service-clients/service-storage/generated/schemas/index.tsindex a3788870ad..7193199afd 100644--- a/apps/web/src/lib/service-clients/service-storage/generated/schemas/index.ts+++ b/apps/web/src/lib/service-clients/service-storage/generated/schemas/index.ts@@ -403,6 +403,8 @@ export * from './documentContent';export * from './documentContentLocation';export * from './documentContentLocationProperty';export * from './documentContentState';+export * from './documentContentUploadedMetadata';+export * from './documentContentUploadedMetadataDocumentVersionId';export * from './documentFilters';export * from './documentFiltersImportance';export * from './documentFiltersIsEmailAttachment';@@ -447,6 +449,7 @@ export * from './documentPreviewOneOfFourAllOfType';export * from './documentPreviewOneOfSeven';export * from './documentPreviewOneOfSevenAllOf';export * from './documentPreviewOneOfSevenAllOfType';+export * from './documentPurgedMetadata';export * from './documentResponse';export * from './documentResponseMetadata';export * from './documentResponseMetadataBranchedFromId';@@ -465,6 +468,8 @@ export * from './documentResponseMetadataWithContentAllOfTeamTaskId';export * from './documentResponsePresignedUrl';export * from './documentStorageServiceApiVersion';export * from './documentSubType';+export * from './documentSyncContentUpdatedMetadata';+export * from './documentSyncContentUpdatedMetadataDocumentVersionId';export * from './documentTeamShareResponse';export * from './documentTeamShareResponseTeamId';export * from './editAnchorRequest';diff --git a/apps/web/src/lib/service-clients/service-storage/openapi.json b/apps/web/src/lib/service-clients/service-storage/openapi.jsonindex a44f839c81..5858e9f80f 100644--- a/apps/web/src/lib/service-clients/service-storage/openapi.json+++ b/apps/web/src/lib/service-clients/service-storage/openapi.json@@ -15046,6 +15046,29 @@"description": "API-visible content lifecycle state derived from current document metadata.","enum": ["unknown", "pending", "ready"]},+ "DocumentContentUploadedMetadata": {+ "type": "object",+ "description": "Metadata for [`DocumentTopicEvent::ContentUploaded`].",+ "required": ["document_id", "owner", "file_type"],+ "properties": {+ "document_id": {+ "type": "string",+ "description": "The id of the document whose stored bytes changed."+ },+ "document_version_id": {+ "type": ["string", "null"],+ "description": "Version written, or the converted-file marker; `None` for unversioned\nwrites (mirrors `SearchExtractorMessage::document_version_id`)."+ },+ "file_type": {+ "$ref": "#/components/schemas/FileType",+ "description": "File type of the uploaded object (may differ from the document's own\ntype, e.g. `pdf` for the converted rendition of a docx)."+ },+ "owner": {+ "$ref": "#/components/schemas/MacroUserIdStr",+ "description": "The owner of the document (used by the extractor to resolve S3 keys)."+ }+ }+ },"DocumentCopiedMetadata": {"type": "object","description": "Metadata for [`DocumentTopicEvent::Copied`].",@@ -15554,6 +15577,17 @@],"description": "The sub type of a document preview with associated properties.\nTask-related properties are encoded within the variant to ensure valid states."},+ "DocumentPurgedMetadata": {+ "type": "object",+ "description": "Metadata for [`DocumentTopicEvent::Purged`].",+ "required": ["document_id"],+ "properties": {+ "document_id": {+ "type": "string",+ "description": "The id of the hard-deleted document."+ }+ }+ },"DocumentResponse": {"type": "object","description": "Document response with content lifecycle metadata.",@@ -15688,6 +15722,25 @@"description": "The document sub type enum represents all values of document sub types.\nThese values should match the `document_sub_type_value` table in macrodb.","enum": ["task", "snippet"]},+ "DocumentSyncContentUpdatedMetadata": {+ "type": "object",+ "description": "Metadata for [`DocumentTopicEvent::SyncContentUpdated`].",+ "required": ["document_id", "file_type"],+ "properties": {+ "document_id": {+ "type": "string",+ "description": "The id of the live-collab document whose content changed."+ },+ "document_version_id": {+ "type": ["string", "null"],+ "description": "Version marker for the sync snapshot, when the caller supplies one."+ },+ "file_type": {+ "$ref": "#/components/schemas/FileType",+ "description": "File type of the sync document (markdown today)."+ }+ }+ },"DocumentTeamShareResponse": {"type": "object","description": "The team-share state of a document. The team is resolved from the document\nowner's team membership.",@@ -15751,6 +15804,51 @@}}},+ {+ "type": "object",+ "description": "The document's stored bytes were (re)written to S3.",+ "required": ["metadata", "event_type"],+ "properties": {+ "event_type": {+ "type": "string",+ "enum": ["document.content_uploaded"]+ },+ "metadata": {+ "$ref": "#/components/schemas/DocumentContentUploadedMetadata",+ "description": "The document's stored bytes were (re)written to S3."+ }+ }+ },+ {+ "type": "object",+ "description": "A live-collab (sync) document's content changed and should be re-extracted.",+ "required": ["metadata", "event_type"],+ "properties": {+ "event_type": {+ "type": "string",+ "enum": ["document.sync_content_updated"]+ },+ "metadata": {+ "$ref": "#/components/schemas/DocumentSyncContentUpdatedMetadata",+ "description": "A live-collab (sync) document's content changed and should be re-extracted."+ }+ }+ },+ {+ "type": "object",+ "description": "The document row was permanently deleted (hard delete).",+ "required": ["metadata", "event_type"],+ "properties": {+ "event_type": {+ "type": "string",+ "enum": ["document.purged"]+ },+ "metadata": {+ "$ref": "#/components/schemas/DocumentPurgedMetadata",+ "description": "The document row was permanently deleted (hard delete)."+ }+ }+ },{"type": "object","description": "A document was copied.",diff --git a/crates/ai_tools/src/build_context.rs b/crates/ai_tools/src/build_context.rsindex 83ac152928..496d5fd75b 100644--- a/crates/ai_tools/src/build_context.rs+++ b/crates/ai_tools/src/build_context.rs@@ -294,10 +294,6 @@ pub async fn build_tool_service_context_from_env(pool.clone(),)),macro_event_broker: macro_event_broker.clone(),- // AI-agent contexts don't own a search event queue (their sqs_client is- // configured without one), so no search-index refresh is published from- // here — matching the properties service built in the same context.- search_indexer: None,};let document_tool_context = DocumentToolContext::new(diff --git a/crates/document_storage_service_client/src/document.rs b/crates/document_storage_service_client/src/document.rsindex 7753342e6b..452a4dba11 100644--- a/crates/document_storage_service_client/src/document.rs+++ b/crates/document_storage_service_client/src/document.rs@@ -5,7 +5,7 @@ use anyhow::Result;use model::document::list::ListDocumentsWithAccessResponse;use model::document::response::{CreateDocumentRequest, CreateDocumentResponse};use model::document::{- DocumentBasic,+ DocumentBasic, FileType,response::{GetDocumentResponse, LocationResponseData, LocationResponseV3},};use model::document_storage_service_internal::{@@ -27,6 +27,12 @@ pub struct ExportDocumentResponse {pub presigned_url: String,}+#[derive(Serialize)]+struct ContentUploadedRequest {+ file_type: FileType,+ document_version_id: Option<String>,+}+impl DocumentStorageServiceClient {pub async fn get_document(&self, document_id: String) -> Result<GetDocumentResponse> {let res = self@@ -41,6 +47,43 @@ impl DocumentStorageServiceClient {Ok(doc_data)}+ /// Notify the document storage service that document content was uploaded.+ #[tracing::instrument(skip(self), err)]+ pub async fn publish_document_content_uploaded(+ &self,+ document_id: &str,+ file_type: FileType,+ document_version_id: Option<String>,+ ) -> Result<()> {+ let response = self+ .client+ .post(format!(+ "{}/internal/documents/{}/content-uploaded",+ self.url, document_id+ ))+ .json(&ContentUploadedRequest {+ file_type,+ document_version_id,+ })+ .send()+ .await?;++ let status = response.status();+ if !status.is_success() {+ let body = response+ .text()+ .await+ .unwrap_or_else(|_| "<failed to read body>".to_string());+ anyhow::bail!(+ "publishing document content upload failed: {} - {}",+ status,+ body+ );+ }++ Ok(())+ }+pub async fn get_document_basic(&self, document_id: &str) -> Result<Option<DocumentBasic>> {let res = self.clientdiff --git a/crates/documents/examples/documents_consumer.rs b/crates/documents/examples/documents_consumer.rsindex 2b9ad05229..738682bacc 100644--- a/crates/documents/examples/documents_consumer.rs+++ b/crates/documents/examples/documents_consumer.rs@@ -69,6 +69,9 @@ fn event_type(event: &DocumentTopicEvent) -> &'static str {DocumentTopicEvent::Created(_) => "document.created",DocumentTopicEvent::Updated(_) => "document.updated",DocumentTopicEvent::Deleted(_) => "document.deleted",+ DocumentTopicEvent::ContentUploaded(_) => "document.content_uploaded",+ DocumentTopicEvent::SyncContentUpdated(_) => "document.sync_content_updated",+ DocumentTopicEvent::Purged(_) => "document.purged",DocumentTopicEvent::Copied(_) => "document.copied",DocumentTopicEvent::Interaction(_) => "document.interaction",}diff --git a/crates/documents/src/domain.rs b/crates/documents/src/domain.rsindex d7c033def6..61b3f6727e 100644--- a/crates/documents/src/domain.rs+++ b/crates/documents/src/domain.rs@@ -3,6 +3,7 @@pub mod branch_name;pub mod content;/// Unified entity-mutation capability impls.+#[cfg(feature = "service")]pub mod entity_mutation;pub mod events;#[cfg(feature = "ports")]diff --git a/crates/documents/src/domain/events.rs b/crates/documents/src/domain/events.rsindex f3cb25141b..9763a35998 100644--- a/crates/documents/src/domain/events.rs+++ b/crates/documents/src/domain/events.rs@@ -4,6 +4,9 @@//! per-variant metadata structs, a [`TopicEvent`] enum tagged by `event_type`,//! and a [`MacroEvent`] wrapper keyed by document id.+#[cfg(test)]+mod test;+use chrono::{DateTime, Utc};use document_sub_type::DocumentSubType;use macro_event_broker::{Event, MacroEvent, TopicEvent};@@ -71,6 +74,42 @@ pub struct DocumentDeletedMetadata {pub project_id: Option<String>,}+/// Metadata for [`DocumentTopicEvent::ContentUploaded`].+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]+pub struct DocumentContentUploadedMetadata {+ /// The id of the document whose stored bytes changed.+ pub document_id: String,+ /// The owner of the document (used by the extractor to resolve S3 keys).+ pub owner: MacroUserIdStr<'static>,+ /// File type of the uploaded object (may differ from the document's own+ /// type, e.g. `pdf` for the converted rendition of a docx).+ pub file_type: FileType,+ /// Version written, or the converted-file marker; `None` for unversioned+ /// writes (mirrors `SearchExtractorMessage::document_version_id`).+ pub document_version_id: Option<String>,+}++/// Metadata for [`DocumentTopicEvent::SyncContentUpdated`].+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]+pub struct DocumentSyncContentUpdatedMetadata {+ /// The id of the live-collab document whose content changed.+ pub document_id: String,+ /// File type of the sync document (markdown today).+ pub file_type: FileType,+ /// Version marker for the sync snapshot, when the caller supplies one.+ pub document_version_id: Option<String>,+}++/// Metadata for [`DocumentTopicEvent::Purged`].+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]+pub struct DocumentPurgedMetadata {+ /// The id of the hard-deleted document.+ pub document_id: String,+}+/// Why a document interaction was reported.#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]@@ -130,6 +169,15 @@ pub enum DocumentTopicEvent {/// A document was soft-deleted.#[serde(rename = "document.deleted")]Deleted(DocumentDeletedMetadata),+ /// The document's stored bytes were (re)written to S3.+ #[serde(rename = "document.content_uploaded")]+ ContentUploaded(DocumentContentUploadedMetadata),+ /// A live-collab (sync) document's content changed and should be re-extracted.+ #[serde(rename = "document.sync_content_updated")]+ SyncContentUpdated(DocumentSyncContentUpdatedMetadata),+ /// The document row was permanently deleted (hard delete).+ #[serde(rename = "document.purged")]+ Purged(DocumentPurgedMetadata),/// A document was copied.#[serde(rename = "document.copied")]Copied(DocumentCopiedMetadata),@@ -166,6 +214,27 @@ impl DocumentMacroEvent {Self::new(key, DocumentTopicEvent::Deleted(metadata))}+ /// Build a content-uploaded event keyed by the document id.+ pub fn content_uploaded(+ key: impl Into<String>,+ metadata: DocumentContentUploadedMetadata,+ ) -> Self {+ Self::new(key, DocumentTopicEvent::ContentUploaded(metadata))+ }++ /// Build a sync-content-updated event keyed by the document id.+ pub fn sync_content_updated(+ key: impl Into<String>,+ metadata: DocumentSyncContentUpdatedMetadata,+ ) -> Self {+ Self::new(key, DocumentTopicEvent::SyncContentUpdated(metadata))+ }++ /// Build a purged event keyed by the document id.+ pub fn purged(key: impl Into<String>, metadata: DocumentPurgedMetadata) -> Self {+ Self::new(key, DocumentTopicEvent::Purged(metadata))+ }+/// Build a copied event keyed by the new document id.pub fn copied(key: impl Into<String>, metadata: DocumentCopiedMetadata) -> Self {Self::new(key, DocumentTopicEvent::Copied(metadata))diff --git a/crates/documents/src/domain/events/test.rs b/crates/documents/src/domain/events/test.rsnew file mode 100644index 0000000000..c01f5694be--- /dev/null+++ b/crates/documents/src/domain/events/test.rs@@ -0,0 +1,215 @@+use macro_event_broker::{Event, MacroEvent as _, TopicEvent};+use macro_user_id::user_id::MacroUserIdStr;+use model::document::FileType;+use serde_json::{Value, json};+use uuid::Uuid;++use super::{+ DocumentContentUploadedMetadata, DocumentInteractionMetadata, DocumentMacroEvent,+ DocumentPurgedMetadata, DocumentSyncContentUpdatedMetadata, DocumentTopicEvent,+ InteractionReason,+};++const DOCUMENT_ID: &str = "11111111-1111-1111-1111-111111111111";++fn owner() -> MacroUserIdStr<'static> {+ MacroUserIdStr::try_from("macro|[email protected]".to_string()).expect("valid user id")+}++fn assert_wire_round_trip(event: Event<DocumentTopicEvent>, expected: Value) {+ assert_eq!(+ serde_json::to_value(&event).expect("event serializes"),+ expected+ );++ let decoded: Event<DocumentTopicEvent> =+ serde_json::from_value(expected).expect("event deserializes");+ assert_eq!(decoded, event);+}++#[test]+fn content_uploaded_serializes_to_the_exact_envelope() {+ let event = Event::with_event_id(+ Uuid::from_u128(1),+ DocumentTopicEvent::ContentUploaded(DocumentContentUploadedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ owner: owner(),+ file_type: FileType::Pdf,+ document_version_id: Some("convert".to_string()),+ }),+ );++ assert_wire_round_trip(+ event,+ json!({+ "event_id": "00000000-0000-0000-0000-000000000001",+ "schema_version": 1,+ "event_type": "document.content_uploaded",+ "metadata": {+ "document_id": DOCUMENT_ID,+ "owner": "macro|[email protected]",+ "file_type": "pdf",+ "document_version_id": "convert",+ },+ }),+ );+}++#[test]+fn sync_content_updated_serializes_to_the_exact_envelope() {+ let event = Event::with_event_id(+ Uuid::from_u128(2),+ DocumentTopicEvent::SyncContentUpdated(DocumentSyncContentUpdatedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ file_type: FileType::Md,+ document_version_id: None,+ }),+ );++ assert_wire_round_trip(+ event,+ json!({+ "event_id": "00000000-0000-0000-0000-000000000002",+ "schema_version": 1,+ "event_type": "document.sync_content_updated",+ "metadata": {+ "document_id": DOCUMENT_ID,+ "file_type": "md",+ "document_version_id": null,+ },+ }),+ );+}++#[test]+fn purged_serializes_to_the_exact_envelope() {+ let event = Event::with_event_id(+ Uuid::from_u128(3),+ DocumentTopicEvent::Purged(DocumentPurgedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ }),+ );++ assert_wire_round_trip(+ event,+ json!({+ "event_id": "00000000-0000-0000-0000-000000000003",+ "schema_version": 1,+ "event_type": "document.purged",+ "metadata": {+ "document_id": DOCUMENT_ID,+ },+ }),+ );+}++#[test]+fn optional_document_versions_support_present_and_absent_values() {+ let events = [+ DocumentTopicEvent::ContentUploaded(DocumentContentUploadedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ owner: owner(),+ file_type: FileType::Pdf,+ document_version_id: None,+ }),+ DocumentTopicEvent::SyncContentUpdated(DocumentSyncContentUpdatedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ file_type: FileType::Md,+ document_version_id: Some("snapshot-7".to_string()),+ }),+ ];++ let content_uploaded = serde_json::to_value(&events[0]).expect("event serializes");+ assert_eq!(+ content_uploaded["metadata"]["document_version_id"],+ Value::Null+ );++ let sync_content_updated = serde_json::to_value(&events[1]).expect("event serializes");+ assert_eq!(+ sync_content_updated["metadata"]["document_version_id"],+ "snapshot-7"+ );++ for event in events {+ let payload = serde_json::to_vec(&event).expect("event serializes");+ let decoded: DocumentTopicEvent =+ serde_json::from_slice(&payload).expect("event deserializes");+ assert_eq!(decoded, event);+ }+}++#[test]+fn search_event_constructors_use_the_document_key_and_schema_v1() {+ assert_eq!(<DocumentTopicEvent as TopicEvent>::SCHEMA_VERSION, 1);++ let content_metadata = DocumentContentUploadedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ owner: owner(),+ file_type: FileType::Pdf,+ document_version_id: Some("version-1".to_string()),+ };+ let content_uploaded =+ DocumentMacroEvent::content_uploaded(DOCUMENT_ID, content_metadata.clone());+ assert_eq!(content_uploaded.key(), DOCUMENT_ID);+ assert_eq!(content_uploaded.event().schema_version, 1);+ assert_eq!(+ &content_uploaded.event().event,+ &DocumentTopicEvent::ContentUploaded(content_metadata)+ );++ let sync_metadata = DocumentSyncContentUpdatedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ file_type: FileType::Md,+ document_version_id: None,+ };+ let sync_content_updated =+ DocumentMacroEvent::sync_content_updated(DOCUMENT_ID, sync_metadata.clone());+ assert_eq!(sync_content_updated.key(), DOCUMENT_ID);+ assert_eq!(sync_content_updated.event().schema_version, 1);+ assert_eq!(+ &sync_content_updated.event().event,+ &DocumentTopicEvent::SyncContentUpdated(sync_metadata)+ );++ let purged_metadata = DocumentPurgedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ };+ let purged = DocumentMacroEvent::purged(DOCUMENT_ID, purged_metadata.clone());+ assert_eq!(purged.key(), DOCUMENT_ID);+ assert_eq!(purged.event().schema_version, 1);+ assert_eq!(+ &purged.event().event,+ &DocumentTopicEvent::Purged(purged_metadata)+ );+}++#[test]+fn existing_v1_document_events_still_decode() {+ let payload = json!({+ "event_id": "00000000-0000-0000-0000-000000000004",+ "schema_version": 1,+ "event_type": "document.interaction",+ "metadata": {+ "document_id": DOCUMENT_ID,+ "reason": "edited",+ },+ });++ let decoded = DocumentMacroEvent::decode(+ DOCUMENT_ID,+ &serde_json::to_vec(&payload).expect("payload serializes"),+ )+ .expect("existing v1 event decodes");++ assert_eq!(decoded.key(), DOCUMENT_ID);+ assert_eq!(decoded.event().schema_version, 1);+ assert_eq!(decoded.event().event_id, Uuid::from_u128(4));+ assert_eq!(+ &decoded.event().event,+ &DocumentTopicEvent::Interaction(DocumentInteractionMetadata {+ document_id: DOCUMENT_ID.to_string(),+ reason: InteractionReason::Edited,+ })+ );+}diff --git a/crates/documents/src/domain/ports/mod.rs b/crates/documents/src/domain/ports/mod.rsindex 1ef214706a..92c1743506 100644--- a/crates/documents/src/domain/ports/mod.rs+++ b/crates/documents/src/domain/ports/mod.rs@@ -13,7 +13,7 @@ use entity_access::domain::models::{EditAccessLevel, EntityAccessReceipt, MemberTeamRole, OwnerAccessLevel, ViewAccessLevel,};use macro_user_id::user_id::MacroUserIdStr;-use model::document::{ContentType, DocumentBasic, DocumentMetadata};+use model::document::{ContentType, DocumentBasic, DocumentMetadata, FileType};use super::content::DocumentContent;use super::events::InteractionReason;@@ -334,19 +334,15 @@ pub trait TaskPropertiesPort: Send + Sync + 'static {) -> impl Future<Output = anyhow::Result<()>> + Send;}-/// Port for refreshing a document's denormalized name in the search index.-///-/// A rename only changes the `document_name` on the parent doc, and nothing-/// else re-indexes parent-only file types, so the rename must publish a-/// targeted metadata refresh. Best-effort — callers log and continue on error.-/// Boxed future so the port stays object-safe and can be held as-/// `Arc<dyn DocumentSearchIndexer>` without adding a service generic.-pub trait DocumentSearchIndexer: Send + Sync + std::fmt::Debug {- /// Enqueue a refresh of the document's indexed name.- fn enqueue_name_update(- &self,- document_id: String,- ) -> std::pin::Pin<Box<dyn Future<Output = anyhow::Result<()>> + Send>>;+/// Use case for relaying document content-upload events.+pub trait DocumentContentEventService: Send + Sync + 'static {+ /// Load the document owner and publish a content-uploaded event.+ fn publish_content_uploaded(+ &self,+ document_id: &str,+ file_type: FileType,+ document_version_id: Option<String>,+ ) -> impl Future<Output = Result<(), DocumentError>> + Send;}/// Service interface for document operations.diff --git a/crates/documents/src/domain/service.rs b/crates/documents/src/domain/service.rsindex 531a2600c3..e7b3ea3af9 100644--- a/crates/documents/src/domain/service.rs+++ b/crates/documents/src/domain/service.rs@@ -9,7 +9,6 @@ use models_properties::EntityReference;use models_properties::api::SetPropertyValue;use std::borrow::Cow;use std::str::FromStr;-use std::sync::Arc;use std::time::{SystemTime, UNIX_EPOCH};use unicode_segmentation::UnicodeSegmentation;@@ -42,8 +41,9 @@ use crate::domain::models::{use super::branch_name::{build_task_branch_name, user_branch_prefix};use super::content::{DocumentContent, DocumentContentLocation, DocumentContentState};use super::events::{- DocumentCopiedMetadata, DocumentCreatedMetadata, DocumentDeletedMetadata,- DocumentInteractionMetadata, DocumentMacroEvent, DocumentUpdatedMetadata, InteractionReason,+ DocumentContentUploadedMetadata, DocumentCopiedMetadata, DocumentCreatedMetadata,+ DocumentDeletedMetadata, DocumentInteractionMetadata, DocumentMacroEvent,+ DocumentUpdatedMetadata, InteractionReason,};use super::models::{CloudFrontConfig, CommentThread, CopyDocumentRepoArgs, CreateDocumentRepoArgs,@@ -54,7 +54,7 @@ use super::models::{#[cfg(feature = "document_create")]use super::ports::create::DocumentCreationService;use super::ports::{- DocumentRepo, DocumentSearchIndexer, DocumentService, PresignedUploadUrlPort,+ DocumentContentEventService, DocumentRepo, DocumentService, PresignedUploadUrlPort,TaskPropertiesPort,};use super::response::{@@ -90,9 +90,6 @@ pub struct DocumentServiceImpl<pub foreign_entity_service: F,/// Macro event broker for publishing document lifecycle eventspub macro_event_broker: B,- /// Optional publisher that refreshes a document's denormalized name in the- /// search index after a rename.- pub search_indexer: Option<Arc<dyn DocumentSearchIndexer>>,}fn ready_content_for_file_type(file_type: Option<FileType>) -> DocumentContent {@@ -159,6 +156,17 @@ fn invalid_team_task_slug() -> DocumentError {DocumentError::BadRequest("invalid team task slug".to_string())}+fn map_basic_document_error(document_id: &str, error: anyhow::Error) -> DocumentError {+ if error+ .to_string()+ .contains("no rows returned by a query that expected to return at least one row")+ {+ DocumentError::NotFound(document_id.to_string())+ } else {+ DocumentError::Internal(error)+ }+}+fn team_task_number_from_slug(slug: &str) -> Result<i32, DocumentError> {let (prefix, number) = slug.rsplit_once('-').ok_or_else(invalid_team_task_slug)?;@@ -266,30 +274,6 @@ impl<entity_access_management_service,foreign_entity_service,macro_event_broker,- search_indexer: None,- }- }-- /// Attach a search-index publisher so renames refresh the denormalized- /// name in the search index. Builder-style so existing constructions are- /// unaffected.- pub fn with_search_indexer(mut self, search_indexer: Arc<dyn DocumentSearchIndexer>) -> Self {- self.search_indexer = Some(search_indexer);- self- }-- /// Best-effort publish of a search-index name refresh after a rename. Logs- /// and continues on failure so a missed refresh never fails the edit- /// itself; a full re-index would eventually correct the name anyway.- async fn enqueue_name_update(&self, document_id: &str) {- let Some(search_indexer) = self.search_indexer.as_ref() else {- return;- };- if let Err(error) = search_indexer- .enqueue_name_update(document_id.to_string())- .await- {- tracing::warn!(error = ?error, document_id = %document_id, "failed to enqueue search name update for rename");}}@@ -609,6 +593,44 @@ impl<}}+impl<+ R: DocumentRepo,+ U: PresignedUploadUrlPort,+ T: TaskPropertiesPort,+ C: ConnectionService,+ Eam: EntityAccessManagementService,+ F: ForeignEntityService,+ B: MacroEventBroker,+> DocumentContentEventService for DocumentServiceImpl<R, U, T, C, Eam, F, B>+{+ #[tracing::instrument(err, skip(self))]+ async fn publish_content_uploaded(+ &self,+ document_id: &str,+ file_type: FileType,+ document_version_id: Option<String>,+ ) -> Result<(), DocumentError> {+ let document = self+ .repo+ .get_basic_document(document_id)+ .await+ .map_err(|error| map_basic_document_error(document_id, error.into()))?;++ self.macro_event_broker+ .send_event(&DocumentMacroEvent::content_uploaded(+ document_id,+ DocumentContentUploadedMetadata {+ document_id: document_id.to_string(),+ owner: document.owner,+ file_type,+ document_version_id,+ },+ ))+ .map(|_| ())+ .map_err(|error| DocumentError::Internal(error.into()))+ }+}+impl<R: DocumentRepo,U: PresignedUploadUrlPort,@@ -838,16 +860,7 @@ impl<self.repo.get_basic_document(document_id).await- .map_err(|e| {- let err: anyhow::Error = e.into();- if err.to_string().contains(- "no rows returned by a query that expected to return at least one row",- ) {- DocumentError::NotFound(document_id.to_string())- } else {- DocumentError::Internal(err)- }- })+ .map_err(|error| map_basic_document_error(document_id, error.into()))}async fn get_document_text(@@ -1243,14 +1256,6 @@ impl<.await.map_err(|e| DocumentError::Internal(e.into()))?;- // A rename changes the denormalized `document_name` on the parent doc in- // the search index; nothing else re-indexes parent-only file types, so- // publish a targeted refresh once the new name is committed.- if document_name.is_some() {- self.enqueue_name_update(&entity_access_receipt.entity().entity_id)- .await;- }-// Update project modified timestamps. args.project_id of None means "no change",// so only move the document out of its old project when a different project (or// "" for no project) was explicitly requested.diff --git a/crates/documents/src/domain/service/tests.rs b/crates/documents/src/domain/service/tests.rsindex ad7cfaac59..4481f36536 100644--- a/crates/documents/src/domain/service/tests.rs+++ b/crates/documents/src/domain/service/tests.rs@@ -5,11 +5,11 @@ use foreign_entity::domain::models::{use foreign_entity::domain::ports::{ForeignEntityListQuery, ForeignEntityService};use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker};use macro_user_id::cowlike::CowLike;-use model::document::DocumentMetadata;+use model::document::{DocumentMetadata, FileType};use std::sync::{Arc, Mutex};use crate::domain::models::GithubPullRequest;-use crate::domain::ports::MockDocumentRepo;+use crate::domain::ports::{DocumentContentEventService, MockDocumentRepo};use super::*;@@ -343,9 +343,17 @@ struct PublishedEvent {#[derive(Clone, Default)]struct TestEventBroker {published: Arc<Mutex<Vec<PublishedEvent>>>,+ fail_send: bool,}impl TestEventBroker {+ fn failing() -> Self {+ Self {+ fail_send: true,+ ..Self::default()+ }+ }+fn published(&self) -> Arc<Mutex<Vec<PublishedEvent>>> {Arc::clone(&self.published)}@@ -356,6 +364,10 @@ impl MacroEventBroker for TestEventBroker {&self,event: &E,) -> Result<tokio::task::JoinHandle<Result<(), EventBrokerError>>, EventBrokerError> {+ if self.fail_send {+ return Err(EventBrokerError::Publish("test broker failure".to_string()));+ }+self.published.lock().unwrap().push(PublishedEvent {topic: event.topic(),key: event.key().to_string(),@@ -365,24 +377,6 @@ impl MacroEventBroker for TestEventBroker {}}-#[derive(Clone, Debug, Default)]-struct TestDocumentSearchIndexer {- enqueued: Arc<Mutex<Vec<String>>>,-}--impl crate::domain::ports::DocumentSearchIndexer for TestDocumentSearchIndexer {- fn enqueue_name_update(- &self,- document_id: String,- ) -> std::pin::Pin<Box<dyn std::future::Future<Output = anyhow::Result<()>> + Send>> {- let enqueued = Arc::clone(&self.enqueued);- Box::pin(async move {- enqueued.lock().unwrap().push(document_id);- Ok(())- })- }-}-type TestDocumentService = DocumentServiceImpl<MockDocumentRepo,TestUploadUrlPort,@@ -449,34 +443,17 @@ fn make_test_service_with_entity_access((service, entity_access)}-/// Build a test service along with a handle to its recording search indexer.-fn make_test_service_with_search_indexer(+/// Build a test service along with a handle to its recording event broker.+fn make_test_service_with_event_broker(repo: MockDocumentRepo,-) -> (TestDocumentService, TestDocumentSearchIndexer) {- let search_indexer = TestDocumentSearchIndexer::default();- let service = DocumentServiceImpl::new(- repo,- test_cloudfront_config(),- sync_service_client::SyncServiceClient::new(- "test-sync-key".to_string(),- "http://sync-service.test".to_string(),- ),- TestUploadUrlPort,- TestTaskPropertiesPort,- TestConnectionService,- TestEntityAccessManagementService::default(),- TestForeignEntityService::default(),- TestEventBroker::default(),- )- .with_search_indexer(Arc::new(search_indexer.clone()));- (service, search_indexer)+) -> (TestDocumentService, TestEventBroker) {+ make_test_service_with_configured_event_broker(repo, TestEventBroker::default())}-/// Build a test service along with a handle to its recording event broker.-fn make_test_service_with_event_broker(+fn make_test_service_with_configured_event_broker(repo: MockDocumentRepo,+ event_broker: TestEventBroker,) -> (TestDocumentService, TestEventBroker) {- let event_broker = TestEventBroker::default();let service = DocumentServiceImpl::new(repo,test_cloudfront_config(),@@ -1537,6 +1514,92 @@ fn edit_receipt(document_id: &str) -> EntityAccessReceipt<EditAccessLevel> {)}+#[tokio::test]+async fn content_uploaded_publishes_document_event_with_owner_and_version() {+ let mut repo = make_mock_repo();+ repo.expect_get_basic_document()+ .withf(|document_id| document_id == "doc-1")+ .return_once(|_| Box::pin(std::future::ready(Ok(task_document_context("doc-1")))));++ let (service, event_broker) = make_test_service_with_event_broker(repo);++ service+ .publish_content_uploaded("doc-1", FileType::Pdf, Some("convert".to_string()))+ .await+ .unwrap();++ let published = event_broker.published();+ let published = published.lock().unwrap();+ assert_eq!(published.len(), 1);+ let event = &published[0];+ assert_eq!(event.topic, "macro.documents");+ assert_eq!(event.key, "doc-1");+ assert_eq!(event.payload["event_type"], "document.content_uploaded");+ assert_eq!(event.payload["metadata"]["document_id"], "doc-1");+ assert_eq!(event.payload["metadata"]["owner"], "macro|[email protected]");+ assert_eq!(event.payload["metadata"]["file_type"], "pdf");+ assert_eq!(event.payload["metadata"]["document_version_id"], "convert");+}++#[tokio::test]+async fn content_uploaded_preserves_an_absent_version() {+ let mut repo = make_mock_repo();+ repo.expect_get_basic_document()+ .return_once(|_| Box::pin(std::future::ready(Ok(task_document_context("doc-1")))));++ let (service, event_broker) = make_test_service_with_event_broker(repo);++ service+ .publish_content_uploaded("doc-1", FileType::Pdf, None)+ .await+ .unwrap();++ let published = event_broker.published();+ let published = published.lock().unwrap();+ assert_eq!(+ published[0].payload["metadata"]["document_version_id"],+ serde_json::Value::Null+ );+}++#[tokio::test]+async fn content_uploaded_maps_a_missing_document_to_not_found() {+ let mut repo = make_mock_repo();+ repo.expect_get_basic_document().return_once(|_| {+ Box::pin(std::future::ready(Err(anyhow!(+ "no rows returned by a query that expected to return at least one row"+ ))))+ });++ let (service, event_broker) = make_test_service_with_event_broker(repo);+ let result = service+ .publish_content_uploaded("missing-doc", FileType::Pdf, None)+ .await;++ assert!(matches!(+ result,+ Err(DocumentError::NotFound(document_id)) if document_id == "missing-doc"+ ));+ assert!(event_broker.published().lock().unwrap().is_empty());+}++#[tokio::test]+async fn content_uploaded_maps_an_immediate_broker_failure_to_internal() {+ let mut repo = make_mock_repo();+ repo.expect_get_basic_document()+ .return_once(|_| Box::pin(std::future::ready(Ok(task_document_context("doc-1")))));+ let event_broker = TestEventBroker::failing();+ let (service, event_broker) =+ make_test_service_with_configured_event_broker(repo, event_broker);++ let result = service+ .publish_content_uploaded("doc-1", FileType::Pdf, None)+ .await;++ assert!(matches!(result, Err(DocumentError::Internal(_))));+ assert!(event_broker.published().lock().unwrap().is_empty());+}+#[tokio::test]async fn test_delete_document_publishes_document_deleted_event() {let mut repo = make_mock_repo();@@ -1675,65 +1738,6 @@ async fn test_edit_document_rename_only_keeps_project_access() {assert!(entity_access.added_to_projects.lock().unwrap().is_empty());}-#[tokio::test]-async fn test_edit_document_rename_enqueues_search_name_update() {- let mut repo = make_mock_repo();- repo.expect_edit_document()- .withf(|args| args.document_id == "doc-1")- .returning(|_| Box::pin(std::future::ready(Ok(()))));-- let (service, search_indexer) = make_test_service_with_search_indexer(repo);-- service- .edit_document(- edit_receipt("doc-1"),- task_document_context("doc-1"),- EditDocumentServiceArgs {- document_name: Some("New name".to_string()),- project_id: None,- share_permission: None,- file_type: None,- },- )- .await- .unwrap();-- assert_eq!(- *search_indexer.enqueued.lock().unwrap(),- vec!["doc-1".to_string()]- );-}--#[tokio::test]-async fn test_edit_document_without_rename_does_not_enqueue_search_name_update() {- let document_id = uuid::Uuid::new_v4().to_string();- let new_project_id = uuid::Uuid::new_v4();-- let mut repo = make_mock_repo();- repo.expect_edit_document()- .returning(|_| Box::pin(std::future::ready(Ok(()))));- repo.expect_update_project_modified()- .returning(|_| Box::pin(std::future::ready(Ok(()))));-- let (service, search_indexer) = make_test_service_with_search_indexer(repo);-- service- .edit_document(- edit_receipt(&document_id),- task_document_context(&document_id),- EditDocumentServiceArgs {- document_name: None,- project_id: Some(new_project_id.to_string()),- share_permission: None,- file_type: None,- },- )- .await- .unwrap();-- assert!(search_indexer.enqueued.lock().unwrap().is_empty());-}-#[tokio::test]async fn test_edit_document_project_change_moves_project_access() {let document_id = uuid::Uuid::new_v4().to_string();diff --git a/crates/documents/src/inbound/axum_router.rs b/crates/documents/src/inbound/axum_router.rsindex 66e023f864..ff7774bbfd 100644--- a/crates/documents/src/inbound/axum_router.rs+++ b/crates/documents/src/inbound/axum_router.rs@@ -16,6 +16,7 @@#[cfg(test)]mod tests;+pub mod content_uploaded;pub mod copy_document;pub mod create_document;#[cfg(feature = "document_create")]diff --git a/crates/documents/src/inbound/axum_router/content_uploaded.rs b/crates/documents/src/inbound/axum_router/content_uploaded.rsnew file mode 100644index 0000000000..c0fdb8766b--- /dev/null+++ b/crates/documents/src/inbound/axum_router/content_uploaded.rs@@ -0,0 +1,43 @@+//! Handler for `POST /internal/documents/{document_id}/content-uploaded`.++use axum::{+ Json,+ extract::{Path, State},+ http::StatusCode,+};+use entity_access::domain::ports::EntityAccessService;+use macro_authorization::{InternalOnly, MacroAuthorizationExtractor, MacroAuthorizationService};+use model::document::FileType;+use serde::Deserialize;++use super::{DocumentRouterState, Params};+use crate::domain::{models::DocumentError, ports::DocumentContentEventService};++/// Request body describing the uploaded document content.+#[derive(Debug, Deserialize)]+pub struct ContentUploadedRequest {+ /// File type of the uploaded object.+ pub file_type: FileType,+ /// Uploaded version or converted-file marker, when present.+ pub document_version_id: Option<String>,+}++/// Publishes a content-uploaded event for an existing document.+#[tracing::instrument(err, skip(state, _internal_authorization))]+pub async fn content_uploaded_handler<+ T: DocumentContentEventService,+ Svc: EntityAccessService,+ Auth: MacroAuthorizationService,+>(+ State(state): State<DocumentRouterState<T, Svc, Auth>>,+ _internal_authorization: MacroAuthorizationExtractor<Auth, InternalOnly>,+ Path(Params { document_id }): Path<Params>,+ Json(request): Json<ContentUploadedRequest>,+) -> Result<StatusCode, DocumentError> {+ state+ .service+ .publish_content_uploaded(&document_id, request.file_type, request.document_version_id)+ .await?;++ Ok(StatusCode::OK)+}diff --git a/crates/documents/src/inbound/axum_router/tests.rs b/crates/documents/src/inbound/axum_router/tests.rsindex 30fc933cf2..13068befe7 100644--- a/crates/documents/src/inbound/axum_router/tests.rs+++ b/crates/documents/src/inbound/axum_router/tests.rs@@ -22,7 +22,7 @@ use macro_authorization::{};use macro_user_id::{lowercased::Lowercase, user_id::MacroUserId, user_id::MacroUserIdStr};use model::{- document::{DocumentBasic, DocumentMetadata, response::DocumentResponseMetadata},+ document::{DocumentBasic, DocumentMetadata, FileType, response::DocumentResponseMetadata},sync_service::SyncServiceVersionID,};use model_entity::Entity;@@ -42,7 +42,7 @@ use task_dedup::{use tower::ServiceExt;use uuid::Uuid;-use super::{DocumentRouterState, documents_router};+use super::{DocumentRouterState, content_uploaded::content_uploaded_handler, documents_router};use crate::{domain::{content::DocumentContent,@@ -53,7 +53,7 @@ use crate::{DocumentTeamShareResponse, EditDocumentServiceArgs, GithubPullRequestsResponse,LocationQueryParams, TaskBranchName,},- ports::{DocumentService, create::DocumentCreationService},+ ports::{DocumentContentEventService, DocumentService, create::DocumentCreationService},response::{CreateDocumentResponseData, DocumentMetadataWithContent, DocumentResponse,DocumentResponseMetadataWithContent, GetDocumentResponseData, LocationResponseV3,@@ -89,6 +89,13 @@ struct UploadSnapshotCall {bytes: Vec<u8>,}+#[derive(Clone, Debug, Eq, PartialEq)]+struct ContentUploadedCall {+ document_id: String,+ file_type: FileType,+ document_version_id: Option<String>,+}+#[derive(Clone, Debug, Eq, PartialEq)]struct TeamSlugCall {team_id: String,@@ -107,6 +114,7 @@ enum TeamSlugResult {struct FakeDocumentService {create_calls: Mutex<Vec<CreateDocumentCall>>,upload_snapshot_calls: Mutex<Vec<UploadSnapshotCall>>,+ content_uploaded_calls: Mutex<Vec<ContentUploadedCall>>,internal_get_calls: Mutex<Vec<String>>,team_slug_calls: Mutex<Vec<TeamSlugCall>>,team_slug_result: Mutex<Option<TeamSlugResult>>,@@ -128,6 +136,13 @@ impl FakeDocumentService {.clone()}+ fn content_uploaded_calls(&self) -> Vec<ContentUploadedCall> {+ self.content_uploaded_calls+ .lock()+ .expect("content uploaded calls lock poisoned")+ .clone()+ }+fn set_team_slug_result(&self, result: TeamSlugResult) {*self.team_slug_result@@ -394,6 +409,25 @@ impl DocumentService for FakeDocumentService {}}+impl DocumentContentEventService for FakeDocumentService {+ async fn publish_content_uploaded(+ &self,+ document_id: &str,+ file_type: FileType,+ document_version_id: Option<String>,+ ) -> Result<(), DocumentError> {+ self.content_uploaded_calls+ .lock()+ .expect("content uploaded calls lock poisoned")+ .push(ContentUploadedCall {+ document_id: document_id.to_string(),+ file_type,+ document_version_id,+ });+ Ok(())+ }+}+impl DocumentCreationService for FakeDocumentService {async fn create_document(&self,@@ -791,8 +825,19 @@ fn test_router() -> (FakeDocumentService,FakeEntityAccessService,FakeAuthorizationService,- (),- >(state);+ DocumentRouterState<FakeDocumentService, FakeEntityAccessService, FakeAuthorizationService>,+ >(state.clone())+ .route(+ "/{document_id}/content-uploaded",+ axum::routing::post(+ content_uploaded_handler::<+ FakeDocumentService,+ FakeEntityAccessService,+ FakeAuthorizationService,+ >,+ ),+ )+ .with_state(state);(router,@@ -969,6 +1014,56 @@ async fn get_document_by_team_slug_preserves_domain_error_statuses() {assert!(document_service.internal_get_calls().is_empty());}+#[tokio::test]+async fn content_uploaded_requires_internal_authentication_and_forwards_request() {+ let (router, document_service, _access_service, authorization_service) = test_router();+ let body = || {+ Body::from(+ serde_json::to_vec(&json!({+ "file_type": "pdf",+ "document_version_id": "convert",+ }))+ .expect("request should serialize"),+ )+ };++ let jwt_request = Request::post("/content-document/content-uploaded")+ .header("authorization", format!("Bearer {JWT_TOKEN}"))+ .header("content-type", "application/json")+ .body(body())+ .expect("request should build");+ assert_eq!(+ send_status(&router, jwt_request).await,+ StatusCode::FORBIDDEN+ );+ assert!(document_service.content_uploaded_calls().is_empty());++ let internal_request = Request::post("/content-document/content-uploaded")+ .header(INTERNAL_API_KEY_HEADER, STANDARD_INTERNAL_KEY)+ .header("content-type", "application/json")+ .body(body())+ .expect("request should build");+ assert_eq!(send_status(&router, internal_request).await, StatusCode::OK);+ assert_eq!(+ document_service.content_uploaded_calls(),+ [ContentUploadedCall {+ document_id: "content-document".to_string(),+ file_type: FileType::Pdf,+ document_version_id: Some("convert".to_string()),+ }]+ );+ assert_eq!(+ authorization_service.calls(),+ [+ AuthorizationCall::Jwt(JWT_TOKEN.to_string()),+ AuthorizationCall::Internal {+ provided_key: STANDARD_INTERNAL_KEY.to_string(),+ claims: InternalIdentityClaims::default(),+ },+ ]+ );+}+#[tokio::test]async fn snapshot_upload_requires_internal_api_key() {let snapshot = b"snapshot bytes";diff --git a/crates/projects/Cargo.toml b/crates/projects/Cargo.tomlindex d23d35017d..96263f337b 100644--- a/crates/projects/Cargo.toml+++ b/crates/projects/Cargo.toml@@ -20,6 +20,7 @@ outbound = ["dep:aws-sdk-s3","dep:base64","dep:document_sub_type",+ "dep:documents","dep:dynamodb_client","dep:entity_access_db_utils","dep:hex",@@ -81,16 +82,20 @@ async-recursion = { version = "1.1.1", optional = true }aws-sdk-s3 = { workspace = true, optional = true }base64 = { workspace = true, optional = true }document_sub_type = { path = "../document_sub_type", optional = true }+documents = { path = "../documents", default-features = false, optional = true }dynamodb_client = { path = "../dynamodb_client", optional = true }entity_access_db_utils = { path = "../entity_access_db_utils", optional = true }hex = { workspace = true, optional = true }macro_sha_count_client = { path = "../macro_sha_count_client", optional = true }sqlx = { workspace = true, optional = true }-sqs_client = { path = "../sqs_client", optional = true }+sqs_client = { path = "../sqs_client", default-features = false, features = [+ "document",+], optional = true }system_properties = { path = "../system_properties", optional = true }workspace-hack = { version = "0.1", path = "../workspace-hack" }[dev-dependencies]+aws-sdk-sqs = { workspace = true }dotenvy = { workspace = true }http-body-util = { workspace = true }kafka_util = { path = "../kafka_util" }diff --git a/crates/projects/src/outbound.rs b/crates/projects/src/outbound.rsindex c49dd7e5a1..888315e3ef 100644--- a/crates/projects/src/outbound.rs+++ b/crates/projects/src/outbound.rs@@ -8,7 +8,7 @@ pub mod pg_project_repo;pub mod s3_upload_urls;/// Redis content-hash counter adapter.pub mod sha_counter;-/// SQS project search-index adapter.+/// Kafka search-cleanup and SQS document-deletion adapter.pub mod sqs_search_indexer;pub use dynamo_bulk_upload::DynamoBulkUploadAdapter;diff --git a/crates/projects/src/outbound/sqs_search_indexer.rs b/crates/projects/src/outbound/sqs_search_indexer.rsindex 4455625854..cb3016ba92 100644--- a/crates/projects/src/outbound/sqs_search_indexer.rs+++ b/crates/projects/src/outbound/sqs_search_indexer.rs@@ -1,35 +1,41 @@-//! SQS adapter for document search cleanup and deletion.+//! Kafka search-cleanup and SQS document-deletion adapter.++#[cfg(test)]+mod test;use std::sync::Arc;-use sqs_client::search::{SearchQueueMessage, document::DocumentId};+use documents::domain::events::{DocumentMacroEvent, DocumentPurgedMetadata};+use macro_event_broker::MacroEventBroker;use crate::domain::ports::ProjectSearchIndexer;-/// SQS-backed document search cleanup and deletion adapter.+/// Kafka-backed document search cleanup and SQS-backed document deletion adapter.#[derive(Clone)]-pub struct SqsProjectSearchIndexer {+pub struct SqsProjectSearchIndexer<B: MacroEventBroker> {sqs: Arc<sqs_client::SQS>,+ event_broker: B,}-impl SqsProjectSearchIndexer {- /// Create a project search indexer from the shared SQS client.- pub fn new(sqs: Arc<sqs_client::SQS>) -> Self {- Self { sqs }+impl<B: MacroEventBroker> SqsProjectSearchIndexer<B> {+ /// Create a project search indexer from the shared SQS client and event broker.+ pub fn new(sqs: Arc<sqs_client::SQS>, event_broker: B) -> Self {+ Self { sqs, event_broker }}}-impl ProjectSearchIndexer for SqsProjectSearchIndexer {+impl<B: MacroEventBroker> ProjectSearchIndexer for SqsProjectSearchIndexer<B> {#[tracing::instrument(skip(self), err)]async fn remove_documents(&self, document_ids: Vec<String>) -> anyhow::Result<()> {- let messages = document_ids- .into_iter()- .map(|document_id| SearchQueueMessage::RemoveDocument(DocumentId { document_id }))- .collect();-- self.sqs- .bulk_send_message_to_search_event_queue(messages)- .await+ for document_id in document_ids {+ let event = DocumentMacroEvent::purged(+ document_id.clone(),+ DocumentPurgedMetadata { document_id },+ );+ drop(self.event_broker.send_event(&event)?);+ }++ Ok(())}#[tracing::instrument(skip(self), err)]diff --git a/crates/projects/src/outbound/sqs_search_indexer/test.rs b/crates/projects/src/outbound/sqs_search_indexer/test.rsnew file mode 100644index 0000000000..3c2cd879c6--- /dev/null+++ b/crates/projects/src/outbound/sqs_search_indexer/test.rs@@ -0,0 +1,135 @@+use std::sync::{Arc, Mutex};++use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker};+use serde_json::{Value, json};+use uuid::Uuid;++use super::*;++#[derive(Clone, Debug, PartialEq)]+struct PublishedEvent {+ topic: String,+ key: String,+ envelope: Value,+}++#[derive(Clone, Default)]+struct RecordingEventBroker {+ state: Arc<Mutex<RecordingState>>,+ fail_send: bool,+}++#[derive(Default)]+struct RecordingState {+ attempts: usize,+ published: Vec<PublishedEvent>,+}++impl RecordingEventBroker {+ fn failing() -> Self {+ Self {+ fail_send: true,+ ..Self::default()+ }+ }++ fn attempts(&self) -> usize {+ self.state.lock().unwrap().attempts+ }++ fn published(&self) -> Vec<PublishedEvent> {+ self.state.lock().unwrap().published.clone()+ }+}++impl MacroEventBroker for RecordingEventBroker {+ fn send_event<E: MacroEvent + ?Sized>(+ &self,+ event: &E,+ ) -> Result<tokio::task::JoinHandle<Result<(), EventBrokerError>>, EventBrokerError> {+ let mut state = self.state.lock().unwrap();+ state.attempts += 1;++ if self.fail_send {+ return Err(EventBrokerError::Publish(+ "event enqueue rejected".to_string(),+ ));+ }++ state.published.push(PublishedEvent {+ topic: event.topic().to_string(),+ key: event.key().to_string(),+ envelope: serde_json::to_value(event.event())?,+ });++ Ok(tokio::spawn(async { Ok(()) }))+ }+}++fn test_sqs_client() -> Arc<sqs_client::SQS> {+ let config = aws_sdk_sqs::Config::builder()+ .behavior_version(aws_sdk_sqs::config::BehaviorVersion::latest())+ .build();++ Arc::new(sqs_client::SQS::new(aws_sdk_sqs::Client::from_conf(config)))+}++#[tokio::test]+async fn empty_document_list_publishes_no_events() {+ let event_broker = RecordingEventBroker::default();+ let indexer = SqsProjectSearchIndexer::new(test_sqs_client(), event_broker.clone());++ indexer.remove_documents(Vec::new()).await.unwrap();++ assert_eq!(event_broker.attempts(), 0);+ assert!(event_broker.published().is_empty());+}++#[tokio::test]+async fn document_removals_publish_separately_keyed_purged_event_envelopes() {+ let event_broker = RecordingEventBroker::default();+ let indexer = SqsProjectSearchIndexer::new(test_sqs_client(), event_broker.clone());+ let document_ids = vec!["document-one".to_string(), "document-two".to_string()];++ indexer+ .remove_documents(document_ids.clone())+ .await+ .unwrap();++ let published = event_broker.published();+ assert_eq!(published.len(), document_ids.len());++ for (event, document_id) in published.iter().zip(document_ids) {+ assert_eq!(event.topic, "macro.documents");+ assert_eq!(event.key, document_id);++ let event_id = event.envelope["event_id"].clone();+ assert!(Uuid::parse_str(event_id.as_str().unwrap()).is_ok());+ assert_eq!(+ event.envelope,+ json!({+ "event_id": event_id,+ "schema_version": 1,+ "event_type": "document.purged",+ "metadata": {+ "document_id": document_id,+ },+ })+ );+ }+}++#[tokio::test]+async fn document_removals_return_immediate_broker_failures() {+ let event_broker = RecordingEventBroker::failing();+ let indexer = SqsProjectSearchIndexer::new(test_sqs_client(), event_broker.clone());++ let error = indexer+ .remove_documents(vec!["document-one".to_string()])+ .await+ .expect_err("immediate broker failure should be returned");++ assert!(format!("{error:#}").contains("event enqueue rejected"));+ assert_eq!(event_broker.attempts(), 1);+ assert!(event_broker.published().is_empty());+}diff --git a/crates/soup_realtime/src/inbound/kafka_consumer.rs b/crates/soup_realtime/src/inbound/kafka_consumer.rsindex 69996feba6..97254876e8 100644--- a/crates/soup_realtime/src/inbound/kafka_consumer.rs+++ b/crates/soup_realtime/src/inbound/kafka_consumer.rs@@ -122,7 +122,10 @@ fn patches_from_document_event(event: &DocumentTopicEvent) -> Vec<SoupRealtimePa{push_unique_update(&mut updates, EntityType::Document, &metadata.document_id);}- DocumentTopicEvent::Interaction(_) => {}+ DocumentTopicEvent::Interaction(_)+ | DocumentTopicEvent::ContentUploaded(_)+ | DocumentTopicEvent::SyncContentUpdated(_)+ | DocumentTopicEvent::Purged(_) => {}}updates}diff --git a/crates/soup_realtime/src/inbound/kafka_consumer/test.rs b/crates/soup_realtime/src/inbound/kafka_consumer/test.rsindex 7184a1a51d..c44cf68b5e 100644--- a/crates/soup_realtime/src/inbound/kafka_consumer/test.rs+++ b/crates/soup_realtime/src/inbound/kafka_consumer/test.rs@@ -10,7 +10,8 @@ use channels::domain::{use chat::domain::events::{ChatMessageDeletedMetadata, ChatTopicEvent, ChatUpdatedMetadata};use chrono::Utc;use documents::domain::events::{- DocumentCreatedMetadata, DocumentDeletedMetadata, DocumentInteractionMetadata,+ DocumentContentUploadedMetadata, DocumentCreatedMetadata, DocumentDeletedMetadata,+ DocumentInteractionMetadata, DocumentPurgedMetadata, DocumentSyncContentUpdatedMetadata,DocumentUpdatedMetadata, InteractionReason,};use email::domain::events::{@@ -178,6 +179,30 @@ fn document_edit_interactions_map_to_updated_patches() {assert!(patches_from_document_event(&first_join).is_empty());}+#[test]+fn search_only_document_events_do_not_emit_patches() {+ let events = [+ DocumentTopicEvent::ContentUploaded(DocumentContentUploadedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ owner: user(),+ file_type: "pdf".parse().expect("valid file type"),+ document_version_id: Some("convert".to_string()),+ }),+ DocumentTopicEvent::SyncContentUpdated(DocumentSyncContentUpdatedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ file_type: "md".parse().expect("valid file type"),+ document_version_id: None,+ }),+ DocumentTopicEvent::Purged(DocumentPurgedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ }),+ ];++ for event in events {+ assert!(patches_from_document_event(&event).is_empty());+ }+}+#[test]fn project_deletion_maps_cascade_entities_to_deleted_patches() {let project_id = Uuid::now_v7().to_string();diff --git a/crates/sqs_client/src/search/document.rs b/crates/sqs_client/src/search/document.rsindex 8c6244ba6e..e827705594 100644--- a/crates/sqs_client/src/search/document.rs+++ b/crates/sqs_client/src/search/document.rs@@ -2,11 +2,6 @@use model::document::{BackfillSearchDocumentInformation, FileType};use s3_key::CONVERTED_DOCUMENT_FILE_NAME;-#[derive(serde::Serialize, serde::Deserialize, PartialEq, Eq, Debug)]-pub struct DocumentId {- pub document_id: String,-}-/// A request to refresh only the indexed properties of a document (e.g. after/// a status / assignee / tag change), without re-extracting its content.#[derive(serde::Serialize, serde::Deserialize, PartialEq, Eq, Debug)]diff --git a/crates/sqs_client/src/search/mod.rs b/crates/sqs_client/src/search/mod.rsindex 44fd2d6b78..59ceb16699 100644--- a/crates/sqs_client/src/search/mod.rs+++ b/crates/sqs_client/src/search/mod.rs@@ -4,7 +4,7 @@ use crate::{call::{CallRecordMessage, RemoveCallRecord},channel::ChannelMessageUpdate,chat::ChatMessage,- document::{DocumentId, DocumentPropertiesUpdate, SearchExtractorMessage},+ document::{DocumentPropertiesUpdate, SearchExtractorMessage},email::{EmailLinkMessage, EmailMessage, EmailThreadBatchMessage, EmailThreadMessage},project::UpsertProject,},@@ -70,10 +70,8 @@ pub enum Operation {pub enum SearchQueueMessage {// DocumentExtractDocumentText(SearchExtractorMessage),- RemoveDocument(DocumentId),ExtractSync(SearchExtractorMessage),UpdateDocumentProperties(DocumentPropertiesUpdate),- UpdateDocumentName(DocumentId),// Chat/// SQS backfill work-queue contract for reconciling a chat message.ChatMessage(ChatMessage),@@ -99,10 +97,8 @@ impl PrimaryId for SearchQueueMessage {fn id(&self) -> String {match self {SearchQueueMessage::ExtractDocumentText(message) => message.document_id.clone(),- SearchQueueMessage::RemoveDocument(message) => message.document_id.clone(),SearchQueueMessage::ExtractSync(message) => message.document_id.clone(),SearchQueueMessage::UpdateDocumentProperties(message) => message.document_id.clone(),- SearchQueueMessage::UpdateDocumentName(message) => message.document_id.clone(),// The message id keeps entries unique within an SQS batch.SearchQueueMessage::ChatMessage(message) => message.message_id.clone(),SearchQueueMessage::ExtractEmailMessage(message)@@ -131,10 +127,8 @@ impl SearchQueueMessage {match self {// DocumentSearchQueueMessage::ExtractDocumentText(_) => Operation::ExtractText,- SearchQueueMessage::RemoveDocument(_) => Operation::Remove,SearchQueueMessage::ExtractSync(_) => Operation::ExtractSync,SearchQueueMessage::UpdateDocumentProperties(_) => Operation::UpdateMetadata,- SearchQueueMessage::UpdateDocumentName(_) => Operation::UpdateMetadata,// ChatSearchQueueMessage::ChatMessage(_) => Operation::ExtractText,@@ -225,31 +219,3 @@ pub async fn bulk_enqueue_search_text_extractor(Ok(())}--#[cfg(test)]-mod tests {- use super::*;- use crate::search::document::DocumentId;-- #[test]- fn update_document_name_maps_to_update_metadata() {- let message = SearchQueueMessage::UpdateDocumentName(DocumentId {- document_id: "doc-1".to_string(),- });-- assert!(matches!(message.operation(), Operation::UpdateMetadata));- assert_eq!(message.id(), "doc-1");- }-- #[test]- fn update_document_name_round_trips() {- let message = SearchQueueMessage::UpdateDocumentName(DocumentId {- document_id: "doc-1".to_string(),- });-- let serialized = serde_json::to_string(&message).unwrap();- let deserialized: SearchQueueMessage = serde_json::from_str(&serialized).unwrap();-- assert_eq!(message, deserialized);- }-}diff --git a/crates/webhook/src/domain/ingestion.rs b/crates/webhook/src/domain/ingestion.rsindex bac974c171..14999d96fe 100644--- a/crates/webhook/src/domain/ingestion.rs+++ b/crates/webhook/src/domain/ingestion.rs@@ -222,7 +222,7 @@ wherefn normalized_document_event(event: &Event<DocumentTopicEvent>,-) -> Result<NormalizedWebhookEvent, WebhookEventIngestionError> {+) -> Result<Option<NormalizedWebhookEvent>, WebhookEventIngestionError> {let (event_name, entity_id) = match &event.event {DocumentTopicEvent::Created(metadata) => ("document.created", &metadata.document_id),DocumentTopicEvent::Updated(metadata) => ("document.updated", &metadata.document_id),@@ -231,6 +231,9 @@ fn normalized_document_event(DocumentTopicEvent::Interaction(metadata) => {("document.interaction", &metadata.document_id)}+ DocumentTopicEvent::ContentUploaded(_)+ | DocumentTopicEvent::SyncContentUpdated(_)+ | DocumentTopicEvent::Purged(_) => return Ok(None),};if Uuid::parse_str(entity_id).is_err() {@@ -241,14 +244,14 @@ fn normalized_document_event(}let broker_envelope = serde_json::to_value(event)?;- Ok(normalized_event(+ Ok(Some(normalized_event(event.event_id,event.schema_version,event_name,DOCUMENT_ENTITY_TYPE,entity_id,broker_envelope,- ))+ )))}fn normalized_channel_event(@@ -369,7 +372,9 @@ where&self,event: Event<DocumentTopicEvent>,) -> Result<(), WebhookEventIngestionError> {- let event = normalized_document_event(&event)?;+ let Some(event) = normalized_document_event(&event)? else {+ return Ok(());+ };self.resolve_entity_access_and_enqueue(event, EntityType::Document).await}diff --git a/crates/webhook/src/domain/ingestion/test.rs b/crates/webhook/src/domain/ingestion/test.rsindex f20ae8735b..86bad88071 100644--- a/crates/webhook/src/domain/ingestion/test.rs+++ b/crates/webhook/src/domain/ingestion/test.rs@@ -21,7 +21,8 @@ use channels::domain::{};use chrono::{DateTime, Utc};use documents::domain::events::{- DocumentCopiedMetadata, DocumentCreatedMetadata, DocumentDeletedMetadata,+ DocumentContentUploadedMetadata, DocumentCopiedMetadata, DocumentCreatedMetadata,+ DocumentDeletedMetadata, DocumentPurgedMetadata, DocumentSyncContentUpdatedMetadata,DocumentUpdatedMetadata,};use entity_access::domain::models::{@@ -493,6 +494,38 @@ fn document_event_cases() -> Vec<EventCase> {]}+fn search_only_document_event_cases() -> Vec<(&'static str, Event<DocumentTopicEvent>)> {+ vec![+ (+ "document.content_uploaded",+ Event::new(DocumentTopicEvent::ContentUploaded(+ DocumentContentUploadedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ owner: user_id("macro|[email protected]"),+ file_type: "pdf".parse().expect("valid file type"),+ document_version_id: Some("convert".to_string()),+ },+ )),+ ),+ (+ "document.sync_content_updated",+ Event::new(DocumentTopicEvent::SyncContentUpdated(+ DocumentSyncContentUpdatedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ file_type: "md".parse().expect("valid file type"),+ document_version_id: None,+ },+ )),+ ),+ (+ "document.purged",+ Event::new(DocumentTopicEvent::Purged(DocumentPurgedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ })),+ ),+ ]+}+fn channel_event_cases() -> Vec<EventCase> {let channel_id = Uuid::parse_str("44444444-4444-4444-4444-444444444444").unwrap();let message_id = Uuid::parse_str("55555555-5555-5555-5555-555555555555").unwrap();@@ -825,6 +858,45 @@ async fn normalizes_and_matches_all_fourteen_event_variants() {}}+#[tokio::test]+async fn search_only_document_events_skip_webhook_delivery() {+ let access = MockAccessService {+ users: vec![user_id(PERSONAL_WORKSPACE_ID)],+ failure: Some(AccessFailure::Internal),+ calls: Arc::default(),+ };+ let repository = MockRepository::new(+ vec![PERSONAL_WORKSPACE_ID.to_string()],+ vec![webhook("wh_match", PERSONAL_WORKSPACE_ID)],+ );+ {+ let mut state = lock(&repository.state);+ state.fail_workspace_resolution = true;+ state.fail_matching = true;+ }+ let enqueuer = MockEnqueuer::default();+ let service = service(access.clone(), repository.clone(), enqueuer.clone());++ for (event_name, event) in search_only_document_event_cases() {+ assert!(+ normalized_document_event(&event)+ .unwrap_or_else(|error| panic!("{event_name} normalization failed: {error}"))+ .is_none(),+ "{event_name} should not normalize to a public webhook event"+ );+ service+ .ingest_document_event(event)+ .await+ .unwrap_or_else(|error| panic!("{event_name} ingestion failed: {error}"));+ }++ assert!(lock(&access.calls).is_empty());+ let repository_state = lock(&repository.state);+ assert!(repository_state.workspace_calls.is_empty());+ assert!(repository_state.match_calls.is_empty());+ assert!(lock(&enqueuer.state).attempted_messages.is_empty());+}+#[tokio::test]async fn normalizes_all_webhook_variants_with_direct_workspace_ownership() {for event_case in webhook_event_cases() {diff --git a/infra/stacks/deleted-item-poller/index.ts b/infra/stacks/deleted-item-poller/index.tsindex 90ff7e1e06..8e00fafd17 100644--- a/infra/stacks/deleted-item-poller/index.ts+++ b/infra/stacks/deleted-item-poller/index.ts@@ -1,6 +1,6 @@import * as aws from '@pulumi/aws';import * as pulumi from '@pulumi/pulumi';-import { config, getSearchEventQueue, stack } from '../../packages/shared';+import { config, stack } from '../../packages/shared';import { get_coparse_api_vpc } from '../../packages/vpc';import { DeleteItemPoller } from './lambda';@@ -42,12 +42,10 @@ const deleteChatQueueArn: pulumi.Output<string> = cloudStorageServiceStack.getOutput('deleteChatQueueArn').apply((arn) => arn as string);-const { searchEventQueueArn } = getSearchEventQueue();-const vpc = get_coparse_api_vpc();const deletedItemPoller = new DeleteItemPoller('deleted-item-poller', {- queueArns: [deleteDocumentQueueArn, deleteChatQueueArn, searchEventQueueArn],+ queueArns: [deleteDocumentQueueArn, deleteChatQueueArn],vpc,envVars: {DATABASE_URL: pulumi.interpolate`${DATABASE_URL}`,diff --git a/infra/stacks/search-upload/index.ts b/infra/stacks/search-upload/index.tsindex 287ed92713..5afe2f32e4 100644--- a/infra/stacks/search-upload/index.ts+++ b/infra/stacks/search-upload/index.ts@@ -2,7 +2,6 @@ import * as aws from '@pulumi/aws';import * as pulumi from '@pulumi/pulumi';import {config,- getSearchEventQueue,getServiceUrl,ServiceUrl,stack,@@ -15,8 +14,6 @@ const tags = {project: 'cloud-storage-search',};-const { searchEventQueueArn } = getSearchEventQueue();-const DOCUMENT_STORAGE_SERVICE_AUTH_KEY = aws.secretsmanager.getSecretVersionOutput({secretId: config.require('document_storage_service_auth_key'),@@ -36,7 +33,6 @@ const searchUploadHandler = new SearchUploadHandler(),DOCUMENT_STORAGE_SERVICE_AUTH_KEY: pulumi.interpolate`${DOCUMENT_STORAGE_SERVICE_AUTH_KEY}`,},- searchEventQueueArn,tags,});diff --git a/infra/stacks/search-upload/search-upload-lambda.ts b/infra/stacks/search-upload/search-upload-lambda.tsindex ac442b56da..9176b4e5a0 100644--- a/infra/stacks/search-upload/search-upload-lambda.ts+++ b/infra/stacks/search-upload/search-upload-lambda.ts@@ -17,7 +17,6 @@ type EnvVars = {type Args = {envVars: EnvVars;- searchEventQueueArn: pulumi.Output<string> | string;tags: { [key: string]: string };};@@ -31,28 +30,10 @@ export class SearchUploadHandler extends pulumi.ComponentResource {opts?: pulumi.ComponentResourceOptions) {super('my:components:SearchUploadHandler', name, {}, opts);- const { envVars, searchEventQueueArn, tags } = args;+ const { envVars, tags } = args;this.tags = tags;- const sqsPolicy = new aws.iam.Policy(- `${LAMBA_BASE_NAME}-sqs-policy`,- {- policy: pulumi.output({- Version: '2012-10-17',- Statement: [- {- Action: ['sqs:SendMessage'],- Resource: [searchEventQueueArn],- Effect: 'Allow',- },- ],- }),- tags: this.tags,- },- { parent: this }- );-this.role = new aws.iam.Role(`${LAMBA_BASE_NAME}-role`,{@@ -73,7 +54,6 @@ export class SearchUploadHandler extends pulumi.ComponentResource {aws.iam.ManagedPolicy.AWSLambdaBasicExecutionRole,aws.iam.ManagedPolicy.AWSLambdaRole,aws.iam.ManagedPolicy.CloudWatchLogsFullAccess,- sqsPolicy.arn,],tags: this.tags,},diff --git a/packages/sdk/generated/storage/index.ts b/packages/sdk/generated/storage/index.tsindex 0a4ed6ff14..97980716b0 100644--- a/packages/sdk/generated/storage/index.ts+++ b/packages/sdk/generated/storage/index.ts@@ -1,4 +1,4 @@// This file is auto-generated by @hey-api/openapi-tsexport { type Options, Sdk } from './sdk.gen';-export type { AccessLevel, Activity, ActivityType, AddFavoriteData, AddFavoriteError, AddFavoriteErrors, AddFavoriteRequest, AddFavoriteResponse, AddFavoriteResponses, AddParticipantsData, AddParticipantsError, AddParticipantsErrors, AddParticipantsRequest, AddParticipantsResponses, AddPinHandlerData, AddPinHandlerError, AddPinHandlerErrors, AddPinHandlerResponse, AddPinHandlerResponses, AddPinRequest, Anchor, AnchorId, AnchorRequest, AnchorResponse, AnnotationIncrementalUpdate, ApiActivity, ApiAttachmentChannelReference, ApiAttachmentEntityReference, ApiAttachmentGenericReference, ApiChannelAttachment, ApiChannelAttachmentsPage, ApiChannelContextMessage, ApiChannelDetail, ApiChannelListMessage, ApiChannelListPage, ApiChannelListParticipant, ApiChannelListType, ApiChannelMessage, ApiChannelMessageKind, ApiChannelMessagesPage, ApiChannelParticipant, ApiChannelWithLatest, ApiCountedReaction, ApiEntityFilterAst, ApiGroupByField, ApiGroupMeta, ApiMessageAttachment, ApiMessageSender, ApiMessageSenderType, ApiParticipantListRole, ApiParticipantRole, ApiPropertyEntityType, ApiResolvedChannelMessage, ApiThreadInfo, ApiThreadReply, BasicDocument, BasicDocumentSubType, BomPart, Bot, BotChannel, BotChannelType, BotId, BotKind, BotOwner, BotToken, BranchNameResponse, BTreeMap, BulkWakeupRequest, BulkWakeupResponse, BulkWakeupSyncServiceDocumentsData, BulkWakeupSyncServiceDocumentsErrors, BulkWakeupSyncServiceDocumentsResponse, BulkWakeupSyncServiceDocumentsResponses, CallActiveResponse, CallFilters, CallRecord, CallRecordParticipant, CallRecordPreview, CallRecordPreviewData, CallRecordTranscriptSegment, CallStatus, CallTokenResponse, CallWebhookData, CallWebhookError, CallWebhookErrors, CallWebhookResponses, Channel, ChannelCreatedMetadata, ChannelDeletedMetadata, ChannelEventAttachment, ChannelFilters, ChannelJoinCodeResponse, ChannelMessage, ChannelMessageAttachmentCreatedMetadata, ChannelMessageAttachmentRemovedMetadata, ChannelMessageDeletedMetadata, ChannelMessageFilters, ChannelMessagePatchedMetadata, ChannelMessagePostedMetadata, ChannelParticipant, ChannelParticipantAddedMetadata, ChannelParticipantRemovedMetadata, ChannelPreview, ChannelPreviewData, ChannelSender, ChannelSharePermission, ChannelThreadFilters, ChannelTopicEvent, ChannelType, ChannelUpdatedMetadata, ChannelWebhookRequest, ChannelWebhookResponse, ChannelWithParticipants, Chat, ChatFilters, CheckActiveCallData, CheckActiveCallError, CheckActiveCallErrors, CheckActiveCallResponse, CheckActiveCallResponses, ClientOptions, CloudStorageItemType, Comment, CommentThread, CopyDocumentData, CopyDocumentError, CopyDocumentErrors, CopyDocumentQueryParams, CopyDocumentRequest, CopyDocumentResponse, CopyDocumentResponse2, CopyDocumentResponses, CreateAnchorData, CreateAnchorError, CreateAnchorErrors, CreateAnchorResponse, CreateAnchorResponses, CreateBulkDocumentResponse, CreateBulkDocumentResponseData, CreateChannelData, CreateChannelError, CreateChannelErrors, CreateChannelRequest, CreateChannelResponse, CreateChannelResponse2, CreateChannelResponses, CreateChannelScopedBotData, CreateChannelScopedBotError, CreateChannelScopedBotErrors, CreateChannelScopedBotRequest, CreateChannelScopedBotResponse, CreateChannelScopedBotResponse2, CreateChannelScopedBotResponses, CreateCommentData, CreateCommentError, CreateCommentErrors, CreateCommentRequest, CreateCommentResponse, CreateCommentResponse2, CreateCommentResponses, CreateCrmCommentData, CreateCrmCommentError, CreateCrmCommentErrors, CreateCrmCommentRequest, CreateCrmCommentResponse, CreateCrmCommentResponses, CreateCrmCompanyData, CreateCrmCompanyError, CreateCrmCompanyErrors, CreateCrmCompanyRequest, CreateCrmCompanyResponse, CreateCrmCompanyResponses, CreateCrmContactData, CreateCrmContactError, CreateCrmContactErrors, CreateCrmContactRequest, CreateCrmContactResponse, CreateCrmContactResponses, CreateDocumentData, CreateDocumentError, CreateDocumentErrors, CreateDocumentRequest, CreateDocumentResponse, CreateDocumentResponseData, CreateDocumentResponses, CreateEntityMentionData, CreateEntityMentionError, CreateEntityMentionErrors, CreateEntityMentionRequest, CreateEntityMentionResponse, CreateEntityMentionResponse2, CreateEntityMentionResponses, CreateInstructionsDocumentResponse, CreateInstructionsHandlerData, CreateInstructionsHandlerError, CreateInstructionsHandlerErrors, CreateInstructionsHandlerResponse, CreateInstructionsHandlerResponses, CreateMarkdownDocumentRequest, CreateMarkdownDocumentResponse, CreateMarkdownHandlerData, CreateMarkdownHandlerError, CreateMarkdownHandlerErrors, CreateMarkdownHandlerResponse, CreateMarkdownHandlerResponses, CreateProjectHandlerData, CreateProjectHandlerError, CreateProjectHandlerErrors, CreateProjectHandlerResponse, CreateProjectHandlerResponses, CreateProjectRequest, CreateProjectResponse, CreateSnippetHandlerData, CreateSnippetHandlerError, CreateSnippetHandlerErrors, CreateSnippetHandlerResponse, CreateSnippetHandlerResponses, CreateSnippetRequest, CreateSnippetResponse, CreateTaskHandlerData, CreateTaskHandlerError, CreateTaskHandlerErrors, CreateTaskHandlerResponse, CreateTaskHandlerResponses, CreateTaskRequest, CreateTaskResponse, CreateUnthreadedAnchorRequest, CreateUnthreadedAnchorResponse, CreateUnthreadedPdfAnchorRequest, CreateViewHandlerData, CreateViewHandlerError, CreateViewHandlerErrors, CreateViewHandlerResponse, CreateViewHandlerResponses, CreateViewRequest, CreateWebhookData, CreateWebhookError, CreateWebhookErrors, CreateWebhookRequest, CreateWebhookResponse, CreateWebhookResponse2, CreateWebhookResponses, CrmComment, CrmCommentEntityType, CrmCommentThread, CrmCompanyFilters, CrmCompanyResponse, CrmContactResponse, CrmDomainResponse, CrmPermissionRole, CrmTeamSettingsResponse, CrmThread, CustomSpeakerAssignment, DataType, DeleteAnchorData, DeleteAnchorError, DeleteAnchorErrors, DeleteAnchorInfo, DeleteAnchorResponse, DeleteAnchorResponses, DeleteCallRecordData, DeleteCallRecordError, DeleteCallRecordErrors, DeleteCallRecordResponse, DeleteCallRecordResponses, DeleteChannelData, DeleteChannelError, DeleteChannelErrors, DeleteChannelResponse, DeleteChannelResponses, DeleteCommentData, DeleteCommentError, DeleteCommentErrors, DeleteCommentRequest, DeleteCommentResponse, DeleteCommentResponse2, DeleteCommentResponses, DeleteCrmCommentData, DeleteCrmCommentError, DeleteCrmCommentErrors, DeleteCrmCommentResponse, DeleteCrmCommentResponses, DeleteCrmCommentResult, DeleteDocumentData, DeleteDocumentError, DeleteDocumentErrors, DeleteDocumentResponse, DeleteDocumentResponses, DeleteEntityMentionData, DeleteEntityMentionError, DeleteEntityMentionErrors, DeleteEntityMentionResponse, DeleteEntityMentionResponse2, DeleteEntityMentionResponses, DeleteHistoryHandlerData, DeleteHistoryHandlerError, DeleteHistoryHandlerErrors, DeleteHistoryHandlerResponse, DeleteHistoryHandlerResponses, DeleteMessageData, DeleteMessageError, DeleteMessageErrors, DeleteMessageQuery, DeleteMessageResponse, DeleteMessageResponses, DeleteProjectHandlerData, DeleteProjectHandlerError, DeleteProjectHandlerErrors, DeleteProjectHandlerResponse, DeleteProjectHandlerResponses, DeleteThreadInfo, DeleteUnthreadedAnchorRequest, DeleteUnthreadedAnchorResponse, DeleteUnthreadedPdfAnchorRequest, DeleteUserDocumentViewLocationData, DeleteUserDocumentViewLocationError, DeleteUserDocumentViewLocationErrors, DeleteUserDocumentViewLocationResponse, DeleteUserDocumentViewLocationResponses, DeleteViewHandlerData, DeleteViewHandlerError, DeleteViewHandlerErrors, DeleteViewHandlerResponses, DeleteWebhookData, DeleteWebhookError, DeleteWebhookErrors, DeleteWebhookResponse, DeleteWebhookResponses, DocumentBasic, DocumentContent, DocumentContentLocation, DocumentContentState, DocumentCopiedMetadata, DocumentCreatedMetadata, DocumentDeletedMetadata, DocumentFilters, DocumentInteractionMetadata, DocumentMetadata, DocumentMetadataWithContent, DocumentPermissionsToken, DocumentPermissionsTokenRequest, DocumentPermissionsTokenResponse, DocumentPreview, DocumentPreviewData, DocumentPreviewDataSubType, DocumentResponse, DocumentResponseMetadata, DocumentResponseMetadataWithContent, DocumentStorageServiceApiVersion, DocumentSubType, DocumentTeamShareResponse, DocumentTopicEvent, DocumentUpdatedMetadata, EditAnchorData, EditAnchorError, EditAnchorErrors, EditAnchorRequest, EditAnchorResponse, EditAnchorResponse2, EditAnchorResponses, EditCallRecordData, EditCallRecordError, EditCallRecordErrors, EditCallRecordRequest, EditCallRecordResponse, EditCallRecordResponses, EditCallTranscriptData, EditCallTranscriptError, EditCallTranscriptErrors, EditCallTranscriptRequest, EditCallTranscriptResponse, EditCallTranscriptResponses, EditCommentData, EditCommentError, EditCommentErrors, EditCommentRequest, EditCommentResponse, EditCommentResponse2, EditCommentResponses, EditCrmCommentData, EditCrmCommentError, EditCrmCommentErrors, EditCrmCommentRequest, EditCrmCommentResponse, EditCrmCommentResponses, EditDocumentData, EditDocumentError, EditDocumentErrors, EditDocumentResponse, EditDocumentResponse2, EditDocumentResponses, EditDocumentServiceArgs, EditPdfAnchorRequest, EditPdfPlaceableCommentAnchorRequest, EditProjectV2Data, EditProjectV2Error, EditProjectV2Errors, EditProjectV2Response, EditProjectV2Responses, EditThreadV2Data, EditThreadV2Error, EditThreadV2Errors, EditThreadV2Response, EditThreadV2Responses, EmailFilters, EmptyResponse, EntityFilters, EntityPermission, EntityPermissionResponse, EntityReference, EntityType, ErrorResponse, ExcludedDefaultView, ExcludeDefaultViewHandlerData, ExcludeDefaultViewHandlerError, ExcludeDefaultViewHandlerErrors, ExcludeDefaultViewHandlerResponses, ExcludeDefaultViewRequest, ExportDocumentData, ExportDocumentError, ExportDocumentErrors, ExportDocumentResponse, ExportDocumentResponse2, ExportDocumentResponses, Favorite, FavoriteEntityRef, FavoritesList, FileSystemNodeWithIds, FileType, FileTypeUpdate, FolderItem, ForeignEntity, ForeignEntityFilters, GenericErrorResponse, GenericResponse, GenericSuccessResponse, GetActivitiesResponse, GetActivityData, GetActivityError, GetActivityErrors, GetActivityResponse, GetActivityResponses, GetAttachmentReferencesData, GetAttachmentReferencesError, GetAttachmentReferencesErrors, GetAttachmentReferencesResponse, GetAttachmentReferencesResponse2, GetAttachmentReferencesResponses, GetBatchCallRecordPreviewData, GetBatchCallRecordPreviewError, GetBatchCallRecordPreviewErrors, GetBatchCallRecordPreviewRequest, GetBatchCallRecordPreviewResponse, GetBatchCallRecordPreviewResponse2, GetBatchCallRecordPreviewResponses, GetBatchChannelPreviewData, GetBatchChannelPreviewError, GetBatchChannelPreviewErrors, GetBatchChannelPreviewRequest, GetBatchChannelPreviewResponse, GetBatchChannelPreviewResponse2, GetBatchChannelPreviewResponses, GetBatchPreviewHandlerData, GetBatchPreviewHandlerError, GetBatchPreviewHandlerErrors, GetBatchPreviewHandlerResponse, GetBatchPreviewHandlerResponses, GetBatchPreviewRequest, GetBatchPreviewResponse, GetBatchProjectPreviewData, GetBatchProjectPreviewError, GetBatchProjectPreviewErrors, GetBatchProjectPreviewRequest, GetBatchProjectPreviewResponse, GetBatchProjectPreviewResponse2, GetBatchProjectPreviewResponses, GetCallRecordData, GetCallRecordError, GetCallRecordErrors, GetCallRecordResponse, GetCallRecordResponses, GetChannelAttachmentsData, GetChannelAttachmentsError, GetChannelAttachmentsErrors, GetChannelAttachmentsResponse, GetChannelAttachmentsResponses, GetChannelData, GetChannelError, GetChannelErrors, GetChannelJoinLinkData, GetChannelJoinLinkError, GetChannelJoinLinkErrors, GetChannelJoinLinkResponse, GetChannelJoinLinkResponses, GetChannelMessagesData, GetChannelMessagesError, GetChannelMessagesErrors, GetChannelMessagesResponse, GetChannelMessagesResponses, GetChannelParticipantsData, GetChannelParticipantsError, GetChannelParticipantsErrors, GetChannelParticipantsResponse, GetChannelParticipantsResponses, GetChannelResponse, GetChannelResponses, GetChannelsData, GetChannelsError, GetChannelsErrors, GetChannelsResponse, GetChannelsResponses, GetCompanyData, GetCompanyError, GetCompanyErrors, GetCompanyResponse, GetCompanyResponses, GetContactData, GetContactError, GetContactErrors, GetContactResponse, GetContactResponses, GetCrmTeamSettingsData, GetCrmTeamSettingsError, GetCrmTeamSettingsErrors, GetCrmTeamSettingsResponse, GetCrmTeamSettingsResponses, GetDocumentAnchorsData, GetDocumentAnchorsError, GetDocumentAnchorsErrors, GetDocumentAnchorsResponse, GetDocumentAnchorsResponses, GetDocumentBranchNameData, GetDocumentBranchNameError, GetDocumentBranchNameErrors, GetDocumentBranchNameResponse, GetDocumentBranchNameResponses, GetDocumentByTeamSlugData, GetDocumentByTeamSlugError, GetDocumentByTeamSlugErrors, GetDocumentByTeamSlugResponse, GetDocumentByTeamSlugResponses, GetDocumentCommentsData, GetDocumentCommentsError, GetDocumentCommentsErrors, GetDocumentCommentsResponse, GetDocumentCommentsResponses, GetDocumentData, GetDocumentError, GetDocumentErrors, GetDocumentGithubPullRequestsData, GetDocumentGithubPullRequestsError, GetDocumentGithubPullRequestsErrors, GetDocumentGithubPullRequestsResponse, GetDocumentGithubPullRequestsResponses, GetDocumentKeyResponse, GetDocumentKeyResponseData, GetDocumentListHandlerData, GetDocumentListHandlerError, GetDocumentListHandlerErrors, GetDocumentListHandlerResponse, GetDocumentListHandlerResponses, GetDocumentListResult, GetDocumentLocationV3Data, GetDocumentLocationV3Error, GetDocumentLocationV3Errors, GetDocumentLocationV3Response, GetDocumentLocationV3Responses, GetDocumentPermissionsResponseDataV2, GetDocumentPermissionsTokenData, GetDocumentPermissionsTokenError, GetDocumentPermissionsTokenErrors, GetDocumentPermissionsTokenResponse, GetDocumentPermissionsTokenResponses, GetDocumentPermissionsV2Data, GetDocumentPermissionsV2Error, GetDocumentPermissionsV2Errors, GetDocumentPermissionsV2Response, GetDocumentPermissionsV2Responses, GetDocumentProcessingResult, GetDocumentProcessingResultData, GetDocumentProcessingResultError, GetDocumentProcessingResultErrors, GetDocumentProcessingResultResponse, GetDocumentProcessingResultResponse2, GetDocumentProcessingResultResponses, GetDocumentResponse, GetDocumentResponse2, GetDocumentResponseData, GetDocumentResponses, GetDocumentSearchResponse, GetDocumentShortIdData, GetDocumentShortIdError, GetDocumentShortIdErrors, GetDocumentShortIdResponse, GetDocumentShortIdResponses, GetDocumentsResponse, GetDocumentTeamShareData, GetDocumentTeamShareError, GetDocumentTeamShareErrors, GetDocumentTeamShareResponse, GetDocumentTeamShareResponses, GetDocumentUserAccessLevelResponse, GetDocumentVersionData, GetDocumentVersionError, GetDocumentVersionErrors, GetDocumentVersionResponse, GetDocumentVersionResponses, GetDocumentViewsHandlerData, GetDocumentViewsHandlerError, GetDocumentViewsHandlerErrors, GetDocumentViewsHandlerResponse, GetDocumentViewsHandlerResponses, GetEntityPermissionData, GetEntityPermissionErrors, GetEntityPermissionResponse, GetEntityPermissionResponses, GetForeignEntityData, GetForeignEntityError, GetForeignEntityErrors, GetForeignEntityResponse, GetForeignEntityResponses, GetHistoryHandlerData, GetHistoryHandlerError, GetHistoryHandlerErrors, GetHistoryHandlerResponse, GetHistoryHandlerResponses, GetInstructionsDocumentResponse, GetInstructionsHandlerData, GetInstructionsHandlerError, GetInstructionsHandlerErrors, GetInstructionsHandlerResponse, GetInstructionsHandlerResponses, GetItemsSoupData, GetItemsSoupError, GetItemsSoupErrors, GetItemsSoupResponse, GetItemsSoupResponses, GetLocationHandlerData, GetLocationHandlerError, GetLocationHandlerErrors, GetLocationHandlerResponse, GetLocationHandlerResponses, GetMessageWithContextData, GetMessageWithContextError, GetMessageWithContextErrors, GetMessageWithContextResponse, GetMessageWithContextResponse2, GetMessageWithContextResponses, GetOrCreateAction, GetOrCreateCallData, GetOrCreateCallError, GetOrCreateCallErrors, GetOrCreateCallResponse, GetOrCreateCallResponses, GetOrCreateChannelResponse, GetOrCreateDmData, GetOrCreateDmError, GetOrCreateDmErrors, GetOrCreateDmRequest, GetOrCreateDmResponse, GetOrCreateDmResponses, GetOrCreatePrivateData, GetOrCreatePrivateError, GetOrCreatePrivateErrors, GetOrCreatePrivateRequest, GetOrCreatePrivateResponse, GetOrCreatePrivateResponses, GetPendingProjectsHandlerData, GetPendingProjectsHandlerError, GetPendingProjectsHandlerErrors, GetPendingProjectsHandlerResponse, GetPendingProjectsHandlerResponses, GetPinsHandlerData, GetPinsHandlerError, GetPinsHandlerErrors, GetPinsHandlerResponse, GetPinsHandlerResponses, GetPinsResponse, GetProjectContentHandlerData, GetProjectContentHandlerError, GetProjectContentHandlerErrors, GetProjectContentHandlerResponse, GetProjectContentHandlerResponses, GetProjectContentResponse, GetProjectHandlerData, GetProjectHandlerError, GetProjectHandlerErrors, GetProjectHandlerResponse, GetProjectHandlerResponses, GetProjectPermissionsV2Data, GetProjectPermissionsV2Error, GetProjectPermissionsV2Errors, GetProjectPermissionsV2Response, GetProjectPermissionsV2Responses, GetProjectResponse, GetProjectResponseData, GetProjectsHandlerData, GetProjectsHandlerError, GetProjectsHandlerErrors, GetProjectsHandlerResponse, GetProjectsHandlerResponses, GetProjectsResponse, GetProjectUserAccessLevelData, GetProjectUserAccessLevelError, GetProjectUserAccessLevelErrors, GetProjectUserAccessLevelResponse, GetProjectUserAccessLevelResponses, GetRecentActivityHandlerData, GetRecentActivityHandlerError, GetRecentActivityHandlerErrors, GetRecentActivityHandlerResponse, GetRecentActivityHandlerResponses, GetRingStatusData, GetRingStatusError, GetRingStatusErrors, GetRingStatusResponse, GetRingStatusResponses, GetThreadRepliesData, GetThreadRepliesError, GetThreadRepliesErrors, GetThreadRepliesResponse, GetThreadRepliesResponses, GetUserDocumentsHandlerData, GetUserDocumentsHandlerError, GetUserDocumentsHandlerErrors, GetUserDocumentsHandlerResponse, GetUserDocumentsHandlerResponses, GetUserDocumentViewLocationData, GetUserDocumentViewLocationError, GetUserDocumentViewLocationErrors, GetUserDocumentViewLocationResponse, GetUserDocumentViewLocationResponses, GetUserHistoryResponse, GetViewsHandlerData, GetViewsHandlerError, GetViewsHandlerErrors, GetViewsHandlerResponse, GetViewsHandlerResponses, GetWebhookData, GetWebhookError, GetWebhookErrors, GetWebhookResponse, GetWebhookResponses, GithubPullRequest, GithubPullRequestCheckRun, GithubPullRequestComment, GithubPullRequestsResponse, GroupedInitialParams, GroupedPageParams, GroupedSoupGroupPage, GroupedSoupInitialPage, GroupedSoupPage, GroupedSoupSort, HandlerData, HandlerError, HandlerErrors, HandlerResponse, HandlerResponses, HashMap, HealthHandlerData, HealthHandlerResponse, HealthHandlerResponses, HighlightType, IngestTranscriptData, IngestTranscriptError, IngestTranscriptErrors, IngestTranscriptResponses, InitializeUserDocumentsData, InitializeUserDocumentsError, InitializeUserDocumentsErrors, InitializeUserDocumentsResponse, InitializeUserDocumentsResponses, InstallSyncData, InteractionReason, Item, ItemWithUserAccessLevel, JobProcessingResultHandlerData, JobProcessingResultHandlerError, JobProcessingResultHandlerErrors, JobProcessingResultHandlerResponse, JobProcessingResultHandlerResponses, JoinChannelByCodeData, JoinChannelByCodeError, JoinChannelByCodeErrors, JoinChannelByCodeResponses, JoinChannelData, JoinChannelError, JoinChannelErrors, JoinChannelResponses, LatestMessage, LeaveCallResponse, LeaveChannelData, LeaveChannelError, LeaveChannelErrors, LeaveChannelResponses, LeaveOrEndCallData, LeaveOrEndCallError, LeaveOrEndCallErrors, LeaveOrEndCallResponse, LeaveOrEndCallResponses, ListBotChannelsData, ListBotChannelsError, ListBotChannelsErrors, ListBotChannelsResponse, ListBotChannelsResponses, ListCompanyContactsData, ListCompanyContactsError, ListCompanyContactsErrors, ListCompanyContactsResponse, ListCompanyContactsResponses, ListCrmCommentsData, ListCrmCommentsError, ListCrmCommentsErrors, ListCrmCommentsResponse, ListCrmCommentsResponses, ListFavoritesData, ListFavoritesError, ListFavoritesErrors, ListFavoritesResponse, ListFavoritesResponses, ListWebhooksData, ListWebhooksError, ListWebhooksErrors, ListWebhooksResponse, ListWebhooksResponse2, ListWebhooksResponses, LocationResponseData, LocationResponseV3, MacroUserIdStr, Mentions, NewChannelAttachment, NotificationFilters, Params, ParticipantRole, PatchChannelData, PatchChannelError, PatchChannelErrors, PatchChannelRequest, PatchChannelResponse, PatchChannelResponses, PatchMessageData, PatchMessageError, PatchMessageErrors, PatchMessageRequest, PatchMessageResponse, PatchMessageResponses, PatchProjectRequestV2, PatchThreadRequestV2, PatchViewHandlerData, PatchViewHandlerError, PatchViewHandlerErrors, PatchViewHandlerResponses, PatchWebhookData, PatchWebhookError, PatchWebhookErrors, PatchWebhookRequest, PatchWebhookResponse, PatchWebhookResponses, PdfAnchor, PdfAnchorId, PdfAnchorRequest, PdfHighlightAnchor, PdfHighlightAnchorRequest, PdfHighlightRect, PdfHighlightRectAnchorRequest, PdfPlaceableCommentAnchor, PdfPlaceableCommentAnchorRequest, PermanentlyDeleteDocumentData, PermanentlyDeleteDocumentError, PermanentlyDeleteDocumentErrors, PermanentlyDeleteDocumentResponse, PermanentlyDeleteDocumentResponses, PermanentlyDeleteProjectData, PermanentlyDeleteProjectError, PermanentlyDeleteProjectErrors, PermanentlyDeleteProjectResponse, PermanentlyDeleteProjectResponses, PinnedItem, PinRequest, PostActivityData, PostActivityError, PostActivityErrors, PostActivityRequest, PostActivityResponse, PostActivityResponses, PostChannelBotWebhookData, PostChannelBotWebhookError, PostChannelBotWebhookErrors, PostChannelBotWebhookResponse, PostChannelBotWebhookResponses, PostChannelMessagesData, PostChannelMessagesError, PostChannelMessagesErrors, PostChannelMessagesResponse, PostChannelMessagesResponses, PostGroupedSoupAstGroupPageRequest, PostGroupedSoupAstInitialRequest, PostGroupedSoupAstRequest, PostItemsSoupAstData, PostItemsSoupAstError, PostItemsSoupAstErrors, PostItemsSoupAstGroupedData, PostItemsSoupAstGroupedError, PostItemsSoupAstGroupedErrors, PostItemsSoupAstGroupedResponse, PostItemsSoupAstGroupedResponses, PostItemsSoupAstResponse, PostItemsSoupAstResponses, PostItemsSoupData, PostItemsSoupError, PostItemsSoupErrors, PostItemsSoupResponse, PostItemsSoupResponses, PostMessageData, PostMessageError, PostMessageErrors, PostMessageRequest, PostMessageResponse, PostMessageResponse2, PostMessageResponses, PostReactionData, PostReactionError, PostReactionErrors, PostReactionRequest, PostReactionResponse, PostReactionResponses, PostSoupAstRequest, PostSoupRequest, PostTypingData, PostTypingError, PostTypingErrors, PostTypingRequest, PostTypingResponse, PostTypingResponses, PresaveDocumentHandlerData, PresaveDocumentHandlerError, PresaveDocumentHandlerErrors, PresaveDocumentHandlerResponse, PresaveDocumentHandlerResponses, PreSaveDocumentRequest, PreSaveDocumentResponse, PreSaveDocumentResponseData, PresignedUrl, Project, ProjectDeleteResponseData, ProjectDocumentStatus, ProjectFilters, ProjectPreview, ProjectPreviewData, PropertyDefinition, PropertyFilter, PropertyInput, PropertyOwner, PropertyValue, PutCrmTeamSettingsData, PutCrmTeamSettingsError, PutCrmTeamSettingsErrors, PutCrmTeamSettingsResponse, PutCrmTeamSettingsResponses, ReactionAction, RecentlyDeletedData, RecentlyDeletedError, RecentlyDeletedErrors, RecentlyDeletedResponse, RecentlyDeletedResponseData, RecentlyDeletedResponses, RemoveBotFromChannelByBotData, RemoveBotFromChannelByBotError, RemoveBotFromChannelByBotErrors, RemoveBotFromChannelByBotResponse, RemoveBotFromChannelByBotResponses, RemoveFavoriteByEntityData, RemoveFavoriteByEntityError, RemoveFavoriteByEntityErrors, RemoveFavoriteByEntityResponses, RemoveParticipantsData, RemoveParticipantsError, RemoveParticipantsErrors, RemoveParticipantsRequest, RemoveParticipantsResponses, RemovePinHandlerData, RemovePinHandlerError, RemovePinHandlerErrors, RemovePinHandlerResponse, RemovePinHandlerResponses, ReorderFavoritesData, ReorderFavoritesError, ReorderFavoritesErrors, ReorderFavoritesRequest, ReorderFavoritesResponses, ReorderPinRequest, ReorderPinsHandlerData, ReorderPinsHandlerError, ReorderPinsHandlerErrors, ReorderPinsHandlerResponse, ReorderPinsHandlerResponses, ResolveChannelMessageData, ResolveChannelMessageError, ResolveChannelMessageErrors, ResolveChannelMessageResponse, ResolveChannelMessageResponses, RevertDeleteDocumentData, RevertDeleteDocumentError, RevertDeleteDocumentErrors, RevertDeleteDocumentResponse, RevertDeleteDocumentResponses, RevertDeleteProjectData, RevertDeleteProjectError, RevertDeleteProjectErrors, RevertDeleteProjectResponse, RevertDeleteProjectResponses, RingStatus, RingStatusResponse, S3ObjectInfo, SaveBomPart, SaveDocumentHandlerData, SaveDocumentHandlerError, SaveDocumentHandlerErrors, SaveDocumentHandlerResponse, SaveDocumentHandlerResponses, SaveDocumentRequest, SaveDocumentResponse, SaveDocumentResponseData, SetCompanyHiddenData, SetCompanyHiddenError, SetCompanyHiddenErrors, SetCompanyHiddenRequest, SetCompanyHiddenResponse, SetCompanyHiddenResponses, SetCompanyNameRequest, SetContactHiddenData, SetContactHiddenError, SetContactHiddenErrors, SetContactHiddenRequest, SetContactHiddenResponse, SetContactHiddenResponses, SetContactNameRequest, SetCrmCompanyNameData, SetCrmCompanyNameError, SetCrmCompanyNameErrors, SetCrmCompanyNameResponse, SetCrmCompanyNameResponses, SetCrmContactNameData, SetCrmContactNameError, SetCrmContactNameErrors, SetCrmContactNameResponse, SetCrmContactNameResponses, SetDocumentTeamShareData, SetDocumentTeamShareError, SetDocumentTeamShareErrors, SetDocumentTeamShareRequest, SetDocumentTeamShareResponse, SetDocumentTeamShareResponses, SetEmailSyncData, SetEmailSyncError, SetEmailSyncErrors, SetEmailSyncRequest, SetEmailSyncResponse, SetEmailSyncResponses, SetPropertyValue, SharedEmailFilter, SharePermissionV2, ShortIdResponse, SimpleMention, SimpleSaveData, SimpleSaveError, SimpleSaveErrors, SimpleSaveResponse, SimpleSaveResponses, SoupApiItem, SoupApiSort, SoupAttachment, SoupCallRecordParticipant, SoupCallRecordSoupPropertiesField, SoupChannel, SoupChannelThread, SoupChatSoupPropertiesField, SoupContact, SoupCountedReaction, SoupCrmCompanySoupPropertiesField, SoupCrmDomain, SoupDocumentSoupPropertiesField, SoupDocumentSubType, SoupEmailThreadPreview, SoupEnrichedEmailThreadPreviewSoupPropertiesField, SoupForeignEntity, SoupItem, SoupLabel, SoupLabelListVisibility, SoupLabelType, SoupMessageAttachment, SoupMessageListVisibility, SoupMessageSender, SoupMessageSenderType, SoupPage, SoupProjectSoupPropertiesField, SoupPropertiesField, SoupProperty, SoupThreadInfo, SoupThreadReply, StarterDocumentsResponse, String, SuccessResponse, SyncServiceVersionId, TagFilterMode, TaskFilters, TeamRole, Thread, ThreadResponse, ToggleShareWithTeamData, ToggleShareWithTeamError, ToggleShareWithTeamErrors, ToggleShareWithTeamResponse, ToggleShareWithTeamResponses, TranscriptSegmentRequest, TypedSuccessResponse, TypedSuccessResponseGetDocumentResponseData, TypingAction, UnthreadedPdfUuidRequest, UpdateChannelSharePermission, UpdateCrmTeamSettingsRequest, UpdateOperation, UpdateSharePermissionRequestV2, UploadDocumentStatus, UploadExtractFolderHandlerData, UploadExtractFolderHandlerError, UploadExtractFolderHandlerErrors, UploadExtractFolderHandlerResponse, UploadExtractFolderHandlerResponses, UploadExtractFolderRequest, UploadFolderHandlerData, UploadFolderHandlerError, UploadFolderHandlerErrors, UploadFolderHandlerResponse, UploadFolderHandlerResponses, UploadFolderRequest, UpsertHistoryHandlerData, UpsertHistoryHandlerError, UpsertHistoryHandlerErrors, UpsertHistoryHandlerResponse, UpsertHistoryHandlerResponses, UpsertUserDocumentViewLocationData, UpsertUserDocumentViewLocationError, UpsertUserDocumentViewLocationErrors, UpsertUserDocumentViewLocationRequest, UpsertUserDocumentViewLocationResponse, UpsertUserDocumentViewLocationResponses, UserActivitiesResponse, UserDocumentsResponse, UserDocumentViewLocation, UserDocumentViewLocationResponse, UserPinsResponse, UserViewsResponse, ValidateDocumentPermissionsTokenData, ValidateDocumentPermissionsTokenError, ValidateDocumentPermissionsTokenErrors, ValidateDocumentPermissionsTokenResponse, ValidateDocumentPermissionsTokenResponses, ValidateWebhookData, ValidateWebhookError, ValidateWebhookErrors, ValidateWebhookResponse, ValidateWebhookResponse2, ValidateWebhookResponses, Vec, View, ViewPatch, ViewsResponse, Webhook, WebhookEvent, WebhookFilter, WebhookScope, WebhookStatus, WebhookValidationTestEvent, WithCallId, WithChannelId, WithDocumentId, WithProjectId } from './types.gen';+export type { AccessLevel, Activity, ActivityType, AddFavoriteData, AddFavoriteError, AddFavoriteErrors, AddFavoriteRequest, AddFavoriteResponse, AddFavoriteResponses, AddParticipantsData, AddParticipantsError, AddParticipantsErrors, AddParticipantsRequest, AddParticipantsResponses, AddPinHandlerData, AddPinHandlerError, AddPinHandlerErrors, AddPinHandlerResponse, AddPinHandlerResponses, AddPinRequest, Anchor, AnchorId, AnchorRequest, AnchorResponse, AnnotationIncrementalUpdate, ApiActivity, ApiAttachmentChannelReference, ApiAttachmentEntityReference, ApiAttachmentGenericReference, ApiChannelAttachment, ApiChannelAttachmentsPage, ApiChannelContextMessage, ApiChannelDetail, ApiChannelListMessage, ApiChannelListPage, ApiChannelListParticipant, ApiChannelListType, ApiChannelMessage, ApiChannelMessageKind, ApiChannelMessagesPage, ApiChannelParticipant, ApiChannelWithLatest, ApiCountedReaction, ApiEntityFilterAst, ApiGroupByField, ApiGroupMeta, ApiMessageAttachment, ApiMessageSender, ApiMessageSenderType, ApiParticipantListRole, ApiParticipantRole, ApiPropertyEntityType, ApiResolvedChannelMessage, ApiThreadInfo, ApiThreadReply, BasicDocument, BasicDocumentSubType, BomPart, Bot, BotChannel, BotChannelType, BotId, BotKind, BotOwner, BotToken, BranchNameResponse, BTreeMap, BulkWakeupRequest, BulkWakeupResponse, BulkWakeupSyncServiceDocumentsData, BulkWakeupSyncServiceDocumentsErrors, BulkWakeupSyncServiceDocumentsResponse, BulkWakeupSyncServiceDocumentsResponses, CallActiveResponse, CallFilters, CallRecord, CallRecordParticipant, CallRecordPreview, CallRecordPreviewData, CallRecordTranscriptSegment, CallStatus, CallTokenResponse, CallWebhookData, CallWebhookError, CallWebhookErrors, CallWebhookResponses, Channel, ChannelCreatedMetadata, ChannelDeletedMetadata, ChannelEventAttachment, ChannelFilters, ChannelJoinCodeResponse, ChannelMessage, ChannelMessageAttachmentCreatedMetadata, ChannelMessageAttachmentRemovedMetadata, ChannelMessageDeletedMetadata, ChannelMessageFilters, ChannelMessagePatchedMetadata, ChannelMessagePostedMetadata, ChannelParticipant, ChannelParticipantAddedMetadata, ChannelParticipantRemovedMetadata, ChannelPreview, ChannelPreviewData, ChannelSender, ChannelSharePermission, ChannelThreadFilters, ChannelTopicEvent, ChannelType, ChannelUpdatedMetadata, ChannelWebhookRequest, ChannelWebhookResponse, ChannelWithParticipants, Chat, ChatFilters, CheckActiveCallData, CheckActiveCallError, CheckActiveCallErrors, CheckActiveCallResponse, CheckActiveCallResponses, ClientOptions, CloudStorageItemType, Comment, CommentThread, CopyDocumentData, CopyDocumentError, CopyDocumentErrors, CopyDocumentQueryParams, CopyDocumentRequest, CopyDocumentResponse, CopyDocumentResponse2, CopyDocumentResponses, CreateAnchorData, CreateAnchorError, CreateAnchorErrors, CreateAnchorResponse, CreateAnchorResponses, CreateBulkDocumentResponse, CreateBulkDocumentResponseData, CreateChannelData, CreateChannelError, CreateChannelErrors, CreateChannelRequest, CreateChannelResponse, CreateChannelResponse2, CreateChannelResponses, CreateChannelScopedBotData, CreateChannelScopedBotError, CreateChannelScopedBotErrors, CreateChannelScopedBotRequest, CreateChannelScopedBotResponse, CreateChannelScopedBotResponse2, CreateChannelScopedBotResponses, CreateCommentData, CreateCommentError, CreateCommentErrors, CreateCommentRequest, CreateCommentResponse, CreateCommentResponse2, CreateCommentResponses, CreateCrmCommentData, CreateCrmCommentError, CreateCrmCommentErrors, CreateCrmCommentRequest, CreateCrmCommentResponse, CreateCrmCommentResponses, CreateCrmCompanyData, CreateCrmCompanyError, CreateCrmCompanyErrors, CreateCrmCompanyRequest, CreateCrmCompanyResponse, CreateCrmCompanyResponses, CreateCrmContactData, CreateCrmContactError, CreateCrmContactErrors, CreateCrmContactRequest, CreateCrmContactResponse, CreateCrmContactResponses, CreateDocumentData, CreateDocumentError, CreateDocumentErrors, CreateDocumentRequest, CreateDocumentResponse, CreateDocumentResponseData, CreateDocumentResponses, CreateEntityMentionData, CreateEntityMentionError, CreateEntityMentionErrors, CreateEntityMentionRequest, CreateEntityMentionResponse, CreateEntityMentionResponse2, CreateEntityMentionResponses, CreateInstructionsDocumentResponse, CreateInstructionsHandlerData, CreateInstructionsHandlerError, CreateInstructionsHandlerErrors, CreateInstructionsHandlerResponse, CreateInstructionsHandlerResponses, CreateMarkdownDocumentRequest, CreateMarkdownDocumentResponse, CreateMarkdownHandlerData, CreateMarkdownHandlerError, CreateMarkdownHandlerErrors, CreateMarkdownHandlerResponse, CreateMarkdownHandlerResponses, CreateProjectHandlerData, CreateProjectHandlerError, CreateProjectHandlerErrors, CreateProjectHandlerResponse, CreateProjectHandlerResponses, CreateProjectRequest, CreateProjectResponse, CreateSnippetHandlerData, CreateSnippetHandlerError, CreateSnippetHandlerErrors, CreateSnippetHandlerResponse, CreateSnippetHandlerResponses, CreateSnippetRequest, CreateSnippetResponse, CreateTaskHandlerData, CreateTaskHandlerError, CreateTaskHandlerErrors, CreateTaskHandlerResponse, CreateTaskHandlerResponses, CreateTaskRequest, CreateTaskResponse, CreateUnthreadedAnchorRequest, CreateUnthreadedAnchorResponse, CreateUnthreadedPdfAnchorRequest, CreateViewHandlerData, CreateViewHandlerError, CreateViewHandlerErrors, CreateViewHandlerResponse, CreateViewHandlerResponses, CreateViewRequest, CreateWebhookData, CreateWebhookError, CreateWebhookErrors, CreateWebhookRequest, CreateWebhookResponse, CreateWebhookResponse2, CreateWebhookResponses, CrmComment, CrmCommentEntityType, CrmCommentThread, CrmCompanyFilters, CrmCompanyResponse, CrmContactResponse, CrmDomainResponse, CrmPermissionRole, CrmTeamSettingsResponse, CrmThread, CustomSpeakerAssignment, DataType, DeleteAnchorData, DeleteAnchorError, DeleteAnchorErrors, DeleteAnchorInfo, DeleteAnchorResponse, DeleteAnchorResponses, DeleteCallRecordData, DeleteCallRecordError, DeleteCallRecordErrors, DeleteCallRecordResponse, DeleteCallRecordResponses, DeleteChannelData, DeleteChannelError, DeleteChannelErrors, DeleteChannelResponse, DeleteChannelResponses, DeleteCommentData, DeleteCommentError, DeleteCommentErrors, DeleteCommentRequest, DeleteCommentResponse, DeleteCommentResponse2, DeleteCommentResponses, DeleteCrmCommentData, DeleteCrmCommentError, DeleteCrmCommentErrors, DeleteCrmCommentResponse, DeleteCrmCommentResponses, DeleteCrmCommentResult, DeleteDocumentData, DeleteDocumentError, DeleteDocumentErrors, DeleteDocumentResponse, DeleteDocumentResponses, DeleteEntityMentionData, DeleteEntityMentionError, DeleteEntityMentionErrors, DeleteEntityMentionResponse, DeleteEntityMentionResponse2, DeleteEntityMentionResponses, DeleteHistoryHandlerData, DeleteHistoryHandlerError, DeleteHistoryHandlerErrors, DeleteHistoryHandlerResponse, DeleteHistoryHandlerResponses, DeleteMessageData, DeleteMessageError, DeleteMessageErrors, DeleteMessageQuery, DeleteMessageResponse, DeleteMessageResponses, DeleteProjectHandlerData, DeleteProjectHandlerError, DeleteProjectHandlerErrors, DeleteProjectHandlerResponse, DeleteProjectHandlerResponses, DeleteThreadInfo, DeleteUnthreadedAnchorRequest, DeleteUnthreadedAnchorResponse, DeleteUnthreadedPdfAnchorRequest, DeleteUserDocumentViewLocationData, DeleteUserDocumentViewLocationError, DeleteUserDocumentViewLocationErrors, DeleteUserDocumentViewLocationResponse, DeleteUserDocumentViewLocationResponses, DeleteViewHandlerData, DeleteViewHandlerError, DeleteViewHandlerErrors, DeleteViewHandlerResponses, DeleteWebhookData, DeleteWebhookError, DeleteWebhookErrors, DeleteWebhookResponse, DeleteWebhookResponses, DocumentBasic, DocumentContent, DocumentContentLocation, DocumentContentState, DocumentContentUploadedMetadata, DocumentCopiedMetadata, DocumentCreatedMetadata, DocumentDeletedMetadata, DocumentFilters, DocumentInteractionMetadata, DocumentMetadata, DocumentMetadataWithContent, DocumentPermissionsToken, DocumentPermissionsTokenRequest, DocumentPermissionsTokenResponse, DocumentPreview, DocumentPreviewData, DocumentPreviewDataSubType, DocumentPurgedMetadata, DocumentResponse, DocumentResponseMetadata, DocumentResponseMetadataWithContent, DocumentStorageServiceApiVersion, DocumentSubType, DocumentSyncContentUpdatedMetadata, DocumentTeamShareResponse, DocumentTopicEvent, DocumentUpdatedMetadata, EditAnchorData, EditAnchorError, EditAnchorErrors, EditAnchorRequest, EditAnchorResponse, EditAnchorResponse2, EditAnchorResponses, EditCallRecordData, EditCallRecordError, EditCallRecordErrors, EditCallRecordRequest, EditCallRecordResponse, EditCallRecordResponses, EditCallTranscriptData, EditCallTranscriptError, EditCallTranscriptErrors, EditCallTranscriptRequest, EditCallTranscriptResponse, EditCallTranscriptResponses, EditCommentData, EditCommentError, EditCommentErrors, EditCommentRequest, EditCommentResponse, EditCommentResponse2, EditCommentResponses, EditCrmCommentData, EditCrmCommentError, EditCrmCommentErrors, EditCrmCommentRequest, EditCrmCommentResponse, EditCrmCommentResponses, EditDocumentData, EditDocumentError, EditDocumentErrors, EditDocumentResponse, EditDocumentResponse2, EditDocumentResponses, EditDocumentServiceArgs, EditPdfAnchorRequest, EditPdfPlaceableCommentAnchorRequest, EditProjectV2Data, EditProjectV2Error, EditProjectV2Errors, EditProjectV2Response, EditProjectV2Responses, EditThreadV2Data, EditThreadV2Error, EditThreadV2Errors, EditThreadV2Response, EditThreadV2Responses, EmailFilters, EmptyResponse, EntityFilters, EntityPermission, EntityPermissionResponse, EntityReference, EntityType, ErrorResponse, ExcludedDefaultView, ExcludeDefaultViewHandlerData, ExcludeDefaultViewHandlerError, ExcludeDefaultViewHandlerErrors, ExcludeDefaultViewHandlerResponses, ExcludeDefaultViewRequest, ExportDocumentData, ExportDocumentError, ExportDocumentErrors, ExportDocumentResponse, ExportDocumentResponse2, ExportDocumentResponses, Favorite, FavoriteEntityRef, FavoritesList, FileSystemNodeWithIds, FileType, FileTypeUpdate, FolderItem, ForeignEntity, ForeignEntityFilters, GenericErrorResponse, GenericResponse, GenericSuccessResponse, GetActivitiesResponse, GetActivityData, GetActivityError, GetActivityErrors, GetActivityResponse, GetActivityResponses, GetAttachmentReferencesData, GetAttachmentReferencesError, GetAttachmentReferencesErrors, GetAttachmentReferencesResponse, GetAttachmentReferencesResponse2, GetAttachmentReferencesResponses, GetBatchCallRecordPreviewData, GetBatchCallRecordPreviewError, GetBatchCallRecordPreviewErrors, GetBatchCallRecordPreviewRequest, GetBatchCallRecordPreviewResponse, GetBatchCallRecordPreviewResponse2, GetBatchCallRecordPreviewResponses, GetBatchChannelPreviewData, GetBatchChannelPreviewError, GetBatchChannelPreviewErrors, GetBatchChannelPreviewRequest, GetBatchChannelPreviewResponse, GetBatchChannelPreviewResponse2, GetBatchChannelPreviewResponses, GetBatchPreviewHandlerData, GetBatchPreviewHandlerError, GetBatchPreviewHandlerErrors, GetBatchPreviewHandlerResponse, GetBatchPreviewHandlerResponses, GetBatchPreviewRequest, GetBatchPreviewResponse, GetBatchProjectPreviewData, GetBatchProjectPreviewError, GetBatchProjectPreviewErrors, GetBatchProjectPreviewRequest, GetBatchProjectPreviewResponse, GetBatchProjectPreviewResponse2, GetBatchProjectPreviewResponses, GetCallRecordData, GetCallRecordError, GetCallRecordErrors, GetCallRecordResponse, GetCallRecordResponses, GetChannelAttachmentsData, GetChannelAttachmentsError, GetChannelAttachmentsErrors, GetChannelAttachmentsResponse, GetChannelAttachmentsResponses, GetChannelData, GetChannelError, GetChannelErrors, GetChannelJoinLinkData, GetChannelJoinLinkError, GetChannelJoinLinkErrors, GetChannelJoinLinkResponse, GetChannelJoinLinkResponses, GetChannelMessagesData, GetChannelMessagesError, GetChannelMessagesErrors, GetChannelMessagesResponse, GetChannelMessagesResponses, GetChannelParticipantsData, GetChannelParticipantsError, GetChannelParticipantsErrors, GetChannelParticipantsResponse, GetChannelParticipantsResponses, GetChannelResponse, GetChannelResponses, GetChannelsData, GetChannelsError, GetChannelsErrors, GetChannelsResponse, GetChannelsResponses, GetCompanyData, GetCompanyError, GetCompanyErrors, GetCompanyResponse, GetCompanyResponses, GetContactData, GetContactError, GetContactErrors, GetContactResponse, GetContactResponses, GetCrmTeamSettingsData, GetCrmTeamSettingsError, GetCrmTeamSettingsErrors, GetCrmTeamSettingsResponse, GetCrmTeamSettingsResponses, GetDocumentAnchorsData, GetDocumentAnchorsError, GetDocumentAnchorsErrors, GetDocumentAnchorsResponse, GetDocumentAnchorsResponses, GetDocumentBranchNameData, GetDocumentBranchNameError, GetDocumentBranchNameErrors, GetDocumentBranchNameResponse, GetDocumentBranchNameResponses, GetDocumentByTeamSlugData, GetDocumentByTeamSlugError, GetDocumentByTeamSlugErrors, GetDocumentByTeamSlugResponse, GetDocumentByTeamSlugResponses, GetDocumentCommentsData, GetDocumentCommentsError, GetDocumentCommentsErrors, GetDocumentCommentsResponse, GetDocumentCommentsResponses, GetDocumentData, GetDocumentError, GetDocumentErrors, GetDocumentGithubPullRequestsData, GetDocumentGithubPullRequestsError, GetDocumentGithubPullRequestsErrors, GetDocumentGithubPullRequestsResponse, GetDocumentGithubPullRequestsResponses, GetDocumentKeyResponse, GetDocumentKeyResponseData, GetDocumentListHandlerData, GetDocumentListHandlerError, GetDocumentListHandlerErrors, GetDocumentListHandlerResponse, GetDocumentListHandlerResponses, GetDocumentListResult, GetDocumentLocationV3Data, GetDocumentLocationV3Error, GetDocumentLocationV3Errors, GetDocumentLocationV3Response, GetDocumentLocationV3Responses, GetDocumentPermissionsResponseDataV2, GetDocumentPermissionsTokenData, GetDocumentPermissionsTokenError, GetDocumentPermissionsTokenErrors, GetDocumentPermissionsTokenResponse, GetDocumentPermissionsTokenResponses, GetDocumentPermissionsV2Data, GetDocumentPermissionsV2Error, GetDocumentPermissionsV2Errors, GetDocumentPermissionsV2Response, GetDocumentPermissionsV2Responses, GetDocumentProcessingResult, GetDocumentProcessingResultData, GetDocumentProcessingResultError, GetDocumentProcessingResultErrors, GetDocumentProcessingResultResponse, GetDocumentProcessingResultResponse2, GetDocumentProcessingResultResponses, GetDocumentResponse, GetDocumentResponse2, GetDocumentResponseData, GetDocumentResponses, GetDocumentSearchResponse, GetDocumentShortIdData, GetDocumentShortIdError, GetDocumentShortIdErrors, GetDocumentShortIdResponse, GetDocumentShortIdResponses, GetDocumentsResponse, GetDocumentTeamShareData, GetDocumentTeamShareError, GetDocumentTeamShareErrors, GetDocumentTeamShareResponse, GetDocumentTeamShareResponses, GetDocumentUserAccessLevelResponse, GetDocumentVersionData, GetDocumentVersionError, GetDocumentVersionErrors, GetDocumentVersionResponse, GetDocumentVersionResponses, GetDocumentViewsHandlerData, GetDocumentViewsHandlerError, GetDocumentViewsHandlerErrors, GetDocumentViewsHandlerResponse, GetDocumentViewsHandlerResponses, GetEntityPermissionData, GetEntityPermissionErrors, GetEntityPermissionResponse, GetEntityPermissionResponses, GetForeignEntityData, GetForeignEntityError, GetForeignEntityErrors, GetForeignEntityResponse, GetForeignEntityResponses, GetHistoryHandlerData, GetHistoryHandlerError, GetHistoryHandlerErrors, GetHistoryHandlerResponse, GetHistoryHandlerResponses, GetInstructionsDocumentResponse, GetInstructionsHandlerData, GetInstructionsHandlerError, GetInstructionsHandlerErrors, GetInstructionsHandlerResponse, GetInstructionsHandlerResponses, GetItemsSoupData, GetItemsSoupError, GetItemsSoupErrors, GetItemsSoupResponse, GetItemsSoupResponses, GetLocationHandlerData, GetLocationHandlerError, GetLocationHandlerErrors, GetLocationHandlerResponse, GetLocationHandlerResponses, GetMessageWithContextData, GetMessageWithContextError, GetMessageWithContextErrors, GetMessageWithContextResponse, GetMessageWithContextResponse2, GetMessageWithContextResponses, GetOrCreateAction, GetOrCreateCallData, GetOrCreateCallError, GetOrCreateCallErrors, GetOrCreateCallResponse, GetOrCreateCallResponses, GetOrCreateChannelResponse, GetOrCreateDmData, GetOrCreateDmError, GetOrCreateDmErrors, GetOrCreateDmRequest, GetOrCreateDmResponse, GetOrCreateDmResponses, GetOrCreatePrivateData, GetOrCreatePrivateError, GetOrCreatePrivateErrors, GetOrCreatePrivateRequest, GetOrCreatePrivateResponse, GetOrCreatePrivateResponses, GetPendingProjectsHandlerData, GetPendingProjectsHandlerError, GetPendingProjectsHandlerErrors, GetPendingProjectsHandlerResponse, GetPendingProjectsHandlerResponses, GetPinsHandlerData, GetPinsHandlerError, GetPinsHandlerErrors, GetPinsHandlerResponse, GetPinsHandlerResponses, GetPinsResponse, GetProjectContentHandlerData, GetProjectContentHandlerError, GetProjectContentHandlerErrors, GetProjectContentHandlerResponse, GetProjectContentHandlerResponses, GetProjectContentResponse, GetProjectHandlerData, GetProjectHandlerError, GetProjectHandlerErrors, GetProjectHandlerResponse, GetProjectHandlerResponses, GetProjectPermissionsV2Data, GetProjectPermissionsV2Error, GetProjectPermissionsV2Errors, GetProjectPermissionsV2Response, GetProjectPermissionsV2Responses, GetProjectResponse, GetProjectResponseData, GetProjectsHandlerData, GetProjectsHandlerError, GetProjectsHandlerErrors, GetProjectsHandlerResponse, GetProjectsHandlerResponses, GetProjectsResponse, GetProjectUserAccessLevelData, GetProjectUserAccessLevelError, GetProjectUserAccessLevelErrors, GetProjectUserAccessLevelResponse, GetProjectUserAccessLevelResponses, GetRecentActivityHandlerData, GetRecentActivityHandlerError, GetRecentActivityHandlerErrors, GetRecentActivityHandlerResponse, GetRecentActivityHandlerResponses, GetRingStatusData, GetRingStatusError, GetRingStatusErrors, GetRingStatusResponse, GetRingStatusResponses, GetThreadRepliesData, GetThreadRepliesError, GetThreadRepliesErrors, GetThreadRepliesResponse, GetThreadRepliesResponses, GetUserDocumentsHandlerData, GetUserDocumentsHandlerError, GetUserDocumentsHandlerErrors, GetUserDocumentsHandlerResponse, GetUserDocumentsHandlerResponses, GetUserDocumentViewLocationData, GetUserDocumentViewLocationError, GetUserDocumentViewLocationErrors, GetUserDocumentViewLocationResponse, GetUserDocumentViewLocationResponses, GetUserHistoryResponse, GetViewsHandlerData, GetViewsHandlerError, GetViewsHandlerErrors, GetViewsHandlerResponse, GetViewsHandlerResponses, GetWebhookData, GetWebhookError, GetWebhookErrors, GetWebhookResponse, GetWebhookResponses, GithubPullRequest, GithubPullRequestCheckRun, GithubPullRequestComment, GithubPullRequestsResponse, GroupedInitialParams, GroupedPageParams, GroupedSoupGroupPage, GroupedSoupInitialPage, GroupedSoupPage, GroupedSoupSort, HandlerData, HandlerError, HandlerErrors, HandlerResponse, HandlerResponses, HashMap, HealthHandlerData, HealthHandlerResponse, HealthHandlerResponses, HighlightType, IngestTranscriptData, IngestTranscriptError, IngestTranscriptErrors, IngestTranscriptResponses, InitializeUserDocumentsData, InitializeUserDocumentsError, InitializeUserDocumentsErrors, InitializeUserDocumentsResponse, InitializeUserDocumentsResponses, InstallSyncData, InteractionReason, Item, ItemWithUserAccessLevel, JobProcessingResultHandlerData, JobProcessingResultHandlerError, JobProcessingResultHandlerErrors, JobProcessingResultHandlerResponse, JobProcessingResultHandlerResponses, JoinChannelByCodeData, JoinChannelByCodeError, JoinChannelByCodeErrors, JoinChannelByCodeResponses, JoinChannelData, JoinChannelError, JoinChannelErrors, JoinChannelResponses, LatestMessage, LeaveCallResponse, LeaveChannelData, LeaveChannelError, LeaveChannelErrors, LeaveChannelResponses, LeaveOrEndCallData, LeaveOrEndCallError, LeaveOrEndCallErrors, LeaveOrEndCallResponse, LeaveOrEndCallResponses, ListBotChannelsData, ListBotChannelsError, ListBotChannelsErrors, ListBotChannelsResponse, ListBotChannelsResponses, ListCompanyContactsData, ListCompanyContactsError, ListCompanyContactsErrors, ListCompanyContactsResponse, ListCompanyContactsResponses, ListCrmCommentsData, ListCrmCommentsError, ListCrmCommentsErrors, ListCrmCommentsResponse, ListCrmCommentsResponses, ListFavoritesData, ListFavoritesError, ListFavoritesErrors, ListFavoritesResponse, ListFavoritesResponses, ListWebhooksData, ListWebhooksError, ListWebhooksErrors, ListWebhooksResponse, ListWebhooksResponse2, ListWebhooksResponses, LocationResponseData, LocationResponseV3, MacroUserIdStr, Mentions, NewChannelAttachment, NotificationFilters, Params, ParticipantRole, PatchChannelData, PatchChannelError, PatchChannelErrors, PatchChannelRequest, PatchChannelResponse, PatchChannelResponses, PatchMessageData, PatchMessageError, PatchMessageErrors, PatchMessageRequest, PatchMessageResponse, PatchMessageResponses, PatchProjectRequestV2, PatchThreadRequestV2, PatchViewHandlerData, PatchViewHandlerError, PatchViewHandlerErrors, PatchViewHandlerResponses, PatchWebhookData, PatchWebhookError, PatchWebhookErrors, PatchWebhookRequest, PatchWebhookResponse, PatchWebhookResponses, PdfAnchor, PdfAnchorId, PdfAnchorRequest, PdfHighlightAnchor, PdfHighlightAnchorRequest, PdfHighlightRect, PdfHighlightRectAnchorRequest, PdfPlaceableCommentAnchor, PdfPlaceableCommentAnchorRequest, PermanentlyDeleteDocumentData, PermanentlyDeleteDocumentError, PermanentlyDeleteDocumentErrors, PermanentlyDeleteDocumentResponse, PermanentlyDeleteDocumentResponses, PermanentlyDeleteProjectData, PermanentlyDeleteProjectError, PermanentlyDeleteProjectErrors, PermanentlyDeleteProjectResponse, PermanentlyDeleteProjectResponses, PinnedItem, PinRequest, PostActivityData, PostActivityError, PostActivityErrors, PostActivityRequest, PostActivityResponse, PostActivityResponses, PostChannelBotWebhookData, PostChannelBotWebhookError, PostChannelBotWebhookErrors, PostChannelBotWebhookResponse, PostChannelBotWebhookResponses, PostChannelMessagesData, PostChannelMessagesError, PostChannelMessagesErrors, PostChannelMessagesResponse, PostChannelMessagesResponses, PostGroupedSoupAstGroupPageRequest, PostGroupedSoupAstInitialRequest, PostGroupedSoupAstRequest, PostItemsSoupAstData, PostItemsSoupAstError, PostItemsSoupAstErrors, PostItemsSoupAstGroupedData, PostItemsSoupAstGroupedError, PostItemsSoupAstGroupedErrors, PostItemsSoupAstGroupedResponse, PostItemsSoupAstGroupedResponses, PostItemsSoupAstResponse, PostItemsSoupAstResponses, PostItemsSoupData, PostItemsSoupError, PostItemsSoupErrors, PostItemsSoupResponse, PostItemsSoupResponses, PostMessageData, PostMessageError, PostMessageErrors, PostMessageRequest, PostMessageResponse, PostMessageResponse2, PostMessageResponses, PostReactionData, PostReactionError, PostReactionErrors, PostReactionRequest, PostReactionResponse, PostReactionResponses, PostSoupAstRequest, PostSoupRequest, PostTypingData, PostTypingError, PostTypingErrors, PostTypingRequest, PostTypingResponse, PostTypingResponses, PresaveDocumentHandlerData, PresaveDocumentHandlerError, PresaveDocumentHandlerErrors, PresaveDocumentHandlerResponse, PresaveDocumentHandlerResponses, PreSaveDocumentRequest, PreSaveDocumentResponse, PreSaveDocumentResponseData, PresignedUrl, Project, ProjectDeleteResponseData, ProjectDocumentStatus, ProjectFilters, ProjectPreview, ProjectPreviewData, PropertyDefinition, PropertyFilter, PropertyInput, PropertyOwner, PropertyValue, PutCrmTeamSettingsData, PutCrmTeamSettingsError, PutCrmTeamSettingsErrors, PutCrmTeamSettingsResponse, PutCrmTeamSettingsResponses, ReactionAction, RecentlyDeletedData, RecentlyDeletedError, RecentlyDeletedErrors, RecentlyDeletedResponse, RecentlyDeletedResponseData, RecentlyDeletedResponses, RemoveBotFromChannelByBotData, RemoveBotFromChannelByBotError, RemoveBotFromChannelByBotErrors, RemoveBotFromChannelByBotResponse, RemoveBotFromChannelByBotResponses, RemoveFavoriteByEntityData, RemoveFavoriteByEntityError, RemoveFavoriteByEntityErrors, RemoveFavoriteByEntityResponses, RemoveParticipantsData, RemoveParticipantsError, RemoveParticipantsErrors, RemoveParticipantsRequest, RemoveParticipantsResponses, RemovePinHandlerData, RemovePinHandlerError, RemovePinHandlerErrors, RemovePinHandlerResponse, RemovePinHandlerResponses, ReorderFavoritesData, ReorderFavoritesError, ReorderFavoritesErrors, ReorderFavoritesRequest, ReorderFavoritesResponses, ReorderPinRequest, ReorderPinsHandlerData, ReorderPinsHandlerError, ReorderPinsHandlerErrors, ReorderPinsHandlerResponse, ReorderPinsHandlerResponses, ResolveChannelMessageData, ResolveChannelMessageError, ResolveChannelMessageErrors, ResolveChannelMessageResponse, ResolveChannelMessageResponses, RevertDeleteDocumentData, RevertDeleteDocumentError, RevertDeleteDocumentErrors, RevertDeleteDocumentResponse, RevertDeleteDocumentResponses, RevertDeleteProjectData, RevertDeleteProjectError, RevertDeleteProjectErrors, RevertDeleteProjectResponse, RevertDeleteProjectResponses, RingStatus, RingStatusResponse, S3ObjectInfo, SaveBomPart, SaveDocumentHandlerData, SaveDocumentHandlerError, SaveDocumentHandlerErrors, SaveDocumentHandlerResponse, SaveDocumentHandlerResponses, SaveDocumentRequest, SaveDocumentResponse, SaveDocumentResponseData, SetCompanyHiddenData, SetCompanyHiddenError, SetCompanyHiddenErrors, SetCompanyHiddenRequest, SetCompanyHiddenResponse, SetCompanyHiddenResponses, SetCompanyNameRequest, SetContactHiddenData, SetContactHiddenError, SetContactHiddenErrors, SetContactHiddenRequest, SetContactHiddenResponse, SetContactHiddenResponses, SetContactNameRequest, SetCrmCompanyNameData, SetCrmCompanyNameError, SetCrmCompanyNameErrors, SetCrmCompanyNameResponse, SetCrmCompanyNameResponses, SetCrmContactNameData, SetCrmContactNameError, SetCrmContactNameErrors, SetCrmContactNameResponse, SetCrmContactNameResponses, SetDocumentTeamShareData, SetDocumentTeamShareError, SetDocumentTeamShareErrors, SetDocumentTeamShareRequest, SetDocumentTeamShareResponse, SetDocumentTeamShareResponses, SetEmailSyncData, SetEmailSyncError, SetEmailSyncErrors, SetEmailSyncRequest, SetEmailSyncResponse, SetEmailSyncResponses, SetPropertyValue, SharedEmailFilter, SharePermissionV2, ShortIdResponse, SimpleMention, SimpleSaveData, SimpleSaveError, SimpleSaveErrors, SimpleSaveResponse, SimpleSaveResponses, SoupApiItem, SoupApiSort, SoupAttachment, SoupCallRecordParticipant, SoupCallRecordSoupPropertiesField, SoupChannel, SoupChannelThread, SoupChatSoupPropertiesField, SoupContact, SoupCountedReaction, SoupCrmCompanySoupPropertiesField, SoupCrmDomain, SoupDocumentSoupPropertiesField, SoupDocumentSubType, SoupEmailThreadPreview, SoupEnrichedEmailThreadPreviewSoupPropertiesField, SoupForeignEntity, SoupItem, SoupLabel, SoupLabelListVisibility, SoupLabelType, SoupMessageAttachment, SoupMessageListVisibility, SoupMessageSender, SoupMessageSenderType, SoupPage, SoupProjectSoupPropertiesField, SoupPropertiesField, SoupProperty, SoupThreadInfo, SoupThreadReply, StarterDocumentsResponse, String, SuccessResponse, SyncServiceVersionId, TagFilterMode, TaskFilters, TeamRole, Thread, ThreadResponse, ToggleShareWithTeamData, ToggleShareWithTeamError, ToggleShareWithTeamErrors, ToggleShareWithTeamResponse, ToggleShareWithTeamResponses, TranscriptSegmentRequest, TypedSuccessResponse, TypedSuccessResponseGetDocumentResponseData, TypingAction, UnthreadedPdfUuidRequest, UpdateChannelSharePermission, UpdateCrmTeamSettingsRequest, UpdateOperation, UpdateSharePermissionRequestV2, UploadDocumentStatus, UploadExtractFolderHandlerData, UploadExtractFolderHandlerError, UploadExtractFolderHandlerErrors, UploadExtractFolderHandlerResponse, UploadExtractFolderHandlerResponses, UploadExtractFolderRequest, UploadFolderHandlerData, UploadFolderHandlerError, UploadFolderHandlerErrors, UploadFolderHandlerResponse, UploadFolderHandlerResponses, UploadFolderRequest, UpsertHistoryHandlerData, UpsertHistoryHandlerError, UpsertHistoryHandlerErrors, UpsertHistoryHandlerResponse, UpsertHistoryHandlerResponses, UpsertUserDocumentViewLocationData, UpsertUserDocumentViewLocationError, UpsertUserDocumentViewLocationErrors, UpsertUserDocumentViewLocationRequest, UpsertUserDocumentViewLocationResponse, UpsertUserDocumentViewLocationResponses, UserActivitiesResponse, UserDocumentsResponse, UserDocumentViewLocation, UserDocumentViewLocationResponse, UserPinsResponse, UserViewsResponse, ValidateDocumentPermissionsTokenData, ValidateDocumentPermissionsTokenError, ValidateDocumentPermissionsTokenErrors, ValidateDocumentPermissionsTokenResponse, ValidateDocumentPermissionsTokenResponses, ValidateWebhookData, ValidateWebhookError, ValidateWebhookErrors, ValidateWebhookResponse, ValidateWebhookResponse2, ValidateWebhookResponses, Vec, View, ViewPatch, ViewsResponse, Webhook, WebhookEvent, WebhookFilter, WebhookScope, WebhookStatus, WebhookValidationTestEvent, WithCallId, WithChannelId, WithDocumentId, WithProjectId } from './types.gen';diff --git a/packages/sdk/generated/storage/types.gen.ts b/packages/sdk/generated/storage/types.gen.tsindex b2e551dfab..005cf9663b 100644--- a/packages/sdk/generated/storage/types.gen.ts+++ b/packages/sdk/generated/storage/types.gen.ts@@ -3186,6 +3186,30 @@ export type DocumentContentLocation = 'object_storage' | 'sync_service' | 'docx_*/export type DocumentContentState = 'unknown' | 'pending' | 'ready';+/**+ * Metadata for [`DocumentTopicEvent::ContentUploaded`].+ */+export type DocumentContentUploadedMetadata = {+ /**+ * The id of the document whose stored bytes changed.+ */+ document_id: string;+ /**+ * Version written, or the converted-file marker; `None` for unversioned+ * writes (mirrors `SearchExtractorMessage::document_version_id`).+ */+ document_version_id?: string | null;+ /**+ * File type of the uploaded object (may differ from the document's own+ * type, e.g. `pdf` for the converted rendition of a docx).+ */+ file_type: FileType;+ /**+ * The owner of the document (used by the extractor to resolve S3 keys).+ */+ owner: MacroUserIdStr;+};+/*** Metadata for [`DocumentTopicEvent::Copied`].*/@@ -3491,6 +3515,16 @@ export type DocumentPreviewDataSubType = {type: 'snippet';};+/**+ * Metadata for [`DocumentTopicEvent::Purged`].+ */+export type DocumentPurgedMetadata = {+ /**+ * The id of the hard-deleted document.+ */+ document_id: string;+};+/*** Document response with content lifecycle metadata.*/@@ -3593,6 +3627,24 @@ export type DocumentStorageServiceApiVersion = 'v1' | 'v2';*/export type DocumentSubType = 'task' | 'snippet';+/**+ * Metadata for [`DocumentTopicEvent::SyncContentUpdated`].+ */+export type DocumentSyncContentUpdatedMetadata = {+ /**+ * The id of the live-collab document whose content changed.+ */+ document_id: string;+ /**+ * Version marker for the sync snapshot, when the caller supplies one.+ */+ document_version_id?: string | null;+ /**+ * File type of the sync document (markdown today).+ */+ file_type: FileType;+};+/*** The team-share state of a document. The team is resolved from the document* owner's team membership.@@ -3630,6 +3682,24 @@ export type DocumentTopicEvent = {* A document was soft-deleted.*/metadata: DocumentDeletedMetadata;+} | {+ event_type: 'document.content_uploaded';+ /**+ * The document's stored bytes were (re)written to S3.+ */+ metadata: DocumentContentUploadedMetadata;+} | {+ event_type: 'document.sync_content_updated';+ /**+ * A live-collab (sync) document's content changed and should be re-extracted.+ */+ metadata: DocumentSyncContentUpdatedMetadata;+} | {+ event_type: 'document.purged';+ /**+ * The document row was permanently deleted (hard delete).+ */+ metadata: DocumentPurgedMetadata;} | {event_type: 'document.copied';/**diff --git a/packages/sdk/specs/storage.json b/packages/sdk/specs/storage.jsonindex a44f839c81..5858e9f80f 100644--- a/packages/sdk/specs/storage.json+++ b/packages/sdk/specs/storage.json@@ -15046,6 +15046,29 @@"description": "API-visible content lifecycle state derived from current document metadata.","enum": ["unknown", "pending", "ready"]},+ "DocumentContentUploadedMetadata": {+ "type": "object",+ "description": "Metadata for [`DocumentTopicEvent::ContentUploaded`].",+ "required": ["document_id", "owner", "file_type"],+ "properties": {+ "document_id": {+ "type": "string",+ "description": "The id of the document whose stored bytes changed."+ },+ "document_version_id": {+ "type": ["string", "null"],+ "description": "Version written, or the converted-file marker; `None` for unversioned\nwrites (mirrors `SearchExtractorMessage::document_version_id`)."+ },+ "file_type": {+ "$ref": "#/components/schemas/FileType",+ "description": "File type of the uploaded object (may differ from the document's own\ntype, e.g. `pdf` for the converted rendition of a docx)."+ },+ "owner": {+ "$ref": "#/components/schemas/MacroUserIdStr",+ "description": "The owner of the document (used by the extractor to resolve S3 keys)."+ }+ }+ },"DocumentCopiedMetadata": {"type": "object","description": "Metadata for [`DocumentTopicEvent::Copied`].",@@ -15554,6 +15577,17 @@],"description": "The sub type of a document preview with associated properties.\nTask-related properties are encoded within the variant to ensure valid states."},+ "DocumentPurgedMetadata": {+ "type": "object",+ "description": "Metadata for [`DocumentTopicEvent::Purged`].",+ "required": ["document_id"],+ "properties": {+ "document_id": {+ "type": "string",+ "description": "The id of the hard-deleted document."+ }+ }+ },"DocumentResponse": {"type": "object","description": "Document response with content lifecycle metadata.",@@ -15688,6 +15722,25 @@"description": "The document sub type enum represents all values of document sub types.\nThese values should match the `document_sub_type_value` table in macrodb.","enum": ["task", "snippet"]},+ "DocumentSyncContentUpdatedMetadata": {+ "type": "object",+ "description": "Metadata for [`DocumentTopicEvent::SyncContentUpdated`].",+ "required": ["document_id", "file_type"],+ "properties": {+ "document_id": {+ "type": "string",+ "description": "The id of the live-collab document whose content changed."+ },+ "document_version_id": {+ "type": ["string", "null"],+ "description": "Version marker for the sync snapshot, when the caller supplies one."+ },+ "file_type": {+ "$ref": "#/components/schemas/FileType",+ "description": "File type of the sync document (markdown today)."+ }+ }+ },"DocumentTeamShareResponse": {"type": "object","description": "The team-share state of a document. The team is resolved from the document\nowner's team membership.",@@ -15751,6 +15804,51 @@}}},+ {+ "type": "object",+ "description": "The document's stored bytes were (re)written to S3.",+ "required": ["metadata", "event_type"],+ "properties": {+ "event_type": {+ "type": "string",+ "enum": ["document.content_uploaded"]+ },+ "metadata": {+ "$ref": "#/components/schemas/DocumentContentUploadedMetadata",+ "description": "The document's stored bytes were (re)written to S3."+ }+ }+ },+ {+ "type": "object",+ "description": "A live-collab (sync) document's content changed and should be re-extracted.",+ "required": ["metadata", "event_type"],+ "properties": {+ "event_type": {+ "type": "string",+ "enum": ["document.sync_content_updated"]+ },+ "metadata": {+ "$ref": "#/components/schemas/DocumentSyncContentUpdatedMetadata",+ "description": "A live-collab (sync) document's content changed and should be re-extracted."+ }+ }+ },+ {+ "type": "object",+ "description": "The document row was permanently deleted (hard delete).",+ "required": ["metadata", "event_type"],+ "properties": {+ "event_type": {+ "type": "string",+ "enum": ["document.purged"]+ },+ "metadata": {+ "$ref": "#/components/schemas/DocumentPurgedMetadata",+ "description": "The document row was permanently deleted (hard delete)."+ }+ }+ },{"type": "object","description": "A document was copied.",diff --git a/services/deleted_item_poller/Cargo.toml b/services/deleted_item_poller/Cargo.tomlindex 1280e61cb1..9830e145f2 100644--- a/services/deleted_item_poller/Cargo.toml+++ b/services/deleted_item_poller/Cargo.toml@@ -12,6 +12,7 @@ aws-smithy-types = { workspace = true }aws_lambda_events = { workspace = true, features = ["eventbridge"] }chat = { path = "../../crates/chat", default-features = false }chrono = { workspace = true }+documents = { path = "../../crates/documents", default-features = false }futures = { workspace = true }lambda_runtime = { workspace = true }macro_db_client = { path = "../../crates/macro_db_client" }@@ -30,7 +31,6 @@ sqlx = { workspace = true }sqs_client = { path = "../../crates/sqs_client", default-features = false, features = ["chat","document",- "search",] }tokio = { workspace = true }tracing = { workspace = true }diff --git a/services/deleted_item_poller/src/config.rs b/services/deleted_item_poller/src/config.rsindex ee3db73e7b..0e0341b183 100644--- a/services/deleted_item_poller/src/config.rs+++ b/services/deleted_item_poller/src/config.rs@@ -13,7 +13,7 @@ pub struct Config {/// The connection URL for the Postgres database this application should use.pub database_url: String,- /// Comma-separated Kafka bootstrap servers for project and chat lifecycle events.+ /// Comma-separated Kafka bootstrap servers for document, project, and chat lifecycle events.pub kafka_brokers: KafkaBrokers,/// The environment we are indiff --git a/services/deleted_item_poller/src/handler.rs b/services/deleted_item_poller/src/handler.rsindex 5fe629c4cf..4c83b94f89 100644--- a/services/deleted_item_poller/src/handler.rs+++ b/services/deleted_item_poller/src/handler.rs@@ -5,6 +5,7 @@ use crate::context::{self};use anyhow::Context;use aws_lambda_events::eventbridge::EventBridgeEvent;use chat::domain::events::{ChatMacroEvent, ChatPermanentlyDeletedMetadata};+use documents::domain::events::{DocumentMacroEvent, DocumentPurgedMetadata};use futures::future::join_all;use lambda_runtime::{Error, LambdaEvent,@@ -14,7 +15,6 @@ use macro_db_client::projects::ProjectToDelete;use macro_event_broker::MacroEventBroker;use macro_user_id::user_id::MacroUserIdStr;use projects::domain::events::{ProjectMacroEvent, ProjectPermanentlyDeletedMetadata};-use sqs_client::search::{SearchQueueMessage, document::DocumentId};#[tracing::instrument(skip(ctx, _event), err)]pub async fn handler(@@ -30,6 +30,43 @@ pub async fn handler(Ok(())}+#[tracing::instrument(skip(event_broker, document_ids), err)]+async fn publish_document_purge_events<B: MacroEventBroker>(+ event_broker: &B,+ document_ids: &[String],+) -> anyhow::Result<()> {+ let events = document_ids+ .iter()+ .map(|document_id| {+ DocumentMacroEvent::purged(+ document_id.clone(),+ DocumentPurgedMetadata {+ document_id: document_id.clone(),+ },+ )+ })+ .collect::<Vec<_>>();++ let publications = events+ .iter()+ .map(|event| event_broker.send_event(event))+ .collect::<Vec<_>>();+ let publication_results = join_all(publications.into_iter().map(|publication| async move {+ let handle = publication.context("failed to enqueue document purge event")?;+ handle+ .await+ .context("document purge event publication task failed")?+ .context("failed to publish document purge event")+ }))+ .await;++ for result in publication_results {+ result?;+ }++ Ok(())+}+#[tracing::instrument(skip(event_broker, projects_to_delete), err)]async fn publish_project_purge_events<B: MacroEventBroker>(event_broker: &B,@@ -184,18 +221,9 @@ async fn handle_documents(ctx: &context::Context) -> anyhow::Result<()> {tracing::debug!(documents_to_delete=?documents_to_delete, "documents to delete");- ctx.sqs_client- .bulk_send_message_to_search_event_queue(- documents_to_delete- .iter()- .map(|id| {- SearchQueueMessage::RemoveDocument(DocumentId {- document_id: id.to_string(),- })- })- .collect(),- )- .await?;+ publish_document_purge_events(&ctx.macro_event_broker, &documents_to_delete)+ .await+ .context("unable to publish document purge events")?;ctx.sqs_client.bulk_enqueue_document_delete(documents_to_delete)diff --git a/services/deleted_item_poller/src/handler/test.rs b/services/deleted_item_poller/src/handler/test.rsindex 10f77529c0..55791e2c81 100644--- a/services/deleted_item_poller/src/handler/test.rs+++ b/services/deleted_item_poller/src/handler/test.rs@@ -93,10 +93,11 @@ impl MacroEventBroker for FakeEventBroker {handle}DeliveryBehavior::Wait(delivery_gate) => tokio::spawn(async move {- let _permit = delivery_gate+ let permit = delivery_gate.acquire_owned().await.expect("delivery gate should remain open");+ permit.forget();Ok(())}),};@@ -105,6 +106,95 @@ impl MacroEventBroker for FakeEventBroker {}}+#[tokio::test]+async fn publish_document_purge_events_publishes_separately_keyed_events() {+ let event_broker = FakeEventBroker::default();+ let document_ids = vec!["document-one".to_string(), "document-two".to_string()];++ publish_document_purge_events(&event_broker, &document_ids)+ .await+ .unwrap();++ let published = event_broker.published();+ assert_eq!(published.len(), document_ids.len());++ for (event, document_id) in published.iter().zip(&document_ids) {+ assert_eq!(event.topic, "macro.documents");+ assert_eq!(event.key, *document_id);+ assert_eq!(event.envelope["schema_version"], json!(1));+ assert_eq!(event.envelope["event_type"], json!("document.purged"));+ assert_eq!(+ event.envelope["metadata"]["document_id"],+ json!(document_id)+ );+ }+}++#[tokio::test]+async fn publish_document_purge_events_returns_immediate_send_failures() {+ let event_broker = FakeEventBroker::failing_send();+ let document_ids = vec!["document-one".to_string()];++ let error = publish_document_purge_events(&event_broker, &document_ids)+ .await+ .expect_err("immediate send failure should be returned");++ assert!(format!("{error:#}").contains("event enqueue rejected"));+}++#[tokio::test]+async fn publish_document_purge_events_returns_delivery_failures() {+ let event_broker = FakeEventBroker::failing_delivery();+ let document_ids = vec!["document-one".to_string()];++ let error = publish_document_purge_events(&event_broker, &document_ids)+ .await+ .expect_err("delivery failure should be returned");++ assert!(format!("{error:#}").contains("publisher unavailable"));+}++#[tokio::test]+async fn publish_document_purge_events_returns_delivery_join_failures() {+ let event_broker = FakeEventBroker::failing_join();+ let document_ids = vec!["document-one".to_string()];++ let error = publish_document_purge_events(&event_broker, &document_ids)+ .await+ .expect_err("delivery join failure should be returned");++ assert!(+ format!("{error:#}").contains("document purge event publication task failed"),+ "unexpected error: {error:#}"+ );+}++#[tokio::test]+async fn publish_document_purge_events_waits_for_every_delivery() {+ let delivery_gate = Arc::new(Semaphore::new(0));+ let event_broker = FakeEventBroker::waiting_for_delivery(delivery_gate.clone());+ let document_ids = vec!["document-one".to_string(), "document-two".to_string()];++ let publication_task = tokio::spawn({+ let event_broker = event_broker.clone();+ let document_ids = document_ids.clone();+ async move { publish_document_purge_events(&event_broker, &document_ids).await }+ });++ while event_broker.published().len() < document_ids.len() {+ tokio::task::yield_now().await;+ }++ assert!(!publication_task.is_finished());++ delivery_gate.add_permits(1);+ tokio::task::yield_now().await;+ assert!(!publication_task.is_finished());++ delivery_gate.add_permits(1);+ publication_task.await.unwrap().unwrap();+}+#[tokio::test]async fn publish_project_purge_events_publishes_separately_keyed_events() {let event_broker = FakeEventBroker::default();diff --git a/services/deleted_item_poller/src/main.rs b/services/deleted_item_poller/src/main.rsindex e4022b7175..c8aa575d6a 100644--- a/services/deleted_item_poller/src/main.rs+++ b/services/deleted_item_poller/src/main.rs@@ -41,13 +41,11 @@ async fn main() -> Result<(), Error> {let document_delete_queue = macro_queues::DocumentDeleteQueue::new();let chat_delete_queue = macro_queues::ChatDeleteQueue::new();- let search_event_queue = macro_queues::SearchEventQueue::new();let sqs_client = sqs_client::SQS::new(aws_sdk_sqs::Client::new(¯o_aws_config::get_macro_aws_config().await,)).document_delete_queue(&document_delete_queue)- .chat_delete_queue(&chat_delete_queue)- .search_event_queue(&search_event_queue);+ .chat_delete_queue(&chat_delete_queue);let ctx = context::Context {db,diff --git a/services/document_storage_service/src/api/context.rs b/services/document_storage_service/src/api/context.rsindex 9f2ab31b72..c77c69d331 100644--- a/services/document_storage_service/src/api/context.rs+++ b/services/document_storage_service/src/api/context.rs@@ -286,7 +286,7 @@ pub(crate) type ProjectService = ProjectServiceImpl<DynamoBulkUploadAdapter,ShaCountAdapter,EntityAccessManagementService,- SqsProjectSearchIndexer,+ SqsProjectSearchIndexer<DssEventBroker>,DssEventBroker,>;@@ -391,7 +391,7 @@ pub(crate) type DssEntityMutationService =ProjectService,EntityAccessService,FavoritesServiceType,- crate::outbound::entity_mutation::DssEntityLifecycleAdapter,+ crate::outbound::entity_mutation::DssEntityLifecycleAdapter<DssEventBroker>,>;/// Type alias for the favorites service.diff --git a/services/document_storage_service/src/api/documents/delete_document.rs b/services/document_storage_service/src/api/documents/delete_document.rsindex f61c14b713..71764120f0 100644--- a/services/document_storage_service/src/api/documents/delete_document.rs+++ b/services/document_storage_service/src/api/documents/delete_document.rs@@ -1,6 +1,7 @@use crate::api::context::ApiContext;use crate::api::context::{AuthorizationService, EntityAccessService};use crate::api::util::count_occurrences;+use crate::service::document_event_publisher::publish_document_purged_event;use axum::Json;use axum::extract::State;use axum::response::Response;@@ -15,7 +16,6 @@ use model::response::{};use models_permissions::share_permission::access_level::OwnerAccessLevel;use serde::Deserialize;-use sqs_client::search::{SearchQueueMessage, document::DocumentId};#[derive(Deserialize)]pub struct Params {@@ -130,22 +130,16 @@ pub async fn permanently_delete_document_handler(.into_response()})?;- state- .sqs_client- .send_message_to_search_event_queue(SearchQueueMessage::RemoveDocument(DocumentId {- document_id,- }))- .await- .map_err(|e| {- tracing::error!(error=?e, "unable to send message to search extractor queue");- (- StatusCode::INTERNAL_SERVER_ERROR,- Json(ErrorResponse {- message: "unable to send message to search extractor queue".into(),- }),- )- .into_response()- })?;+ publish_document_purged_event(&state.macro_event_broker, &document_id).map_err(|e| {+ tracing::error!(error=?e, "unable to publish document purged event");+ (+ StatusCode::INTERNAL_SERVER_ERROR,+ Json(ErrorResponse {+ message: "unable to publish document purged event".into(),+ }),+ )+ .into_response()+ })?;let response_data = GenericSuccessResponse { success: true };diff --git a/services/document_storage_service/src/api/internal/mod.rs b/services/document_storage_service/src/api/internal/mod.rsindex 3d3ba747d3..f1d3bd8ba6 100644--- a/services/document_storage_service/src/api/internal/mod.rs+++ b/services/document_storage_service/src/api/internal/mod.rs@@ -58,6 +58,16 @@ pub fn router(state: ApiContext) -> Router<ApiContext> {get(get_document::get_document_basic_handler).layer(ensure_document_exists_middleware.clone()),)+ .route(+ "/documents/{document_id}/content-uploaded",+ post(+ documents_hex::inbound::axum_router::content_uploaded::content_uploaded_handler::<+ DocumentService,+ EntityAccessService,+ AuthorizationService,+ >,+ ),+ ).route("/documents/{document_id}/export",get(export_document::handler),diff --git a/services/document_storage_service/src/main.rs b/services/document_storage_service/src/main.rsindex 13cdbf1773..ebb12c4026 100644--- a/services/document_storage_service/src/main.rs+++ b/services/document_storage_service/src/main.rs@@ -470,7 +470,7 @@ async fn main() -> anyhow::Result<()> {DynamoBulkUploadAdapter::new(dynamodb_client.clone()),ShaCountAdapter::new(Redis::new(redis_client.clone())),entity_access_management_service.clone(),- SqsProjectSearchIndexer::new(Arc::new(sqs_client.clone())),+ SqsProjectSearchIndexer::new(Arc::new(sqs_client.clone()), macro_event_broker.clone()),if cfg!(feature = "local") {Some(uuid::uuid!("d50676e2-0a12-4c62-bc07-4b1cb6d8e9bc"))} else {@@ -479,28 +479,21 @@ async fn main() -> anyhow::Result<()> {macro_event_broker.clone(),));- let document_service = Arc::new(- DocumentServiceImpl::new(- document_repo,- cloudfront_config,- sync_service_client.as_ref().clone(),- s3_upload_adapter,- TaskPropertiesAdapter {- system_properties: system_properties_service.clone(),- properties: properties_service.clone(),- entity_access_service: entity_access_service.clone(),- },- connection_service,- entity_access_management_service.clone(),- ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),- macro_event_broker.clone(),- )- .with_search_indexer(Arc::new(- crate::service::document_search_indexer::SqsDocumentSearchIndexer::new(- sqs_client.clone(),- ),- )),- );+ let document_service = Arc::new(DocumentServiceImpl::new(+ document_repo,+ cloudfront_config,+ sync_service_client.as_ref().clone(),+ s3_upload_adapter,+ TaskPropertiesAdapter {+ system_properties: system_properties_service.clone(),+ properties: properties_service.clone(),+ entity_access_service: entity_access_service.clone(),+ },+ connection_service,+ entity_access_management_service.clone(),+ ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),+ macro_event_broker.clone(),+ ));let foreign_entity_service = Arc::new(ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone(),@@ -1010,6 +1003,7 @@ async fn main() -> anyhow::Result<()> {db.clone(),redis_sha_client.clone(),sqs_client.clone(),+ macro_event_broker.clone(),)),));diff --git a/services/document_storage_service/src/outbound/entity_mutation.rs b/services/document_storage_service/src/outbound/entity_mutation.rsindex 365cf905b4..0c6d2feac4 100644--- a/services/document_storage_service/src/outbound/entity_mutation.rs+++ b/services/document_storage_service/src/outbound/entity_mutation.rs@@ -5,14 +5,17 @@ use std::sync::Arc;use crate::{api::util::count_occurrences,- service::entity_mutation::{EntityLifecycleService, LifecycleError},+ service::{+ document_event_publisher::publish_document_purged_event,+ entity_mutation::{EntityLifecycleService, LifecycleError},+ },};use entity_mutation::EntityMutationActor;+use macro_event_broker::MacroEventBroker;use macro_sha_count_client::Redis;use model_entity::{Entity, EntityType};use models_permissions::share_permission::UpdateSharePermissionRequestV2;use sqlx::PgPool;-use sqs_client::search::{SearchQueueMessage, document::DocumentId};/// Wrap a legacy client failure as an internal lifecycle error.macro_rules! internal {@@ -30,20 +33,26 @@ fn row_error(error: sqlx::Error) -> LifecycleError {}/// Production lifecycle adapter backed by the legacy persistence clients.-pub struct DssEntityLifecycleAdapter {+pub struct DssEntityLifecycleAdapter<B: MacroEventBroker> {db: PgPool,redis: Arc<Redis>,sqs: Arc<sqs_client::SQS>,+ event_broker: B,}-impl DssEntityLifecycleAdapter {+impl<B: MacroEventBroker> DssEntityLifecycleAdapter<B> {/// Construct the adapter from concrete outbound dependencies.- pub fn new(db: PgPool, redis: Arc<Redis>, sqs: Arc<sqs_client::SQS>) -> Self {- Self { db, redis, sqs }+ pub fn new(db: PgPool, redis: Arc<Redis>, sqs: Arc<sqs_client::SQS>, event_broker: B) -> Self {+ Self {+ db,+ redis,+ sqs,+ event_broker,+ }}}-impl EntityLifecycleService for DssEntityLifecycleAdapter {+impl<B: MacroEventBroker> EntityLifecycleService for DssEntityLifecycleAdapter<B> {async fn update_thread_share_policy(&self,_actor: &EntityMutationActor,@@ -133,11 +142,7 @@ impl EntityLifecycleService for DssEntityLifecycleAdapter {.enqueue_document_delete(document.owner.as_ref(), &entity.entity_id).await.map_err(|error| internal!(error))?;- self.sqs- .send_message_to_search_event_queue(SearchQueueMessage::RemoveDocument(DocumentId {- document_id: entity.entity_id.to_string(),- }))- .await+ publish_document_purged_event(&self.event_broker, &entity.entity_id).map_err(|error| internal!(error))?;Ok(document.project_iddiff --git a/services/document_storage_service/src/service/document_event_publisher.rs b/services/document_storage_service/src/service/document_event_publisher.rsnew file mode 100644index 0000000000..acbf87687e--- /dev/null+++ b/services/document_storage_service/src/service/document_event_publisher.rs@@ -0,0 +1,19 @@+#[cfg(test)]+mod test;++use documents_hex::domain::events::{DocumentMacroEvent, DocumentPurgedMetadata};+use macro_event_broker::{EventBrokerError, MacroEventBroker};++/// Schedules a document-purged event for asynchronous broker delivery.+#[tracing::instrument(skip(event_broker), err)]+pub(crate) fn publish_document_purged_event<B: MacroEventBroker>(+ event_broker: &B,+ document_id: &str,+) -> Result<(), EventBrokerError> {+ let document_id = document_id.to_owned();+ let event =+ DocumentMacroEvent::purged(document_id.clone(), DocumentPurgedMetadata { document_id });++ drop(event_broker.send_event(&event)?);+ Ok(())+}diff --git a/services/document_storage_service/src/service/document_event_publisher/test.rs b/services/document_storage_service/src/service/document_event_publisher/test.rsnew file mode 100644index 0000000000..cb83de8086--- /dev/null+++ b/services/document_storage_service/src/service/document_event_publisher/test.rs@@ -0,0 +1,79 @@+use std::sync::Mutex;++use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker};+use serde_json::{Value, json};+use uuid::Uuid;++use super::*;++#[derive(Debug, PartialEq)]+struct PublishedEvent {+ topic: String,+ key: String,+ envelope: Value,+}++#[derive(Default)]+struct RecordingEventBroker {+ published: Mutex<Vec<PublishedEvent>>,+}++impl MacroEventBroker for RecordingEventBroker {+ fn send_event<E: MacroEvent + ?Sized>(+ &self,+ event: &E,+ ) -> Result<tokio::task::JoinHandle<Result<(), EventBrokerError>>, EventBrokerError> {+ self.published.lock().unwrap().push(PublishedEvent {+ topic: event.topic().to_owned(),+ key: event.key().to_owned(),+ envelope: serde_json::to_value(event.event())?,+ });++ Ok(tokio::spawn(async { Ok(()) }))+ }+}++struct FailingEventBroker;++impl MacroEventBroker for FailingEventBroker {+ fn send_event<E: MacroEvent + ?Sized>(+ &self,+ _event: &E,+ ) -> Result<tokio::task::JoinHandle<Result<(), EventBrokerError>>, EventBrokerError> {+ Err(EventBrokerError::Publish(+ "event enqueue rejected".to_string(),+ ))+ }+}++#[tokio::test]+async fn publishes_document_purged_event_to_documents_topic() {+ let event_broker = RecordingEventBroker::default();++ publish_document_purged_event(&event_broker, "document-one").unwrap();++ let published = event_broker.published.lock().unwrap();+ assert_eq!(published.len(), 1);++ let event = &published[0];+ assert_eq!(event.topic, "macro.documents");+ assert_eq!(event.key, "document-one");+ assert!(Uuid::parse_str(event.envelope["event_id"].as_str().unwrap()).is_ok());+ assert_eq!(event.envelope["schema_version"], json!(1));+ assert_eq!(event.envelope["event_type"], json!("document.purged"));+ assert_eq!(+ event.envelope["metadata"],+ json!({ "document_id": "document-one" })+ );+}++#[test]+fn returns_immediate_broker_failures() {+ let error = publish_document_purged_event(&FailingEventBroker, "document-one")+ .expect_err("immediate broker failure should be returned");++ assert!(matches!(+ error,+ EventBrokerError::Publish(message) if message == "event enqueue rejected"+ ));+}diff --git a/services/document_storage_service/src/service/document_search_indexer.rs b/services/document_storage_service/src/service/document_search_indexer.rsdeleted file mode 100644index 34a07193c1..0000000000--- a/services/document_storage_service/src/service/document_search_indexer.rs+++ /dev/null@@ -1,36 +0,0 @@-//! Adapter that publishes document name refreshes to the search queue.--use documents_hex::domain::ports::DocumentSearchIndexer;-use sqs_client::SQS;-use sqs_client::search::SearchQueueMessage;-use sqs_client::search::document::DocumentId;--/// Publishes a refresh of a document's denormalized name to the shared search-/// event queue so the search-processing service updates the indexed-/// `document_name` after a rename.-#[derive(Debug)]-pub struct SqsDocumentSearchIndexer {- sqs: SQS,-}--impl SqsDocumentSearchIndexer {- pub fn new(sqs: SQS) -> Self {- Self { sqs }- }-}--impl DocumentSearchIndexer for SqsDocumentSearchIndexer {- fn enqueue_name_update(- &self,- document_id: String,- ) -> std::pin::Pin<Box<dyn std::future::Future<Output = anyhow::Result<()>> + Send>> {- let sqs = self.sqs.clone();- Box::pin(async move {- sqs.send_message_to_search_event_queue(SearchQueueMessage::UpdateDocumentName(- DocumentId { document_id },- ))- .await?;- Ok(())- })- }-}diff --git a/services/document_storage_service/src/service/mod.rs b/services/document_storage_service/src/service/mod.rsindex 1d71089da3..f4d84f1703 100644--- a/services/document_storage_service/src/service/mod.rs+++ b/services/document_storage_service/src/service/mod.rs@@ -1,7 +1,7 @@pub mod conn_gateway;#[cfg(feature = "delete_document_worker")]pub mod delete_document_worker;-pub mod document_search_indexer;+pub mod document_event_publisher;pub mod entity_mutation;pub mod property_search_indexer;pub mod s3;diff --git a/services/mcp_service/src/context.rs b/services/mcp_service/src/context.rsindex aef036ace6..86c5365809 100644--- a/services/mcp_service/src/context.rs+++ b/services/mcp_service/src/context.rs@@ -261,9 +261,6 @@ async fn build_tool_context(args: ToolContextBuildArgs<'_>) -> anyhow::Result<To),foreign_entity_service: ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),macro_event_broker: macro_event_broker.clone(),- // No search event queue is configured in this context, so no- // search-index refresh is published from here.- search_indexer: None,};let lexical_client_for_tools = (*lexical_client).clone();let document_tool_context = DocumentToolContext::new(diff --git a/services/search_processing_service/Cargo.toml b/services/search_processing_service/Cargo.tomlindex 4b3957c742..1e7075884e 100644--- a/services/search_processing_service/Cargo.toml+++ b/services/search_processing_service/Cargo.toml@@ -11,6 +11,7 @@ processing = ["dep:call","dep:channels","dep:chat",+ "dep:documents","dep:kafka_util","dep:macro_event_broker","dep:macro_event_topics",@@ -19,7 +20,13 @@ processing = ["dep:rootcause","dep:tokio-retry",]-service = ["macro_db_client", "models_properties", "properties"]+service = [+ "dep:documents",+ "dep:macro_event_broker",+ "macro_db_client",+ "models_properties",+ "properties",+]pdf = ["dep:pdfium-render"][[bin]]@@ -66,6 +73,7 @@ macro_user_id = { path = "../../crates/macro_user_id" }macro_uuid = { path = "../../crates/macro_uuid" }mention_utils = { path = "../../crates/mention_utils" }document_sub_type = { path = "../../crates/document_sub_type" }+documents = { path = "../../crates/documents", default-features = false, optional = true }doppler-config = { workspace = true }model = { path = "../../crates/model" }s3_key = { path = "../../crates/s3_key" }@@ -97,7 +105,7 @@ strum = { workspace = true }thiserror = { workspace = true }tokio = { workspace = true, features = ["signal", "sync", "time"] }tokio-retry = { workspace = true, optional = true }-tokio-util = { workspace = true }+tokio-util = { workspace = true, features = ["rt"] }tower = { workspace = true }tower-http = { workspace = true }tracing = { workspace = true }diff --git a/services/search_processing_service/src/api/context.rs b/services/search_processing_service/src/api/context.rsindex 427b1e81ff..24b5dd085b 100644--- a/services/search_processing_service/src/api/context.rs+++ b/services/search_processing_service/src/api/context.rs@@ -2,21 +2,24 @@ use axum::extract::FromRef;use macro_authorization::{MacroAuthorizationServiceImpl, MacroAuthorizationState, NoopMacroAuthJwtValidator,};+use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};use std::sync::Arc;+use tokio_util::task::TaskTracker;use crate::BackfillServiceImpl;use crate::config::Config;use crate::domain::jobs::BackfillJobs;pub(crate) type AuthorizationService = MacroAuthorizationServiceImpl<NoopMacroAuthJwtValidator>;+pub(crate) type SpsEventBroker = MacroEventBrokerService<KafkaEventPublisher, TaskTracker>;#[derive(Clone, FromRef)]pub(crate) struct ApiContext {pub db: sqlx::Pool<sqlx::Postgres>,- pub sqs_client: Arc<sqs_client::SQS>,pub opensearch_client: Arc<opensearch_client::OpensearchClient>,pub authorization_state: MacroAuthorizationState<AuthorizationService>,pub config: Arc<Config>,pub backfill_service: Arc<BackfillServiceImpl>,pub backfill_jobs: BackfillJobs,+ pub macro_event_broker: SpsEventBroker,}diff --git a/services/search_processing_service/src/api/internal/extract_sync.rs b/services/search_processing_service/src/api/internal/extract_sync.rsindex 3ae6f7a95f..b7c1d3c335 100644--- a/services/search_processing_service/src/api/internal/extract_sync.rs+++ b/services/search_processing_service/src/api/internal/extract_sync.rs@@ -4,10 +4,13 @@ use axum::{http::StatusCode,response::{IntoResponse, Response},};+use documents::domain::events::{DocumentMacroEvent, DocumentSyncContentUpdatedMetadata};use macro_authorization::{InternalOnly, MacroAuthorizationExtractor};+use macro_event_broker::MacroEventBroker as _;use model::document::FileType;-use sqs_client::search::{SearchQueueMessage, document::SearchExtractorMessage};-use uuid::Uuid;++#[cfg(test)]+mod test;#[derive(serde::Deserialize, serde::Serialize)]pub struct SyncDocument {@@ -23,32 +26,24 @@ pub struct ExtractSyncRequest {pub documents: Vec<SyncDocument>,}-fn documents_to_messages(documents: Vec<SyncDocument>) -> Vec<SearchQueueMessage> {+fn documents_to_events(documents: Vec<SyncDocument>) -> Vec<DocumentMacroEvent> {documents- .iter()- .map(|doc| {- // generate random UUIDv4 for DeduplicationId trait- let document_version_id = if let Some(id) = &doc.document_version_id {- Some(id.clone())- } else {- Some(Uuid::new_v4().to_string())- };- SearchQueueMessage::ExtractSync(SearchExtractorMessage {- // HACK: we can leave this empty because we know that this code path doesn't use it- // it is mainly needed for get_document_info()- user_id: "".to_string(),- document_id: doc.document_id.clone(),- // NOTE: for now, we only expect markdown files from sync service. this could- // change- file_type: doc.file_type,- document_version_id,- index_override: None,- })+ .into_iter()+ .map(|document| {+ let document_id = document.document_id;+ DocumentMacroEvent::sync_content_updated(+ document_id.clone(),+ DocumentSyncContentUpdatedMetadata {+ document_id,+ file_type: document.file_type,+ document_version_id: document.document_version_id,+ },+ )}).collect()}-/// internal handler to batch queue messages to the worker queue+/// Internal handler to publish sync-content extraction events.#[tracing::instrument(skip(ctx, _internal_authorization, req))]pub async fn handler(State(ctx): State<ApiContext>,@@ -61,87 +56,17 @@ pub async fn handler(.map(|d| d.document_id.as_str()).collect();tracing::info!(?document_ids, "extract_sync received");- let messages: Vec<SearchQueueMessage> = documents_to_messages(req.documents);- ctx.sqs_client- .bulk_send_message_to_search_event_queue(messages)- .await- .map_err(|e| {- tracing::error!(error=?e, "failed to enqueue documents");+ let events = documents_to_events(req.documents);+ for event in events {+ drop(ctx.macro_event_broker.send_event(&event).map_err(|error| {+ tracing::error!(error=?error, "failed to publish document sync event");(StatusCode::INTERNAL_SERVER_ERROR,"failed to enqueue documents",).into_response()- })?;- Ok(StatusCode::OK.into_response())-}--#[cfg(test)]-mod tests {- use super::*;-- #[test]- fn test_documents_to_messsages() {- let documents: Vec<SyncDocument> = [- ["AAA", "macro|[email protected]", "md"],- ["CCC", "macro|[email protected]", "oops_not_a_file_format"],- ["BBB", "macro|[email protected]", "md"],- ]- .iter()- .map(|v| SyncDocument {- document_id: v[0].to_string(),- file_type: FileType::Md,- document_version_id: None,- })- .collect();- let messages = documents_to_messages(documents);- assert_eq!(messages.len(), 3);- let messages: Vec<SearchExtractorMessage> = messages- .into_iter()- .filter_map(|m| match m {- SearchQueueMessage::ExtractSync(body) => Some(body),- _ => None,- })- .collect();- assert_eq!(messages.len(), 3);- assert_eq!(&messages[0].document_id, "AAA");- assert_eq!(&messages[2].document_id, "BBB");+ })?);}- // Should be able to set it explicitly or have it auto-generated- #[test]- fn test_document_version_id() {- let documents: Vec<SyncDocument> = [- ["AAA", "macro|[email protected]", "md", "zzz"],- ["BBB", "macro|[email protected]", "md", ""],- ]- .iter()- .map(|v| SyncDocument {- document_id: v[0].to_string(),- //user_id: v[1].to_string(),- file_type: FileType::Md,- document_version_id: if v[3].is_empty() {- None- } else {- Some(v[3].to_string())- },- })- .collect();- let messages = documents_to_messages(documents);- assert_eq!(messages.len(), 2);- let messages: Vec<SearchExtractorMessage> = messages- .into_iter()- .filter_map(|m| match m {- SearchQueueMessage::ExtractSync(body) => Some(body),- _ => None,- })- .collect();- assert_eq!(messages.len(), 2);- assert!(&messages[0].document_version_id.is_some());- assert!(&messages[1].document_version_id.is_some());-- if let Some(id) = &messages[0].document_version_id {- assert_eq!(id, "zzz");- }- }+ Ok(StatusCode::OK.into_response())}diff --git a/services/search_processing_service/src/api/internal/extract_sync/test.rs b/services/search_processing_service/src/api/internal/extract_sync/test.rsnew file mode 100644index 0000000000..f1a75ccc78--- /dev/null+++ b/services/search_processing_service/src/api/internal/extract_sync/test.rs@@ -0,0 +1,58 @@+use documents::domain::events::{DocumentSyncContentUpdatedMetadata, DocumentTopicEvent};+use macro_event_broker::MacroEvent as _;+use model::document::FileType;++use super::{SyncDocument, documents_to_events};++#[test]+fn documents_map_to_events_with_matching_keys_and_exact_metadata() {+ let events = documents_to_events(vec![+ SyncDocument {+ document_id: "document-a".to_string(),+ document_version_id: Some("version-42".to_string()),+ file_type: FileType::Md,+ },+ SyncDocument {+ document_id: "document-b".to_string(),+ document_version_id: None,+ file_type: FileType::Pdf,+ },+ ]);++ assert_eq!(events.len(), 2);+ assert_eq!(events[0].key(), "document-a");+ assert_eq!(+ events[0].event().event,+ DocumentTopicEvent::SyncContentUpdated(DocumentSyncContentUpdatedMetadata {+ document_id: "document-a".to_string(),+ file_type: FileType::Md,+ document_version_id: Some("version-42".to_string()),+ })+ );+ assert_eq!(events[1].key(), "document-b");+ assert_eq!(+ events[1].event().event,+ DocumentTopicEvent::SyncContentUpdated(DocumentSyncContentUpdatedMetadata {+ document_id: "document-b".to_string(),+ file_type: FileType::Pdf,+ document_version_id: None,+ })+ );+}++#[test]+fn documents_map_to_one_event_each_in_input_order() {+ let events = documents_to_events(+ ["document-a", "document-c", "document-b"]+ .into_iter()+ .map(|document_id| SyncDocument {+ document_id: document_id.to_string(),+ document_version_id: None,+ file_type: FileType::Md,+ })+ .collect(),+ );++ let event_keys: Vec<&str> = events.iter().map(|event| event.key()).collect();+ assert_eq!(event_keys, ["document-a", "document-c", "document-b"]);+}diff --git a/services/search_processing_service/src/config.rs b/services/search_processing_service/src/config.rsindex 3233ba0b6f..14cb871fa3 100644--- a/services/search_processing_service/src/config.rs+++ b/services/search_processing_service/src/config.rs@@ -11,7 +11,7 @@ env_vars! {pub struct OpensearchPassword;pub struct DocumentStorageBucket;pub struct BackfillJobsTable;- /// Comma-separated Kafka bootstrap servers for live call lifecycle events.+ /// Comma-separated Kafka bootstrap servers for live search-index events.pub struct KafkaBrokers;}@@ -113,7 +113,7 @@ pub struct Config {/// The bucket where documents are storedpub document_storage_bucket: DocumentStorageBucket,- /// Comma-separated Kafka bootstrap servers for live call lifecycle events.+ /// Comma-separated Kafka bootstrap servers for live search-index events.pub kafka_brokers: KafkaBrokers,/// The number of workers to spawndiff --git a/services/search_processing_service/src/inbound/kafka_consumer.rs b/services/search_processing_service/src/inbound/kafka_consumer.rsindex cde317db90..a0f920a058 100644--- a/services/search_processing_service/src/inbound/kafka_consumer.rs+++ b/services/search_processing_service/src/inbound/kafka_consumer.rs@@ -7,14 +7,16 @@//! offsets are already committed.//!//! Per-entity event mapping and processing live in the [`call`], [`channel`],-//! [`chat`], and [`project`] submodules; this module owns the poll loop, worker,-//! retry policy, and commit semantics.+//! [`chat`], [`document`], and [`project`] submodules; this module owns the poll+//! loop, worker, retry policy, and commit semantics.#![allow(clippy::enum_variant_names)]mod call;mod channel;mod chat;+mod context;+mod document;mod project;#[cfg(test)]mod test;@@ -24,24 +26,25 @@ use std::{future::Future, time::Duration};use ::call::domain::events::CallMacroEvent;use ::chat::domain::events::ChatMacroEvent;use channels::domain::broker_events::ChannelMacroEvent;+use documents::domain::events::DocumentMacroEvent;use kafka_util::{GroupName, KafkaEventConsumer};use macro_event_broker::{KafkaConsumerAdapter, MacroEventCollection, MacroEventConsumerService};-use opensearch_client::OpensearchClient;use projects::domain::events::ProjectMacroEvent;use rdkafka::{consumer::CommitMode,message::{BorrowedMessage, Message as _},};use rootcause::prelude::{Report, ResultExt as _};-use sqlx::PgPool;use tokio::sync::mpsc;use tokio_retry::{Retry, strategy::ExponentialBackoff};use self::{call::process_call_event, channel::process_channel_event, chat::process_chat_event,- project::process_project_event,+ document::process_document_event, project::process_project_event,};+pub(crate) use self::context::KafkaProcessingContext;+/// Consumer group used for live search-index event offsets.pub(crate) struct SearchProcessingConsumerGroup;@@ -59,6 +62,7 @@ macro_event_broker::declare_topics!(CallMacroEvent,ChannelMacroEvent,ChatMacroEvent,+ DocumentMacroEvent,ProjectMacroEvent,);@@ -159,14 +163,16 @@ fn attach_event_coordinates<E>(})}-#[tracing::instrument(skip(db, opensearch_client, event), fields(partition, offset))]+#[tracing::instrument(skip(context, event), fields(partition, offset))]async fn process_event(- db: &PgPool,- opensearch_client: &OpensearchClient,+ context: &KafkaProcessingContext,event: &DeclaredMacroEvent,partition: i32,offset: i64,) -> EventOutcome {+ let db = &context.db;+ let opensearch_client = context.opensearch_client.as_ref();+match event {DeclaredMacroEvent::CallMacroEvent(event) => {process_call_event(db, opensearch_client, event, partition, offset).await@@ -177,6 +183,9 @@ async fn process_event(DeclaredMacroEvent::ChatMacroEvent(event) => {process_chat_event(db, opensearch_client, event, partition, offset).await}+ DeclaredMacroEvent::DocumentMacroEvent(event) => {+ process_document_event(context, event, partition, offset).await+ }DeclaredMacroEvent::ProjectMacroEvent(event) => {process_project_event(db, opensearch_client, event, partition, offset).await}@@ -184,14 +193,12 @@ async fn process_event(}async fn run_event_worker(- db: PgPool,- opensearch_client: OpensearchClient,+ context: KafkaProcessingContext,mut events: mpsc::Receiver<ReceivedEvent>,) {while let Some(received) = events.recv().await {let _ = process_event(- &db,- &opensearch_client,+ &context,&received.event,received.partition,received.offset,@@ -287,11 +294,10 @@ async fn poll_events(/// channel returns an error without committing the current Kafka message. On/// normal shutdown the sender is dropped and all buffered events are processed/// before this function returns.-#[tracing::instrument(skip(db, opensearch_client, shutdown), fields(brokers), err)]+#[tracing::instrument(skip(context, shutdown), fields(brokers), err)]pub(crate) async fn run_event_consumer(brokers: &str,- db: PgPool,- opensearch_client: OpensearchClient,+ context: KafkaProcessingContext,shutdown: impl Future<Output = ()> + Send,) -> Result<(), Report> {let consumer = KafkaEventConsumer::<SearchProcessingConsumerGroup>::from_env(brokers)?;@@ -306,7 +312,7 @@ pub(crate) async fn run_event_consumer();let (events_tx, events_rx) = mpsc::channel(CHANNEL_CAPACITY);- let worker = tokio::spawn(run_event_worker(db, opensearch_client, events_rx));+ let worker = tokio::spawn(run_event_worker(context, events_rx));let poll_result = poll_events(&consumer, events_tx, shutdown).await;let worker_result = worker.await;diff --git a/services/search_processing_service/src/inbound/kafka_consumer/context.rs b/services/search_processing_service/src/inbound/kafka_consumer/context.rsnew file mode 100644index 0000000000..dfdc66112f--- /dev/null+++ b/services/search_processing_service/src/inbound/kafka_consumer/context.rs@@ -0,0 +1,16 @@+use std::sync::Arc;++use lexical_client::LexicalClient;+use opensearch_client::OpensearchClient;+use s3_client::S3;+use sqlx::PgPool;++/// Shared dependencies used to process search-index events from Kafka.+#[derive(Clone)]+pub(crate) struct KafkaProcessingContext {+ pub(crate) db: PgPool,+ pub(crate) opensearch_client: Arc<OpensearchClient>,+ pub(crate) s3_client: Arc<S3>,+ pub(crate) document_storage_bucket: String,+ pub(crate) lexical_client: Arc<LexicalClient>,+}diff --git a/services/search_processing_service/src/inbound/kafka_consumer/document.rs b/services/search_processing_service/src/inbound/kafka_consumer/document.rsnew file mode 100644index 0000000000..c4aeaa7c70--- /dev/null+++ b/services/search_processing_service/src/inbound/kafka_consumer/document.rs@@ -0,0 +1,241 @@+//! Maps document lifecycle events to search-index actions and processes them.++use documents::domain::events::{DocumentMacroEvent, DocumentTopicEvent};+use macro_event_broker::MacroEvent as _;+use model::document::FileType;+use sqs_client::search::document::SearchExtractorMessage;++use super::{+ EventOutcome, KafkaProcessingContext, MAX_PROCESSING_ATTEMPTS, PROCESSING_RETRY_BASE_DELAY,+ retry_processing,+};+use crate::process::document::{+ process_extract_sync_message, process_extract_text_message, process_remove_message,+ process_update_name_message,+};++#[derive(Debug, Clone, PartialEq, Eq)]+pub(super) enum DocumentIndexAction {+ ExtractText {+ owner: String,+ file_type: FileType,+ document_version_id: Option<String>,+ },+ ExtractSync {+ file_type: FileType,+ document_version_id: Option<String>,+ },+ RefreshName,+ Remove,+ Ignore,+}++#[derive(Debug, Clone, PartialEq, Eq)]+pub(super) struct DocumentEventDescription {+ pub(super) action: DocumentIndexAction,+ pub(super) document_id: String,+ pub(super) event_type: &'static str,+}++pub(super) fn describe_document_event(event: &DocumentTopicEvent) -> DocumentEventDescription {+ match event {+ DocumentTopicEvent::Created(metadata) => DocumentEventDescription {+ action: DocumentIndexAction::Ignore,+ document_id: metadata.document_id.clone(),+ event_type: "document.created",+ },+ DocumentTopicEvent::Updated(metadata) => DocumentEventDescription {+ action: if metadata.document_name.is_some() {+ DocumentIndexAction::RefreshName+ } else {+ DocumentIndexAction::Ignore+ },+ document_id: metadata.document_id.clone(),+ event_type: "document.updated",+ },+ DocumentTopicEvent::Deleted(metadata) => DocumentEventDescription {+ action: DocumentIndexAction::Ignore,+ document_id: metadata.document_id.clone(),+ event_type: "document.deleted",+ },+ DocumentTopicEvent::ContentUploaded(metadata) => DocumentEventDescription {+ action: DocumentIndexAction::ExtractText {+ owner: metadata.owner.to_string(),+ file_type: metadata.file_type,+ document_version_id: metadata.document_version_id.clone(),+ },+ document_id: metadata.document_id.clone(),+ event_type: "document.content_uploaded",+ },+ DocumentTopicEvent::SyncContentUpdated(metadata) => DocumentEventDescription {+ action: DocumentIndexAction::ExtractSync {+ file_type: metadata.file_type,+ document_version_id: metadata.document_version_id.clone(),+ },+ document_id: metadata.document_id.clone(),+ event_type: "document.sync_content_updated",+ },+ DocumentTopicEvent::Purged(metadata) => DocumentEventDescription {+ action: DocumentIndexAction::Remove,+ document_id: metadata.document_id.clone(),+ event_type: "document.purged",+ },+ DocumentTopicEvent::Copied(metadata) => DocumentEventDescription {+ action: DocumentIndexAction::Ignore,+ document_id: metadata.document_id.clone(),+ event_type: "document.copied",+ },+ DocumentTopicEvent::Interaction(metadata) => DocumentEventDescription {+ action: DocumentIndexAction::Ignore,+ document_id: metadata.document_id.clone(),+ event_type: "document.interaction",+ },+ }+}++pub(super) fn stored_extractor_message(+ document_id: &str,+ owner: String,+ file_type: FileType,+ document_version_id: Option<String>,+) -> SearchExtractorMessage {+ SearchExtractorMessage {+ user_id: owner,+ document_id: document_id.to_string(),+ file_type,+ document_version_id,+ index_override: None,+ }+}++pub(super) fn sync_extractor_message(+ document_id: &str,+ file_type: FileType,+ document_version_id: Option<String>,+) -> SearchExtractorMessage {+ SearchExtractorMessage {+ user_id: String::new(),+ document_id: document_id.to_string(),+ file_type,+ document_version_id,+ index_override: None,+ }+}++async fn process_document_index_action(+ context: &KafkaProcessingContext,+ document_id: &str,+ action: DocumentIndexAction,+) -> anyhow::Result<()> {+ let opensearch_client = context.opensearch_client.as_ref();++ match action {+ DocumentIndexAction::ExtractText {+ owner,+ file_type,+ document_version_id,+ } => {+ let message =+ stored_extractor_message(document_id, owner, file_type, document_version_id);+ process_extract_text_message(+ opensearch_client,+ &context.db,+ context.s3_client.as_ref(),+ &context.document_storage_bucket,+ &message,+ )+ .await+ }+ DocumentIndexAction::ExtractSync {+ file_type,+ document_version_id,+ } => {+ let message = sync_extractor_message(document_id, file_type, document_version_id);+ process_extract_sync_message(+ opensearch_client,+ &context.db,+ context.s3_client.as_ref(),+ &context.document_storage_bucket,+ context.lexical_client.as_ref(),+ &message,+ )+ .await+ }+ DocumentIndexAction::RefreshName => {+ process_update_name_message(opensearch_client, &context.db, document_id).await+ }+ DocumentIndexAction::Remove => process_remove_message(opensearch_client, document_id).await,+ DocumentIndexAction::Ignore => Ok(()),+ }+}++pub(super) async fn process_document_event(+ context: &KafkaProcessingContext,+ event: &DocumentMacroEvent,+ partition: i32,+ offset: i64,+) -> EventOutcome {+ let description = describe_document_event(&event.event().event);+ let document_id = description.document_id.as_str();+ let event_type = description.event_type;++ if description.action == DocumentIndexAction::Ignore {+ tracing::trace!(+ document_id,+ event_type,+ partition,+ offset,+ "ignoring document event without a search-index action"+ );+ return EventOutcome::Ignored;+ }++ let result = retry_processing(|attempt| {+ let action = description.action.clone();+ async move {+ tracing::trace!(+ document_id,+ event_type,+ partition,+ offset,+ attempt,+ "processing document search-index event"+ );+ process_document_index_action(context, document_id, action)+ .await+ .inspect_err(|error| {+ if attempt < MAX_PROCESSING_ATTEMPTS {+ let retry_delay =+ PROCESSING_RETRY_BASE_DELAY * 2u32.pow(attempt.saturating_sub(1));+ tracing::warn!(+ error = ?error,+ document_id,+ event_type,+ partition,+ offset,+ attempt,+ delay_secs = retry_delay.as_secs(),+ "document search-index processing failed, retrying"+ );+ }+ })+ }+ })+ .await;++ match result {+ Ok(()) => EventOutcome::Indexed,+ Err(error) => {+ tracing::error!(+ error = ?error,+ document_id,+ event_type,+ partition,+ offset,+ attempts = MAX_PROCESSING_ATTEMPTS,+ "dropping document event after processing retries were exhausted"+ );+ EventOutcome::Dropped+ }+ }+}diff --git a/services/search_processing_service/src/inbound/kafka_consumer/test.rs b/services/search_processing_service/src/inbound/kafka_consumer/test.rsindex b62352a43f..54336ba4cd 100644--- a/services/search_processing_service/src/inbound/kafka_consumer/test.rs+++ b/services/search_processing_service/src/inbound/kafka_consumer/test.rs@@ -22,11 +22,19 @@ use channels::domain::{models::{ChannelSender, ChannelType},};use chrono::Utc;+use documents::domain::events::{+ DocumentContentUploadedMetadata, DocumentCopiedMetadata, DocumentCreatedMetadata,+ DocumentDeletedMetadata, DocumentInteractionMetadata, DocumentPurgedMetadata,+ DocumentSyncContentUpdatedMetadata, DocumentTopicEvent, DocumentUpdatedMetadata,+ InteractionReason,+};use macro_event_broker::{Event, EventBrokerError, MacroEvent as _, MessageParts};use macro_event_topics::{- MacroCallsTopic, MacroChannelsTopic, MacroChatsTopic, MacroProjectsTopic, Topic as _,+ MacroCallsTopic, MacroChannelsTopic, MacroChatsTopic, MacroDocumentsTopic, MacroProjectsTopic,+ Topic as _,};use macro_user_id::user_id::MacroUserIdStr;+use model::document::FileType;use projects::domain::events::{ProjectCreatedMetadata, ProjectDeletedMetadata, ProjectPermanentlyDeletedMetadata,ProjectRestoredMetadata, ProjectTopicEvent, ProjectUpdatedMetadata, ProjectUploadedMetadata,@@ -37,6 +45,10 @@ use super::{call::{CallEventDescription, CallIndexAction, describe_call_event},channel::{ChannelEventDescription, ChannelIndexAction, describe_channel_event},chat::{ChatEventDescription, ChatIndexAction, describe_chat_event},+ document::{+ DocumentEventDescription, DocumentIndexAction, describe_document_event,+ stored_extractor_message, sync_extractor_message,+ },project::{ProjectEventDescription, ProjectIndexAction, collect_project_ids, describe_project_event,},@@ -49,6 +61,8 @@ const MESSAGE_ID: Uuid = Uuid::from_u128(3);const CHAT_ID: &str = "chat-id";const SECOND_CHAT_ID: &str = "second-chat-id";const CHAT_MESSAGE_ID: &str = "chat-message-id";+const DOCUMENT_ID: &str = "document-id";+const SOURCE_DOCUMENT_ID: &str = "source-document-id";const PROJECT_ID: &str = "project-root";const CHILD_PROJECT_ID: &str = "project-child";const PARENT_PROJECT_ID: &str = "project-parent";@@ -430,6 +444,145 @@ fn channel_event_cases() -> Vec<(ChannelTopicEvent, ChannelEventDescription)> {]}+fn document_event_cases() -> Vec<(DocumentTopicEvent, DocumentEventDescription)> {+ let owner = user_id();++ vec![+ (+ DocumentTopicEvent::Created(DocumentCreatedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ owner: owner.clone(),+ document_name: "Document".to_string(),+ file_type: Some(FileType::Pdf),+ project_id: Some(PROJECT_ID.to_string()),+ sub_type: None,+ created_at: None,+ }),+ DocumentEventDescription {+ action: DocumentIndexAction::Ignore,+ document_id: DOCUMENT_ID.to_string(),+ event_type: "document.created",+ },+ ),+ (+ DocumentTopicEvent::Updated(DocumentUpdatedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ owner: owner.clone(),+ actor_user_id: Some(owner.clone()),+ document_name: Some("Renamed document".to_string()),+ previous_project_id: Some(PROJECT_ID.to_string()),+ project_id: None,+ file_type: None,+ share_permission_updated: false,+ }),+ DocumentEventDescription {+ action: DocumentIndexAction::RefreshName,+ document_id: DOCUMENT_ID.to_string(),+ event_type: "document.updated",+ },+ ),+ (+ DocumentTopicEvent::Updated(DocumentUpdatedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ owner: owner.clone(),+ actor_user_id: Some(owner.clone()),+ document_name: None,+ previous_project_id: None,+ project_id: Some(PROJECT_ID.to_string()),+ file_type: None,+ share_permission_updated: true,+ }),+ DocumentEventDescription {+ action: DocumentIndexAction::Ignore,+ document_id: DOCUMENT_ID.to_string(),+ event_type: "document.updated",+ },+ ),+ (+ DocumentTopicEvent::Deleted(DocumentDeletedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ actor_user_id: Some(owner.clone()),+ project_id: Some(PROJECT_ID.to_string()),+ }),+ DocumentEventDescription {+ action: DocumentIndexAction::Ignore,+ document_id: DOCUMENT_ID.to_string(),+ event_type: "document.deleted",+ },+ ),+ (+ DocumentTopicEvent::ContentUploaded(DocumentContentUploadedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ owner: owner.clone(),+ file_type: FileType::Pdf,+ document_version_id: Some("convert".to_string()),+ }),+ DocumentEventDescription {+ action: DocumentIndexAction::ExtractText {+ owner: owner.to_string(),+ file_type: FileType::Pdf,+ document_version_id: Some("convert".to_string()),+ },+ document_id: DOCUMENT_ID.to_string(),+ event_type: "document.content_uploaded",+ },+ ),+ (+ DocumentTopicEvent::SyncContentUpdated(DocumentSyncContentUpdatedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ file_type: FileType::Md,+ document_version_id: None,+ }),+ DocumentEventDescription {+ action: DocumentIndexAction::ExtractSync {+ file_type: FileType::Md,+ document_version_id: None,+ },+ document_id: DOCUMENT_ID.to_string(),+ event_type: "document.sync_content_updated",+ },+ ),+ (+ DocumentTopicEvent::Purged(DocumentPurgedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ }),+ DocumentEventDescription {+ action: DocumentIndexAction::Remove,+ document_id: DOCUMENT_ID.to_string(),+ event_type: "document.purged",+ },+ ),+ (+ DocumentTopicEvent::Copied(DocumentCopiedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ source_document_id: SOURCE_DOCUMENT_ID.to_string(),+ source_version_id: Some(7),+ owner,+ document_name: "Copied document".to_string(),+ file_type: Some(FileType::Pdf),+ project_id: Some(PROJECT_ID.to_string()),+ sub_type: None,+ }),+ DocumentEventDescription {+ action: DocumentIndexAction::Ignore,+ document_id: DOCUMENT_ID.to_string(),+ event_type: "document.copied",+ },+ ),+ (+ DocumentTopicEvent::Interaction(DocumentInteractionMetadata {+ document_id: DOCUMENT_ID.to_string(),+ reason: InteractionReason::Edited,+ }),+ DocumentEventDescription {+ action: DocumentIndexAction::Ignore,+ document_id: DOCUMENT_ID.to_string(),+ event_type: "document.interaction",+ },+ ),+ ]+}+fn project_event_cases() -> Vec<(ProjectTopicEvent, ProjectEventDescription<'static>)> {let owner = user_id();@@ -600,6 +753,7 @@ fn subscribes_to_declared_search_processing_topics_with_durable_group() {assert!(topics.contains(&MacroCallsTopic::TOPIC_STR));assert!(topics.contains(&MacroChannelsTopic::TOPIC_STR));assert!(topics.contains(&MacroChatsTopic::TOPIC_STR));+ assert!(topics.contains(&MacroDocumentsTopic::TOPIC_STR));assert!(topics.contains(&MacroProjectsTopic::TOPIC_STR));}@@ -682,6 +836,72 @@ fn maps_all_channel_lifecycle_events_to_index_actions() {}}+#[test]+fn maps_all_document_lifecycle_events_to_index_actions() {+ let cases = document_event_cases();+ assert_eq!(cases.len(), 9);++ for (event, expected) in cases {+ let serialized = serde_json::to_value(&event).expect("serializable document event");+ assert_eq!(serialized["event_type"], expected.event_type);+ assert_eq!(describe_document_event(&event), expected);+ }+}++#[test]+fn document_extraction_actions_preserve_optional_versions() {+ let content_uploaded = DocumentTopicEvent::ContentUploaded(DocumentContentUploadedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ owner: user_id(),+ file_type: FileType::Pdf,+ document_version_id: None,+ });+ assert_eq!(+ describe_document_event(&content_uploaded).action,+ DocumentIndexAction::ExtractText {+ owner: "macro|[email protected]".to_string(),+ file_type: FileType::Pdf,+ document_version_id: None,+ }+ );++ let sync_content_updated =+ DocumentTopicEvent::SyncContentUpdated(DocumentSyncContentUpdatedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ file_type: FileType::Md,+ document_version_id: Some("snapshot-7".to_string()),+ });+ assert_eq!(+ describe_document_event(&sync_content_updated).action,+ DocumentIndexAction::ExtractSync {+ file_type: FileType::Md,+ document_version_id: Some("snapshot-7".to_string()),+ }+ );+}++#[test]+fn document_extractor_messages_disable_index_overrides_and_set_expected_users() {+ let stored = stored_extractor_message(+ DOCUMENT_ID,+ "macro|[email protected]".to_string(),+ FileType::Pdf,+ None,+ );+ assert_eq!(stored.user_id, "macro|[email protected]");+ assert_eq!(stored.document_id, DOCUMENT_ID);+ assert_eq!(stored.file_type, FileType::Pdf);+ assert_eq!(stored.document_version_id, None);+ assert_eq!(stored.index_override, None);++ let sync = sync_extractor_message(DOCUMENT_ID, FileType::Md, Some("snapshot-7".to_string()));+ assert_eq!(sync.user_id, "");+ assert_eq!(sync.document_id, DOCUMENT_ID);+ assert_eq!(sync.file_type, FileType::Md);+ assert_eq!(sync.document_version_id.as_deref(), Some("snapshot-7"));+ assert_eq!(sync.index_override, None);+}+#[test]fn maps_all_project_lifecycle_events_to_reconciliation_actions() {let cases = project_event_cases();@@ -775,6 +995,83 @@ fn project_envelope_decodes_round_trip_with_string_key() {assert_eq!(decoded_event.event().event, event);}+#[test]+fn exact_macro_documents_envelopes_decode_into_document_events() {+ let cases: Vec<(&[u8], Event<DocumentTopicEvent>)> = vec![+ (+ br#"{+ "event_id":"00000000-0000-0000-0000-000000000001",+ "schema_version":1,+ "event_type":"document.content_uploaded",+ "metadata":{+ "document_id":"document-id",+ "owner":"macro|[email protected]",+ "file_type":"pdf",+ "document_version_id":"convert"+ }+ }"#,+ Event::with_event_id(+ Uuid::from_u128(1),+ DocumentTopicEvent::ContentUploaded(DocumentContentUploadedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ owner: user_id(),+ file_type: FileType::Pdf,+ document_version_id: Some("convert".to_string()),+ }),+ ),+ ),+ (+ br#"{+ "event_id":"00000000-0000-0000-0000-000000000002",+ "schema_version":1,+ "event_type":"document.sync_content_updated",+ "metadata":{+ "document_id":"document-id",+ "file_type":"md",+ "document_version_id":null+ }+ }"#,+ Event::with_event_id(+ Uuid::from_u128(2),+ DocumentTopicEvent::SyncContentUpdated(DocumentSyncContentUpdatedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ file_type: FileType::Md,+ document_version_id: None,+ }),+ ),+ ),+ (+ br#"{+ "event_id":"00000000-0000-0000-0000-000000000003",+ "schema_version":1,+ "event_type":"document.purged",+ "metadata":{"document_id":"document-id"}+ }"#,+ Event::with_event_id(+ Uuid::from_u128(3),+ DocumentTopicEvent::Purged(DocumentPurgedMetadata {+ document_id: DOCUMENT_ID.to_string(),+ }),+ ),+ ),+ ];++ for (payload, expected) in cases {+ let message = TestMessage {+ topic: MacroDocumentsTopic::TOPIC_STR,+ key: Some(DOCUMENT_ID.to_string()),+ payload: Some(payload.to_vec()),+ };++ let decoded = DeclaredMacroEvent::decode(&message).expect("decodable document event");+ let DeclaredMacroEvent::DocumentMacroEvent(decoded_event) = decoded else {+ panic!("expected document event");+ };+ assert_eq!(decoded_event.key(), DOCUMENT_ID);+ assert_eq!(decoded_event.event(), &expected);+ }+}+#[tokio::test]async fn malformed_and_unsupported_chat_messages_are_commit_safe() {let malformed = TestMessage {diff --git a/services/search_processing_service/src/main.rs b/services/search_processing_service/src/main.rsindex ccd8fbf563..5bdd81317f 100644--- a/services/search_processing_service/src/main.rs+++ b/services/search_processing_service/src/main.rs@@ -1,10 +1,8 @@#![recursion_limit = "256"]-use std::sync::Arc;-#[cfg(feature = "processing")]-use std::time::Duration;+use std::{sync::Arc, time::Duration};#[cfg(feature = "processing")]-use crate::inbound::kafka_consumer::run_event_consumer;+use crate::inbound::kafka_consumer::{KafkaProcessingContext, run_event_consumer};use crate::{api::context::{ApiContext, AuthorizationService},config::DatabaseUrlReadonly,@@ -17,6 +15,7 @@ use config::{Config, Environment};use lexical_client::LexicalClient;use macro_authorization::{InternalAuthConfig, MacroAuthorizationState, NoopMacroAuthJwtValidator};use macro_entrypoint::MacroEntrypoint;+use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};use opensearch_client::OpensearchClient;#[cfg(feature = "pdf")]use rust_embed::RustEmbed;@@ -24,7 +23,7 @@ use sqlx::PgPool;use sqlx::postgres::PgPoolOptions;#[cfg(feature = "processing")]use tokio_retry::{Retry, strategy::FixedInterval};-use tokio_util::sync::CancellationToken;+use tokio_util::{sync::CancellationToken, task::TaskTracker};mod api;mod config;@@ -74,18 +73,17 @@ struct PdfiumLib;#[cfg(feature = "processing")]const CONSUMER_RESTART_DELAY: Duration = Duration::from_secs(5);+const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);#[cfg(feature = "processing")]async fn supervise_event_consumer(brokers: String,- db: PgPool,- opensearch_client: OpensearchClient,+ context: KafkaProcessingContext,shutdown_token: CancellationToken,) {let _ = Retry::start(FixedInterval::new(CONSUMER_RESTART_DELAY), || {let consumer_brokers = brokers.clone();- let consumer_db = db.clone();- let consumer_opensearch_client = opensearch_client.clone();+ let consumer_context = context.clone();let consumer_shutdown_token = shutdown_token.clone();async move {@@ -97,8 +95,7 @@ async fn supervise_event_consumer(let consumer_result = tokio::spawn(async move {run_event_consumer(&consumer_brokers,- consumer_db,- consumer_opensearch_client,+ consumer_context,task_shutdown_token.cancelled_owned(),).await@@ -159,7 +156,7 @@ async fn main() -> anyhow::Result<()> {let sqs_client = sqs_client::SQS::new(aws_sdk_sqs::Client::new(&aws_config)).search_event_queue(&search_event_queue);- let s3_client = s3_client::S3::new(macro_aws_config::s3_client().await);+ let s3_client = Arc::new(s3_client::S3::new(macro_aws_config::s3_client().await));let (min_connections, max_connections): (u32, u32) = match config.environment {Environment::Production => (5, 50),@@ -180,12 +177,14 @@ async fn main() -> anyhow::Result<()> {"initialized db connection");- let opensearch_client = OpensearchClient::new(- config.opensearch_url.to_string(),- config.opensearch_username.to_string(),- config.opensearch_password.as_ref().to_string(),- )- .context("unable to create opensearch client")?;+ let opensearch_client = Arc::new(+ OpensearchClient::new(+ config.opensearch_url.to_string(),+ config.opensearch_username.to_string(),+ config.opensearch_password.as_ref().to_string(),+ )+ .context("unable to create opensearch client")?,+ );if let Err(e) = opensearch_client.health().await {tracing::error!(error=?e, "error connecting to opensearch");@@ -215,11 +214,15 @@ async fn main() -> anyhow::Result<()> {));let shutdown_token = CancellationToken::new();+ let event_broker_tracker = TaskTracker::new();+ let macro_event_broker = MacroEventBrokerService::new(+ KafkaEventPublisher::new(config.kafka_brokers.as_ref())+ .context("failed to create kafka event publisher")?,+ event_broker_tracker.clone(),+ );#[cfg(feature = "processing")]let consumer_supervisor = {- use std::sync::Arc;-// Ensures that pdfium binary exists so we can kill the container early on failure#[cfg(feature = "pdf")]if !std::fs::exists("./pdfium-lib/linux/libpdfium.so").expect("able to find file") {@@ -228,10 +231,10 @@ async fn main() -> anyhow::Result<()> {tracing::trace!("libpdfium is present");}- let lexical_client = LexicalClient::new(+ let lexical_client = Arc::new(LexicalClient::new(config.internal_api_key.to_string(),config.lexical_service_url.clone(),- );+ ));let worker = sqs_worker::SQSWorker::new(aws_sdk_sqs::Client::new(&aws_config),@@ -239,20 +242,26 @@ async fn main() -> anyhow::Result<()> {config.queue_max_messages,config.queue_wait_time_seconds,);- let ctx = SearchProcessingContext {+ let search_processing_context = SearchProcessingContext {db: db.clone(),worker: Arc::new(worker.clone()),document_storage_bucket: config.document_storage_bucket.to_string(),- s3_client: Arc::new(s3_client),- opensearch_client: Arc::new(opensearch_client.clone()),- lexical_client: Arc::new(lexical_client),+ s3_client: s3_client.clone(),+ opensearch_client: opensearch_client.clone(),+ lexical_client: lexical_client.clone(),};- run_search_processing_workers(ctx, config.worker_count);+ run_search_processing_workers(search_processing_context, config.worker_count);+ let kafka_processing_context = KafkaProcessingContext {+ db: db.clone(),+ opensearch_client: opensearch_client.clone(),+ s3_client: s3_client.clone(),+ document_storage_bucket: config.document_storage_bucket.to_string(),+ lexical_client,+ };tokio::spawn(supervise_event_consumer(config.kafka_brokers.as_ref().to_owned(),- db.clone(),- opensearch_client.clone(),+ kafka_processing_context,shutdown_token.clone(),))};@@ -274,11 +283,11 @@ async fn main() -> anyhow::Result<()> {ApiContext {db,authorization_state,- sqs_client,- opensearch_client: Arc::new(opensearch_client),+ opensearch_client,config: Arc::new(config),backfill_service,backfill_jobs,+ macro_event_broker,},shutdown_token.clone(),)@@ -286,9 +295,24 @@ async fn main() -> anyhow::Result<()> {shutdown_token.cancel();#[cfg(feature = "processing")]- consumer_supervisor+ let consumer_result = consumer_supervisor.await- .context("search processing event consumer supervisor failed")?;+ .context("search processing event consumer supervisor failed");+ tracing::info!("waiting for event broker publishes to drain");+ event_broker_tracker.close();+ match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {+ Ok(()) => tracing::info!("event broker publishes drained"),+ Err(error) => {+ tracing::warn!(+ error=?error,+ timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),+ "timed out waiting for event broker publishes to drain"+ );+ }+ }++ #[cfg(feature = "processing")]+ consumer_result?;api_result}diff --git a/services/search_processing_service/src/process/document/mod.rs b/services/search_processing_service/src/process/document/mod.rsindex 7cfdd78bbd..52e5f03aa1 100644--- a/services/search_processing_service/src/process/document/mod.rs+++ b/services/search_processing_service/src/process/document/mod.rs@@ -1,17 +1,15 @@use anyhow::Context;use opensearch_client::OpensearchClient;-use sqs_client::search::document::{DocumentId, SearchExtractorMessage};+use sqs_client::search::document::SearchExtractorMessage;mod document_info;mod raw_document;pub async fn process_remove_message(opensearch_client: &OpensearchClient,- remove_message: &DocumentId,+ document_id: &str,) -> anyhow::Result<()> {- opensearch_client- .delete_document(remove_message.document_id.as_str(), None)- .await?;+ opensearch_client.delete_document(document_id, None).await?;Ok(())}@@ -25,18 +23,13 @@ pub async fn process_remove_message(pub async fn process_update_name_message(opensearch_client: &OpensearchClient,db: &sqlx::Pool<sqlx::Postgres>,- update_message: &DocumentId,+ document_id: &str,) -> anyhow::Result<()> {- let document = match macro_db_client::document::get_basic_document(- db,- update_message.document_id.as_str(),- )- .await- {+ let document = match macro_db_client::document::get_basic_document(db, document_id).await {Ok(document) => document,Err(sqlx::Error::RowNotFound) => {tracing::debug!(- document_id = %update_message.document_id,+ document_id,"document not found; skipping search name update");return Ok(());@@ -46,14 +39,14 @@ pub async fn process_update_name_message(if document.deleted_at.is_some() {tracing::debug!(- document_id = %update_message.document_id,+ document_id,"document is deleted; skipping search name update");return Ok(());}opensearch_client- .update_document_metadata(update_message.document_id.as_str(), &document.document_name)+ .update_document_metadata(document_id, &document.document_name).await.context("failed to update document name in search index")?;diff --git a/services/search_processing_service/src/process/document/raw_document.rs b/services/search_processing_service/src/process/document/raw_document.rsindex 4cd0c1acd2..2f27301eea 100644--- a/services/search_processing_service/src/process/document/raw_document.rs+++ b/services/search_processing_service/src/process/document/raw_document.rs@@ -233,6 +233,7 @@ pub async fn update_search_with_raw_document(};let document_name = document_info.document_name;+ let owner_id = document_info.owner.to_string();let sub_type = document_info.sub_type.map(|st| st.to_string());// TODO: this is hacky, update the search event message to use the correctly serialized@@ -313,7 +314,7 @@ pub async fn update_search_with_raw_document(raw_content: None,document_name: document_name.clone(),content: page_content.clone(),- owner_id: search_extractor_message.user_id.clone(),+ owner_id: owner_id.clone(),file_type: file_type.to_string(),updated_at_millis,sub_type: sub_type.clone(),@@ -342,7 +343,7 @@ pub async fn update_search_with_raw_document(raw_content: Some(result.raw_content),document_name: document_name.clone(),content: result.content,- owner_id: search_extractor_message.user_id.clone(),+ owner_id: owner_id.clone(),file_type: file_type.to_string(),updated_at_millis,sub_type: sub_type.clone(),@@ -361,7 +362,7 @@ pub async fn update_search_with_raw_document(raw_content: None,document_name,content: content.clone(),- owner_id: search_extractor_message.user_id.clone(),+ owner_id,file_type: file_type.to_string(),updated_at_millis,sub_type: sub_type.clone(),diff --git a/services/search_processing_service/src/process/mod.rs b/services/search_processing_service/src/process/mod.rsindex b7c1b81daa..4350fc1935 100644--- a/services/search_processing_service/src/process/mod.rs+++ b/services/search_processing_service/src/process/mod.rs@@ -2,7 +2,7 @@ pub(crate) mod call;pub(crate) mod channel;pub(crate) mod chat;pub mod context;-mod document;+pub(crate) mod document;mod email;pub(crate) mod project;mod properties;@@ -80,9 +80,6 @@ pub async fn process_message(email::upsert::process_upsert_message(&ctx.opensearch_client, &ctx.db, &message).await?;}- SearchQueueMessage::RemoveDocument(message) => {- document::process_remove_message(&ctx.opensearch_client, &message).await?;- }SearchQueueMessage::ExtractDocumentText(message) => {document::process_extract_text_message(&ctx.opensearch_client,@@ -108,10 +105,6 @@ pub async fn process_message(properties::process_entity_property_update(&ctx.opensearch_client, &ctx.db, &message).await?;}- SearchQueueMessage::UpdateDocumentName(message) => {- document::process_update_name_message(&ctx.opensearch_client, &ctx.db, &message)- .await?;- }SearchQueueMessage::ChatMessage(message) => {chat::insert_chat_message(&ctx.opensearch_client, &ctx.db, &message).await?;}diff --git a/services/search_upload_handler/Cargo.toml b/services/search_upload_handler/Cargo.tomlindex 7fa0776e27..d34b429af0 100644--- a/services/search_upload_handler/Cargo.toml+++ b/services/search_upload_handler/Cargo.toml@@ -6,21 +6,13 @@ version = "0.1.0"[dependencies]anyhow = { workspace = true }-aws-sdk-sqs = { workspace = true }-document_storage_service_client = { path = "../../crates/document_storage_service_client" }-macro_aws_config = { path = "../../crates/macro_aws_config" }aws_lambda_events = { workspace = true, features = ["eventbridge"] }+document_storage_service_client = { path = "../../crates/document_storage_service_client" }lambda_runtime = { workspace = true }macro_entrypoint = { path = "../../crates/macro_entrypoint" }macro_env_var = { path = "../../crates/macro_env_var" }-macro_queues = { path = "../../crates/macro_queues" }+macro_service_urls = { path = "../../crates/macro_service_urls" }s3_key = { path = "../../crates/s3_key" }-sqs_client = { path = "../../crates/sqs_client", default-features = false, features = [- "search",-] }tokio = { workspace = true }tracing = { workspace = true }---macro_service_urls = { path = "../../crates/macro_service_urls" }workspace-hack = { version = "0.1", path = "../../crates/workspace-hack" }diff --git a/services/search_upload_handler/src/handler.rs b/services/search_upload_handler/src/handler.rsindex fde704e35f..437ea37fac 100644--- a/services/search_upload_handler/src/handler.rs+++ b/services/search_upload_handler/src/handler.rs@@ -6,12 +6,10 @@ use lambda_runtime::{tracing::{self},};use s3_key::DocumentKey;-use sqs_client::search::{SearchQueueMessage, document::SearchExtractorMessage};/// Handles the Eventbridge event-#[tracing::instrument(skip(sqs_client, dss_client), err)]+#[tracing::instrument(skip(dss_client), err)]pub async fn handler(- sqs_client: &sqs_client::SQS,dss_client: &DocumentStorageServiceClient,event: LambdaEvent<EventBridgeEvent>,) -> Result<(), Error> {@@ -57,22 +55,12 @@ pub async fn handler(}};- let search_extractor_message = SearchExtractorMessage {- user_id: document_basic.owner.to_string(),- document_id: document_id.to_string(),- document_version_id: document_key.version_id_string(),- file_type,- index_override: None,- };-- // All other file types are to be sent to the search text extractor queue- let message_id = sqs_client- .send_message_to_search_event_queue(SearchQueueMessage::ExtractDocumentText(- search_extractor_message,- ))+ let document_version_id = document_key.version_id_string();+ dss_client+ .publish_document_content_uploaded(document_id, file_type, document_version_id).await?;- tracing::info!(message_id=?message_id, "sent message to search extractor queue");+ tracing::info!(document_id, "relayed document content upload");Ok(())}diff --git a/services/search_upload_handler/src/main.rs b/services/search_upload_handler/src/main.rsindex 87e91f5e87..309f1a3f80 100644--- a/services/search_upload_handler/src/main.rs+++ b/services/search_upload_handler/src/main.rs@@ -6,7 +6,6 @@ use lambda_runtime::{Error, LambdaEvent, run, service_fn, tracing};use macro_entrypoint::MacroEntrypoint;use macro_env_var::env_vars;use macro_service_urls::DocumentStorageServiceUrl;-use std::sync::Arc;mod handler;@@ -20,26 +19,16 @@ async fn main() -> Result<(), Error> {tracing::trace!("initiating lambda");- let search_event_queue = macro_queues::SearchEventQueue::new();-let dss_url = DocumentStorageServiceUrl::new()?.to_string();let dss_auth_key = DocumentStorageServiceAuthKey::new().context("DOCUMENT_STORAGE_SERVICE_AUTH_KEY must be provided")?.to_string();- let sqs_client = sqs_client::SQS::new(aws_sdk_sqs::Client::new(- ¯o_aws_config::get_macro_aws_config().await,- ))- .search_event_queue(search_event_queue.as_ref());-- let shared_sqs_client = Arc::new(sqs_client);- let shared_dss_client = Arc::new(DocumentStorageServiceClient::new(dss_auth_key, dss_url));+ let dss_client = DocumentStorageServiceClient::new(dss_auth_key, dss_url);let func = service_fn(move |event: LambdaEvent<EventBridgeEvent>| {- let sqs_client = shared_sqs_client.clone();- let dss_client = shared_dss_client.clone();-- async move { handler(&sqs_client, &dss_client, event).await }+ let dss_client = dss_client.clone();+ async move { handler(&dss_client, event).await }});run(func).await