Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

feat(search_processing_service): documents kafka (#5313)

Will Hutchinsond0295c70156c
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.json
index b8c11e1a33..e3f6adedb9 100644
--- a/.github/workspace-dep-closures.json
+++ b/.github/workspace-dep-closures.json
@@ -1297,24 +1297,34 @@
"crates/attachment",
"crates/authentication_service_client",
"crates/bot_id",
+ "crates/call",
"crates/channel_sender",
+ "crates/channels",
"crates/chat",
+ "crates/connection",
+ "crates/connection_gateway_client",
+ "crates/connection_gateway_models",
+ "crates/contacts",
"crates/cowlike",
"crates/crm",
"crates/document_sub_type",
+ "crates/documents",
"crates/dynamodb_client",
"crates/email",
"crates/email_utils",
+ "crates/embedding",
"crates/entity_access",
"crates/entity_access_db_utils",
"crates/entity_access_management",
"crates/entity_mutation",
"crates/filter_ast",
+ "crates/foreign_entity",
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
"crates/kafka_util",
"crates/last_online_tracker",
+ "crates/lexical_client",
"crates/macro_auth",
"crates/macro_authorization",
"crates/macro_aws_config",
@@ -1327,6 +1337,7 @@
"crates/macro_queues",
"crates/macro_service_urls",
"crates/macro_sha_count_client",
+ "crates/macro_sync_service_jwt",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -1345,9 +1356,14 @@
"crates/models_pagination",
"crates/models_permissions",
"crates/models_properties",
+ "crates/models_search",
+ "crates/models_search_cursor",
+ "crates/models_soup",
"crates/models_team",
"crates/non_empty",
"crates/notification",
+ "crates/opensearch_client",
+ "crates/opensearch_query_builder",
"crates/projects",
"crates/prompt",
"crates/properties",
@@ -1355,8 +1371,14 @@
"crates/remote_env_var",
"crates/roles_and_permissions",
"crates/s3_key",
+ "crates/secretsmanager_client",
+ "crates/share_permission_db_utils",
"crates/sqs_client",
+ "crates/sqs_worker",
+ "crates/sync_service",
+ "crates/sync_service_client",
"crates/system_properties",
+ "crates/task_dedup",
"crates/unfurl",
"crates/user_quota",
"crates/workspace-hack",
@@ -4564,32 +4586,44 @@
"crates/attachment",
"crates/authentication_service_client",
"crates/bot_id",
+ "crates/call",
"crates/channel_sender",
+ "crates/channels",
+ "crates/connection",
+ "crates/connection_gateway_client",
+ "crates/connection_gateway_models",
+ "crates/contacts",
"crates/cowlike",
"crates/crm",
"crates/document_sub_type",
+ "crates/documents",
"crates/dynamodb_client",
"crates/email",
"crates/email_utils",
+ "crates/embedding",
"crates/entity_access",
"crates/entity_access_db_utils",
"crates/entity_access_management",
"crates/entity_mutation",
"crates/filter_ast",
+ "crates/foreign_entity",
"crates/frecency",
"crates/invite_email",
"crates/item_filters",
"crates/kafka_util",
"crates/last_online_tracker",
+ "crates/lexical_client",
"crates/macro_auth",
"crates/macro_authorization",
"crates/macro_aws_config",
+ "crates/macro_entrypoint",
"crates/macro_env",
"crates/macro_env_var",
"crates/macro_event_broker",
"crates/macro_event_topics",
"crates/macro_service_urls",
"crates/macro_sha_count_client",
+ "crates/macro_sync_service_jwt",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -4603,19 +4637,31 @@
"crates/models_bulk_upload",
"crates/models_email",
"crates/models_entity_access_management",
+ "crates/models_opensearch",
"crates/models_pagination",
"crates/models_permissions",
"crates/models_properties",
+ "crates/models_search",
+ "crates/models_search_cursor",
+ "crates/models_soup",
"crates/non_empty",
"crates/notification",
+ "crates/opensearch_client",
+ "crates/opensearch_query_builder",
"crates/projects",
"crates/prompt",
"crates/properties",
"crates/rate_limit",
"crates/remote_env_var",
"crates/s3_key",
+ "crates/secretsmanager_client",
+ "crates/share_permission_db_utils",
"crates/sqs_client",
+ "crates/sqs_worker",
+ "crates/sync_service",
+ "crates/sync_service_client",
"crates/system_properties",
+ "crates/task_dedup",
"crates/unfurl",
"crates/workspace-hack"
],
@@ -4898,16 +4944,19 @@
"crates/cowlike",
"crates/crm",
"crates/document_sub_type",
+ "crates/documents",
"crates/dynamodb_client",
"crates/email",
"crates/email_db_client",
"crates/email_utils",
+ "crates/embedding",
"crates/ensure_exists",
"crates/entity_access",
"crates/entity_access_db_utils",
"crates/entity_access_management",
"crates/entity_mutation",
"crates/filter_ast",
+ "crates/foreign_entity",
"crates/frecency",
"crates/invite_email",
"crates/ip_extractor",
@@ -4931,6 +4980,7 @@
"crates/macro_queues",
"crates/macro_service_urls",
"crates/macro_sha_count_client",
+ "crates/macro_sync_service_jwt",
"crates/macro_user_id",
"crates/macro_uuid",
"crates/mention_utils",
@@ -4965,10 +5015,14 @@
"crates/roles_and_permissions",
"crates/s3_client",
"crates/s3_key",
+ "crates/secretsmanager_client",
"crates/share_permission_db_utils",
"crates/sqs_client",
"crates/sqs_worker",
+ "crates/sync_service",
+ "crates/sync_service_client",
"crates/system_properties",
+ "crates/task_dedup",
"crates/unfurl",
"crates/user_quota",
"crates/workspace-hack",
@@ -5129,61 +5183,30 @@
"crates/ai_toolset",
"crates/ai_usage",
"crates/attachment",
- "crates/authentication_service_client",
"crates/bot_id",
- "crates/channel_sender",
"crates/cowlike",
- "crates/crm",
"crates/document_storage_service_client",
"crates/document_sub_type",
- "crates/email",
- "crates/email_utils",
- "crates/entity_access",
- "crates/entity_access_db_utils",
- "crates/entity_access_management",
- "crates/entity_mutation",
- "crates/filter_ast",
- "crates/frecency",
- "crates/invite_email",
- "crates/item_filters",
- "crates/kafka_util",
- "crates/last_online_tracker",
"crates/macro_auth",
"crates/macro_authorization",
- "crates/macro_aws_config",
"crates/macro_entrypoint",
"crates/macro_env",
"crates/macro_env_var",
- "crates/macro_event_broker",
- "crates/macro_event_topics",
- "crates/macro_queues",
"crates/macro_service_urls",
"crates/macro_user_id",
"crates/macro_uuid",
- "crates/mention_utils",
"crates/model",
"crates/model-entity",
"crates/model-error-response",
"crates/model_file_type",
- "crates/model_notifications",
"crates/model_user",
- "crates/models_ai_projection",
"crates/models_bulk_upload",
- "crates/models_email",
- "crates/models_entity_access_management",
"crates/models_pagination",
"crates/models_permissions",
- "crates/models_properties",
"crates/non_empty",
- "crates/notification",
"crates/prompt",
- "crates/properties",
- "crates/rate_limit",
"crates/remote_env_var",
"crates/s3_key",
- "crates/sqs_client",
- "crates/system_properties",
- "crates/unfurl",
"crates/workspace-hack",
"services/search_upload_handler"
],
diff --git a/Cargo.lock b/Cargo.lock
index 02686eeb9a..a67c1b90a3 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -3997,6 +3997,7 @@ dependencies = [
"aws_lambda_events",
"chat",
"chrono",
+ "documents",
"futures",
"lambda_runtime",
"macro_aws_config",
@@ -11417,11 +11418,13 @@ dependencies = [
"anyhow",
"async-recursion",
"aws-sdk-s3",
+ "aws-sdk-sqs",
"axum",
"axum-extra",
"base64 0.22.1",
"chrono",
"document_sub_type",
+ "documents",
"dotenvy",
"dynamodb_client",
"entity_access",
@@ -12893,6 +12896,7 @@ dependencies = [
"chrono",
"comms_db_client",
"document_sub_type",
+ "documents",
"doppler-config",
"email_db_client",
"ensure_exists",
@@ -13020,17 +13024,13 @@ name = "search_upload_handler"
version = "0.1.0"
dependencies = [
"anyhow",
- "aws-sdk-sqs",
"aws_lambda_events",
"document_storage_service_client",
"lambda_runtime",
- "macro_aws_config",
"macro_entrypoint",
"macro_env_var",
- "macro_queues",
"macro_service_urls",
"s3_key",
- "sqs_client",
"tokio",
"tracing",
"workspace-hack",
diff --git a/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentContentUploadedMetadata.ts b/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentContentUploadedMetadata.ts
new file mode 100644
index 0000000000..0561091f1a
--- /dev/null
+++ b/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentContentUploadedMetadata.ts
@@ -0,0 +1,25 @@
+/**
+ * Generated by orval v7.21.0 🍺
+ * Do not edit manually.
+ * document_storage_service
+ * OpenAPI spec version: 0.1.0
+ */
+import type { DocumentContentUploadedMetadataDocumentVersionId } from './documentContentUploadedMetadataDocumentVersionId';
+import type { FileType } from './fileType';
+import type { MacroUserIdStr } from './macroUserIdStr';
+
+/**
+ * Metadata for [`DocumentTopicEvent::ContentUploaded`].
+ */
+export interface DocumentContentUploadedMetadata {
+ /** The id of the document whose stored bytes changed. */
+ document_id: string;
+ /** Version written, or the converted-file marker; `None` for unversioned
+writes (mirrors `SearchExtractorMessage::document_version_id`). */
+ document_version_id?: DocumentContentUploadedMetadataDocumentVersionId;
+ /** File type of the uploaded object (may differ from the document's own
+type, e.g. `pdf` for the converted rendition of a docx). */
+ file_type: FileType;
+ /** The owner of the document (used by the extractor to resolve S3 keys). */
+ owner: MacroUserIdStr;
+}
diff --git a/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentContentUploadedMetadataDocumentVersionId.ts b/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentContentUploadedMetadataDocumentVersionId.ts
new file mode 100644
index 0000000000..48cd42890c
--- /dev/null
+++ b/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentContentUploadedMetadataDocumentVersionId.ts
@@ -0,0 +1,12 @@
+/**
+ * Generated by orval v7.21.0 🍺
+ * Do not edit manually.
+ * document_storage_service
+ * OpenAPI spec version: 0.1.0
+ */
+
+/**
+ * Version written, or the converted-file marker; `None` for unversioned
+writes (mirrors `SearchExtractorMessage::document_version_id`).
+ */
+export type DocumentContentUploadedMetadataDocumentVersionId = string | null;
diff --git a/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentPurgedMetadata.ts b/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentPurgedMetadata.ts
new file mode 100644
index 0000000000..f2c92c3bd7
--- /dev/null
+++ b/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentPurgedMetadata.ts
@@ -0,0 +1,14 @@
+/**
+ * Generated by orval v7.21.0 🍺
+ * Do not edit manually.
+ * document_storage_service
+ * OpenAPI spec version: 0.1.0
+ */
+
+/**
+ * Metadata for [`DocumentTopicEvent::Purged`].
+ */
+export interface DocumentPurgedMetadata {
+ /** The id of the hard-deleted document. */
+ document_id: string;
+}
diff --git a/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentSyncContentUpdatedMetadata.ts b/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentSyncContentUpdatedMetadata.ts
new file mode 100644
index 0000000000..42bd7a7400
--- /dev/null
+++ b/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentSyncContentUpdatedMetadata.ts
@@ -0,0 +1,20 @@
+/**
+ * Generated by orval v7.21.0 🍺
+ * Do not edit manually.
+ * document_storage_service
+ * OpenAPI spec version: 0.1.0
+ */
+import type { DocumentSyncContentUpdatedMetadataDocumentVersionId } from './documentSyncContentUpdatedMetadataDocumentVersionId';
+import type { FileType } from './fileType';
+
+/**
+ * Metadata for [`DocumentTopicEvent::SyncContentUpdated`].
+ */
+export interface DocumentSyncContentUpdatedMetadata {
+ /** The id of the live-collab document whose content changed. */
+ document_id: string;
+ /** Version marker for the sync snapshot, when the caller supplies one. */
+ document_version_id?: DocumentSyncContentUpdatedMetadataDocumentVersionId;
+ /** File type of the sync document (markdown today). */
+ file_type: FileType;
+}
diff --git a/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentSyncContentUpdatedMetadataDocumentVersionId.ts b/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentSyncContentUpdatedMetadataDocumentVersionId.ts
new file mode 100644
index 0000000000..f241d7be3c
--- /dev/null
+++ b/apps/web/src/lib/service-clients/service-storage/generated/schemas/documentSyncContentUpdatedMetadataDocumentVersionId.ts
@@ -0,0 +1,11 @@
+/**
+ * Generated by orval v7.21.0 🍺
+ * Do not edit manually.
+ * document_storage_service
+ * OpenAPI spec version: 0.1.0
+ */
+
+/**
+ * Version marker for the sync snapshot, when the caller supplies one.
+ */
+export type DocumentSyncContentUpdatedMetadataDocumentVersionId = string | null;
diff --git a/apps/web/src/lib/service-clients/service-storage/generated/schemas/index.ts b/apps/web/src/lib/service-clients/service-storage/generated/schemas/index.ts
index a3788870ad..7193199afd 100644
--- a/apps/web/src/lib/service-clients/service-storage/generated/schemas/index.ts
+++ b/apps/web/src/lib/service-clients/service-storage/generated/schemas/index.ts
@@ -403,6 +403,8 @@ export * from './documentContent';
export * from './documentContentLocation';
export * from './documentContentLocationProperty';
export * from './documentContentState';
+export * from './documentContentUploadedMetadata';
+export * from './documentContentUploadedMetadataDocumentVersionId';
export * from './documentFilters';
export * from './documentFiltersImportance';
export * from './documentFiltersIsEmailAttachment';
@@ -447,6 +449,7 @@ export * from './documentPreviewOneOfFourAllOfType';
export * from './documentPreviewOneOfSeven';
export * from './documentPreviewOneOfSevenAllOf';
export * from './documentPreviewOneOfSevenAllOfType';
+export * from './documentPurgedMetadata';
export * from './documentResponse';
export * from './documentResponseMetadata';
export * from './documentResponseMetadataBranchedFromId';
@@ -465,6 +468,8 @@ export * from './documentResponseMetadataWithContentAllOfTeamTaskId';
export * from './documentResponsePresignedUrl';
export * from './documentStorageServiceApiVersion';
export * from './documentSubType';
+export * from './documentSyncContentUpdatedMetadata';
+export * from './documentSyncContentUpdatedMetadataDocumentVersionId';
export * from './documentTeamShareResponse';
export * from './documentTeamShareResponseTeamId';
export * from './editAnchorRequest';
diff --git a/apps/web/src/lib/service-clients/service-storage/openapi.json b/apps/web/src/lib/service-clients/service-storage/openapi.json
index a44f839c81..5858e9f80f 100644
--- a/apps/web/src/lib/service-clients/service-storage/openapi.json
+++ b/apps/web/src/lib/service-clients/service-storage/openapi.json
@@ -15046,6 +15046,29 @@
"description": "API-visible content lifecycle state derived from current document metadata.",
"enum": ["unknown", "pending", "ready"]
},
+ "DocumentContentUploadedMetadata": {
+ "type": "object",
+ "description": "Metadata for [`DocumentTopicEvent::ContentUploaded`].",
+ "required": ["document_id", "owner", "file_type"],
+ "properties": {
+ "document_id": {
+ "type": "string",
+ "description": "The id of the document whose stored bytes changed."
+ },
+ "document_version_id": {
+ "type": ["string", "null"],
+ "description": "Version written, or the converted-file marker; `None` for unversioned\nwrites (mirrors `SearchExtractorMessage::document_version_id`)."
+ },
+ "file_type": {
+ "$ref": "#/components/schemas/FileType",
+ "description": "File type of the uploaded object (may differ from the document's own\ntype, e.g. `pdf` for the converted rendition of a docx)."
+ },
+ "owner": {
+ "$ref": "#/components/schemas/MacroUserIdStr",
+ "description": "The owner of the document (used by the extractor to resolve S3 keys)."
+ }
+ }
+ },
"DocumentCopiedMetadata": {
"type": "object",
"description": "Metadata for [`DocumentTopicEvent::Copied`].",
@@ -15554,6 +15577,17 @@
],
"description": "The sub type of a document preview with associated properties.\nTask-related properties are encoded within the variant to ensure valid states."
},
+ "DocumentPurgedMetadata": {
+ "type": "object",
+ "description": "Metadata for [`DocumentTopicEvent::Purged`].",
+ "required": ["document_id"],
+ "properties": {
+ "document_id": {
+ "type": "string",
+ "description": "The id of the hard-deleted document."
+ }
+ }
+ },
"DocumentResponse": {
"type": "object",
"description": "Document response with content lifecycle metadata.",
@@ -15688,6 +15722,25 @@
"description": "The document sub type enum represents all values of document sub types.\nThese values should match the `document_sub_type_value` table in macrodb.",
"enum": ["task", "snippet"]
},
+ "DocumentSyncContentUpdatedMetadata": {
+ "type": "object",
+ "description": "Metadata for [`DocumentTopicEvent::SyncContentUpdated`].",
+ "required": ["document_id", "file_type"],
+ "properties": {
+ "document_id": {
+ "type": "string",
+ "description": "The id of the live-collab document whose content changed."
+ },
+ "document_version_id": {
+ "type": ["string", "null"],
+ "description": "Version marker for the sync snapshot, when the caller supplies one."
+ },
+ "file_type": {
+ "$ref": "#/components/schemas/FileType",
+ "description": "File type of the sync document (markdown today)."
+ }
+ }
+ },
"DocumentTeamShareResponse": {
"type": "object",
"description": "The team-share state of a document. The team is resolved from the document\nowner's team membership.",
@@ -15751,6 +15804,51 @@
}
}
},
+ {
+ "type": "object",
+ "description": "The document's stored bytes were (re)written to S3.",
+ "required": ["metadata", "event_type"],
+ "properties": {
+ "event_type": {
+ "type": "string",
+ "enum": ["document.content_uploaded"]
+ },
+ "metadata": {
+ "$ref": "#/components/schemas/DocumentContentUploadedMetadata",
+ "description": "The document's stored bytes were (re)written to S3."
+ }
+ }
+ },
+ {
+ "type": "object",
+ "description": "A live-collab (sync) document's content changed and should be re-extracted.",
+ "required": ["metadata", "event_type"],
+ "properties": {
+ "event_type": {
+ "type": "string",
+ "enum": ["document.sync_content_updated"]
+ },
+ "metadata": {
+ "$ref": "#/components/schemas/DocumentSyncContentUpdatedMetadata",
+ "description": "A live-collab (sync) document's content changed and should be re-extracted."
+ }
+ }
+ },
+ {
+ "type": "object",
+ "description": "The document row was permanently deleted (hard delete).",
+ "required": ["metadata", "event_type"],
+ "properties": {
+ "event_type": {
+ "type": "string",
+ "enum": ["document.purged"]
+ },
+ "metadata": {
+ "$ref": "#/components/schemas/DocumentPurgedMetadata",
+ "description": "The document row was permanently deleted (hard delete)."
+ }
+ }
+ },
{
"type": "object",
"description": "A document was copied.",
diff --git a/crates/ai_tools/src/build_context.rs b/crates/ai_tools/src/build_context.rs
index 83ac152928..496d5fd75b 100644
--- a/crates/ai_tools/src/build_context.rs
+++ b/crates/ai_tools/src/build_context.rs
@@ -294,10 +294,6 @@ pub async fn build_tool_service_context_from_env(
pool.clone(),
)),
macro_event_broker: macro_event_broker.clone(),
- // AI-agent contexts don't own a search event queue (their sqs_client is
- // configured without one), so no search-index refresh is published from
- // here — matching the properties service built in the same context.
- search_indexer: None,
};
let document_tool_context = DocumentToolContext::new(
diff --git a/crates/document_storage_service_client/src/document.rs b/crates/document_storage_service_client/src/document.rs
index 7753342e6b..452a4dba11 100644
--- a/crates/document_storage_service_client/src/document.rs
+++ b/crates/document_storage_service_client/src/document.rs
@@ -5,7 +5,7 @@ use anyhow::Result;
use model::document::list::ListDocumentsWithAccessResponse;
use model::document::response::{CreateDocumentRequest, CreateDocumentResponse};
use model::document::{
- DocumentBasic,
+ DocumentBasic, FileType,
response::{GetDocumentResponse, LocationResponseData, LocationResponseV3},
};
use model::document_storage_service_internal::{
@@ -27,6 +27,12 @@ pub struct ExportDocumentResponse {
pub presigned_url: String,
}
+#[derive(Serialize)]
+struct ContentUploadedRequest {
+ file_type: FileType,
+ document_version_id: Option<String>,
+}
+
impl DocumentStorageServiceClient {
pub async fn get_document(&self, document_id: String) -> Result<GetDocumentResponse> {
let res = self
@@ -41,6 +47,43 @@ impl DocumentStorageServiceClient {
Ok(doc_data)
}
+ /// Notify the document storage service that document content was uploaded.
+ #[tracing::instrument(skip(self), err)]
+ pub async fn publish_document_content_uploaded(
+ &self,
+ document_id: &str,
+ file_type: FileType,
+ document_version_id: Option<String>,
+ ) -> Result<()> {
+ let response = self
+ .client
+ .post(format!(
+ "{}/internal/documents/{}/content-uploaded",
+ self.url, document_id
+ ))
+ .json(&ContentUploadedRequest {
+ file_type,
+ document_version_id,
+ })
+ .send()
+ .await?;
+
+ let status = response.status();
+ if !status.is_success() {
+ let body = response
+ .text()
+ .await
+ .unwrap_or_else(|_| "<failed to read body>".to_string());
+ anyhow::bail!(
+ "publishing document content upload failed: {} - {}",
+ status,
+ body
+ );
+ }
+
+ Ok(())
+ }
+
pub async fn get_document_basic(&self, document_id: &str) -> Result<Option<DocumentBasic>> {
let res = self
.client
diff --git a/crates/documents/examples/documents_consumer.rs b/crates/documents/examples/documents_consumer.rs
index 2b9ad05229..738682bacc 100644
--- a/crates/documents/examples/documents_consumer.rs
+++ b/crates/documents/examples/documents_consumer.rs
@@ -69,6 +69,9 @@ fn event_type(event: &DocumentTopicEvent) -> &'static str {
DocumentTopicEvent::Created(_) => "document.created",
DocumentTopicEvent::Updated(_) => "document.updated",
DocumentTopicEvent::Deleted(_) => "document.deleted",
+ DocumentTopicEvent::ContentUploaded(_) => "document.content_uploaded",
+ DocumentTopicEvent::SyncContentUpdated(_) => "document.sync_content_updated",
+ DocumentTopicEvent::Purged(_) => "document.purged",
DocumentTopicEvent::Copied(_) => "document.copied",
DocumentTopicEvent::Interaction(_) => "document.interaction",
}
diff --git a/crates/documents/src/domain.rs b/crates/documents/src/domain.rs
index d7c033def6..61b3f6727e 100644
--- a/crates/documents/src/domain.rs
+++ b/crates/documents/src/domain.rs
@@ -3,6 +3,7 @@
pub mod branch_name;
pub mod content;
/// Unified entity-mutation capability impls.
+#[cfg(feature = "service")]
pub mod entity_mutation;
pub mod events;
#[cfg(feature = "ports")]
diff --git a/crates/documents/src/domain/events.rs b/crates/documents/src/domain/events.rs
index f3cb25141b..9763a35998 100644
--- a/crates/documents/src/domain/events.rs
+++ b/crates/documents/src/domain/events.rs
@@ -4,6 +4,9 @@
//! per-variant metadata structs, a [`TopicEvent`] enum tagged by `event_type`,
//! and a [`MacroEvent`] wrapper keyed by document id.
+#[cfg(test)]
+mod test;
+
use chrono::{DateTime, Utc};
use document_sub_type::DocumentSubType;
use macro_event_broker::{Event, MacroEvent, TopicEvent};
@@ -71,6 +74,42 @@ pub struct DocumentDeletedMetadata {
pub project_id: Option<String>,
}
+/// Metadata for [`DocumentTopicEvent::ContentUploaded`].
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]
+pub struct DocumentContentUploadedMetadata {
+ /// The id of the document whose stored bytes changed.
+ pub document_id: String,
+ /// The owner of the document (used by the extractor to resolve S3 keys).
+ pub owner: MacroUserIdStr<'static>,
+ /// File type of the uploaded object (may differ from the document's own
+ /// type, e.g. `pdf` for the converted rendition of a docx).
+ pub file_type: FileType,
+ /// Version written, or the converted-file marker; `None` for unversioned
+ /// writes (mirrors `SearchExtractorMessage::document_version_id`).
+ pub document_version_id: Option<String>,
+}
+
+/// Metadata for [`DocumentTopicEvent::SyncContentUpdated`].
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]
+pub struct DocumentSyncContentUpdatedMetadata {
+ /// The id of the live-collab document whose content changed.
+ pub document_id: String,
+ /// File type of the sync document (markdown today).
+ pub file_type: FileType,
+ /// Version marker for the sync snapshot, when the caller supplies one.
+ pub document_version_id: Option<String>,
+}
+
+/// Metadata for [`DocumentTopicEvent::Purged`].
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]
+pub struct DocumentPurgedMetadata {
+ /// The id of the hard-deleted document.
+ pub document_id: String,
+}
+
/// Why a document interaction was reported.
#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
#[cfg_attr(feature = "schema", derive(utoipa::ToSchema))]
@@ -130,6 +169,15 @@ pub enum DocumentTopicEvent {
/// A document was soft-deleted.
#[serde(rename = "document.deleted")]
Deleted(DocumentDeletedMetadata),
+ /// The document's stored bytes were (re)written to S3.
+ #[serde(rename = "document.content_uploaded")]
+ ContentUploaded(DocumentContentUploadedMetadata),
+ /// A live-collab (sync) document's content changed and should be re-extracted.
+ #[serde(rename = "document.sync_content_updated")]
+ SyncContentUpdated(DocumentSyncContentUpdatedMetadata),
+ /// The document row was permanently deleted (hard delete).
+ #[serde(rename = "document.purged")]
+ Purged(DocumentPurgedMetadata),
/// A document was copied.
#[serde(rename = "document.copied")]
Copied(DocumentCopiedMetadata),
@@ -166,6 +214,27 @@ impl DocumentMacroEvent {
Self::new(key, DocumentTopicEvent::Deleted(metadata))
}
+ /// Build a content-uploaded event keyed by the document id.
+ pub fn content_uploaded(
+ key: impl Into<String>,
+ metadata: DocumentContentUploadedMetadata,
+ ) -> Self {
+ Self::new(key, DocumentTopicEvent::ContentUploaded(metadata))
+ }
+
+ /// Build a sync-content-updated event keyed by the document id.
+ pub fn sync_content_updated(
+ key: impl Into<String>,
+ metadata: DocumentSyncContentUpdatedMetadata,
+ ) -> Self {
+ Self::new(key, DocumentTopicEvent::SyncContentUpdated(metadata))
+ }
+
+ /// Build a purged event keyed by the document id.
+ pub fn purged(key: impl Into<String>, metadata: DocumentPurgedMetadata) -> Self {
+ Self::new(key, DocumentTopicEvent::Purged(metadata))
+ }
+
/// Build a copied event keyed by the new document id.
pub fn copied(key: impl Into<String>, metadata: DocumentCopiedMetadata) -> Self {
Self::new(key, DocumentTopicEvent::Copied(metadata))
diff --git a/crates/documents/src/domain/events/test.rs b/crates/documents/src/domain/events/test.rs
new file mode 100644
index 0000000000..c01f5694be
--- /dev/null
+++ b/crates/documents/src/domain/events/test.rs
@@ -0,0 +1,215 @@
+use macro_event_broker::{Event, MacroEvent as _, TopicEvent};
+use macro_user_id::user_id::MacroUserIdStr;
+use model::document::FileType;
+use serde_json::{Value, json};
+use uuid::Uuid;
+
+use super::{
+ DocumentContentUploadedMetadata, DocumentInteractionMetadata, DocumentMacroEvent,
+ DocumentPurgedMetadata, DocumentSyncContentUpdatedMetadata, DocumentTopicEvent,
+ InteractionReason,
+};
+
+const DOCUMENT_ID: &str = "11111111-1111-1111-1111-111111111111";
+
+fn owner() -> MacroUserIdStr<'static> {
+ MacroUserIdStr::try_from("macro|[email protected]".to_string()).expect("valid user id")
+}
+
+fn assert_wire_round_trip(event: Event<DocumentTopicEvent>, expected: Value) {
+ assert_eq!(
+ serde_json::to_value(&event).expect("event serializes"),
+ expected
+ );
+
+ let decoded: Event<DocumentTopicEvent> =
+ serde_json::from_value(expected).expect("event deserializes");
+ assert_eq!(decoded, event);
+}
+
+#[test]
+fn content_uploaded_serializes_to_the_exact_envelope() {
+ let event = Event::with_event_id(
+ Uuid::from_u128(1),
+ DocumentTopicEvent::ContentUploaded(DocumentContentUploadedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ owner: owner(),
+ file_type: FileType::Pdf,
+ document_version_id: Some("convert".to_string()),
+ }),
+ );
+
+ assert_wire_round_trip(
+ event,
+ json!({
+ "event_id": "00000000-0000-0000-0000-000000000001",
+ "schema_version": 1,
+ "event_type": "document.content_uploaded",
+ "metadata": {
+ "document_id": DOCUMENT_ID,
+ "owner": "macro|[email protected]",
+ "file_type": "pdf",
+ "document_version_id": "convert",
+ },
+ }),
+ );
+}
+
+#[test]
+fn sync_content_updated_serializes_to_the_exact_envelope() {
+ let event = Event::with_event_id(
+ Uuid::from_u128(2),
+ DocumentTopicEvent::SyncContentUpdated(DocumentSyncContentUpdatedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ file_type: FileType::Md,
+ document_version_id: None,
+ }),
+ );
+
+ assert_wire_round_trip(
+ event,
+ json!({
+ "event_id": "00000000-0000-0000-0000-000000000002",
+ "schema_version": 1,
+ "event_type": "document.sync_content_updated",
+ "metadata": {
+ "document_id": DOCUMENT_ID,
+ "file_type": "md",
+ "document_version_id": null,
+ },
+ }),
+ );
+}
+
+#[test]
+fn purged_serializes_to_the_exact_envelope() {
+ let event = Event::with_event_id(
+ Uuid::from_u128(3),
+ DocumentTopicEvent::Purged(DocumentPurgedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ }),
+ );
+
+ assert_wire_round_trip(
+ event,
+ json!({
+ "event_id": "00000000-0000-0000-0000-000000000003",
+ "schema_version": 1,
+ "event_type": "document.purged",
+ "metadata": {
+ "document_id": DOCUMENT_ID,
+ },
+ }),
+ );
+}
+
+#[test]
+fn optional_document_versions_support_present_and_absent_values() {
+ let events = [
+ DocumentTopicEvent::ContentUploaded(DocumentContentUploadedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ owner: owner(),
+ file_type: FileType::Pdf,
+ document_version_id: None,
+ }),
+ DocumentTopicEvent::SyncContentUpdated(DocumentSyncContentUpdatedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ file_type: FileType::Md,
+ document_version_id: Some("snapshot-7".to_string()),
+ }),
+ ];
+
+ let content_uploaded = serde_json::to_value(&events[0]).expect("event serializes");
+ assert_eq!(
+ content_uploaded["metadata"]["document_version_id"],
+ Value::Null
+ );
+
+ let sync_content_updated = serde_json::to_value(&events[1]).expect("event serializes");
+ assert_eq!(
+ sync_content_updated["metadata"]["document_version_id"],
+ "snapshot-7"
+ );
+
+ for event in events {
+ let payload = serde_json::to_vec(&event).expect("event serializes");
+ let decoded: DocumentTopicEvent =
+ serde_json::from_slice(&payload).expect("event deserializes");
+ assert_eq!(decoded, event);
+ }
+}
+
+#[test]
+fn search_event_constructors_use_the_document_key_and_schema_v1() {
+ assert_eq!(<DocumentTopicEvent as TopicEvent>::SCHEMA_VERSION, 1);
+
+ let content_metadata = DocumentContentUploadedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ owner: owner(),
+ file_type: FileType::Pdf,
+ document_version_id: Some("version-1".to_string()),
+ };
+ let content_uploaded =
+ DocumentMacroEvent::content_uploaded(DOCUMENT_ID, content_metadata.clone());
+ assert_eq!(content_uploaded.key(), DOCUMENT_ID);
+ assert_eq!(content_uploaded.event().schema_version, 1);
+ assert_eq!(
+ &content_uploaded.event().event,
+ &DocumentTopicEvent::ContentUploaded(content_metadata)
+ );
+
+ let sync_metadata = DocumentSyncContentUpdatedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ file_type: FileType::Md,
+ document_version_id: None,
+ };
+ let sync_content_updated =
+ DocumentMacroEvent::sync_content_updated(DOCUMENT_ID, sync_metadata.clone());
+ assert_eq!(sync_content_updated.key(), DOCUMENT_ID);
+ assert_eq!(sync_content_updated.event().schema_version, 1);
+ assert_eq!(
+ &sync_content_updated.event().event,
+ &DocumentTopicEvent::SyncContentUpdated(sync_metadata)
+ );
+
+ let purged_metadata = DocumentPurgedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ };
+ let purged = DocumentMacroEvent::purged(DOCUMENT_ID, purged_metadata.clone());
+ assert_eq!(purged.key(), DOCUMENT_ID);
+ assert_eq!(purged.event().schema_version, 1);
+ assert_eq!(
+ &purged.event().event,
+ &DocumentTopicEvent::Purged(purged_metadata)
+ );
+}
+
+#[test]
+fn existing_v1_document_events_still_decode() {
+ let payload = json!({
+ "event_id": "00000000-0000-0000-0000-000000000004",
+ "schema_version": 1,
+ "event_type": "document.interaction",
+ "metadata": {
+ "document_id": DOCUMENT_ID,
+ "reason": "edited",
+ },
+ });
+
+ let decoded = DocumentMacroEvent::decode(
+ DOCUMENT_ID,
+ &serde_json::to_vec(&payload).expect("payload serializes"),
+ )
+ .expect("existing v1 event decodes");
+
+ assert_eq!(decoded.key(), DOCUMENT_ID);
+ assert_eq!(decoded.event().schema_version, 1);
+ assert_eq!(decoded.event().event_id, Uuid::from_u128(4));
+ assert_eq!(
+ &decoded.event().event,
+ &DocumentTopicEvent::Interaction(DocumentInteractionMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ reason: InteractionReason::Edited,
+ })
+ );
+}
diff --git a/crates/documents/src/domain/ports/mod.rs b/crates/documents/src/domain/ports/mod.rs
index 1ef214706a..92c1743506 100644
--- a/crates/documents/src/domain/ports/mod.rs
+++ b/crates/documents/src/domain/ports/mod.rs
@@ -13,7 +13,7 @@ use entity_access::domain::models::{
EditAccessLevel, EntityAccessReceipt, MemberTeamRole, OwnerAccessLevel, ViewAccessLevel,
};
use macro_user_id::user_id::MacroUserIdStr;
-use model::document::{ContentType, DocumentBasic, DocumentMetadata};
+use model::document::{ContentType, DocumentBasic, DocumentMetadata, FileType};
use super::content::DocumentContent;
use super::events::InteractionReason;
@@ -334,19 +334,15 @@ pub trait TaskPropertiesPort: Send + Sync + 'static {
) -> impl Future<Output = anyhow::Result<()>> + Send;
}
-/// Port for refreshing a document's denormalized name in the search index.
-///
-/// A rename only changes the `document_name` on the parent doc, and nothing
-/// else re-indexes parent-only file types, so the rename must publish a
-/// targeted metadata refresh. Best-effort — callers log and continue on error.
-/// Boxed future so the port stays object-safe and can be held as
-/// `Arc<dyn DocumentSearchIndexer>` without adding a service generic.
-pub trait DocumentSearchIndexer: Send + Sync + std::fmt::Debug {
- /// Enqueue a refresh of the document's indexed name.
- fn enqueue_name_update(
- &self,
- document_id: String,
- ) -> std::pin::Pin<Box<dyn Future<Output = anyhow::Result<()>> + Send>>;
+/// Use case for relaying document content-upload events.
+pub trait DocumentContentEventService: Send + Sync + 'static {
+ /// Load the document owner and publish a content-uploaded event.
+ fn publish_content_uploaded(
+ &self,
+ document_id: &str,
+ file_type: FileType,
+ document_version_id: Option<String>,
+ ) -> impl Future<Output = Result<(), DocumentError>> + Send;
}
/// Service interface for document operations.
diff --git a/crates/documents/src/domain/service.rs b/crates/documents/src/domain/service.rs
index 531a2600c3..e7b3ea3af9 100644
--- a/crates/documents/src/domain/service.rs
+++ b/crates/documents/src/domain/service.rs
@@ -9,7 +9,6 @@ use models_properties::EntityReference;
use models_properties::api::SetPropertyValue;
use std::borrow::Cow;
use std::str::FromStr;
-use std::sync::Arc;
use std::time::{SystemTime, UNIX_EPOCH};
use unicode_segmentation::UnicodeSegmentation;
@@ -42,8 +41,9 @@ use crate::domain::models::{
use super::branch_name::{build_task_branch_name, user_branch_prefix};
use super::content::{DocumentContent, DocumentContentLocation, DocumentContentState};
use super::events::{
- DocumentCopiedMetadata, DocumentCreatedMetadata, DocumentDeletedMetadata,
- DocumentInteractionMetadata, DocumentMacroEvent, DocumentUpdatedMetadata, InteractionReason,
+ DocumentContentUploadedMetadata, DocumentCopiedMetadata, DocumentCreatedMetadata,
+ DocumentDeletedMetadata, DocumentInteractionMetadata, DocumentMacroEvent,
+ DocumentUpdatedMetadata, InteractionReason,
};
use super::models::{
CloudFrontConfig, CommentThread, CopyDocumentRepoArgs, CreateDocumentRepoArgs,
@@ -54,7 +54,7 @@ use super::models::{
#[cfg(feature = "document_create")]
use super::ports::create::DocumentCreationService;
use super::ports::{
- DocumentRepo, DocumentSearchIndexer, DocumentService, PresignedUploadUrlPort,
+ DocumentContentEventService, DocumentRepo, DocumentService, PresignedUploadUrlPort,
TaskPropertiesPort,
};
use super::response::{
@@ -90,9 +90,6 @@ pub struct DocumentServiceImpl<
pub foreign_entity_service: F,
/// Macro event broker for publishing document lifecycle events
pub macro_event_broker: B,
- /// Optional publisher that refreshes a document's denormalized name in the
- /// search index after a rename.
- pub search_indexer: Option<Arc<dyn DocumentSearchIndexer>>,
}
fn ready_content_for_file_type(file_type: Option<FileType>) -> DocumentContent {
@@ -159,6 +156,17 @@ fn invalid_team_task_slug() -> DocumentError {
DocumentError::BadRequest("invalid team task slug".to_string())
}
+fn map_basic_document_error(document_id: &str, error: anyhow::Error) -> DocumentError {
+ if error
+ .to_string()
+ .contains("no rows returned by a query that expected to return at least one row")
+ {
+ DocumentError::NotFound(document_id.to_string())
+ } else {
+ DocumentError::Internal(error)
+ }
+}
+
fn team_task_number_from_slug(slug: &str) -> Result<i32, DocumentError> {
let (prefix, number) = slug.rsplit_once('-').ok_or_else(invalid_team_task_slug)?;
@@ -266,30 +274,6 @@ impl<
entity_access_management_service,
foreign_entity_service,
macro_event_broker,
- search_indexer: None,
- }
- }
-
- /// Attach a search-index publisher so renames refresh the denormalized
- /// name in the search index. Builder-style so existing constructions are
- /// unaffected.
- pub fn with_search_indexer(mut self, search_indexer: Arc<dyn DocumentSearchIndexer>) -> Self {
- self.search_indexer = Some(search_indexer);
- self
- }
-
- /// Best-effort publish of a search-index name refresh after a rename. Logs
- /// and continues on failure so a missed refresh never fails the edit
- /// itself; a full re-index would eventually correct the name anyway.
- async fn enqueue_name_update(&self, document_id: &str) {
- let Some(search_indexer) = self.search_indexer.as_ref() else {
- return;
- };
- if let Err(error) = search_indexer
- .enqueue_name_update(document_id.to_string())
- .await
- {
- tracing::warn!(error = ?error, document_id = %document_id, "failed to enqueue search name update for rename");
}
}
@@ -609,6 +593,44 @@ impl<
}
}
+impl<
+ R: DocumentRepo,
+ U: PresignedUploadUrlPort,
+ T: TaskPropertiesPort,
+ C: ConnectionService,
+ Eam: EntityAccessManagementService,
+ F: ForeignEntityService,
+ B: MacroEventBroker,
+> DocumentContentEventService for DocumentServiceImpl<R, U, T, C, Eam, F, B>
+{
+ #[tracing::instrument(err, skip(self))]
+ async fn publish_content_uploaded(
+ &self,
+ document_id: &str,
+ file_type: FileType,
+ document_version_id: Option<String>,
+ ) -> Result<(), DocumentError> {
+ let document = self
+ .repo
+ .get_basic_document(document_id)
+ .await
+ .map_err(|error| map_basic_document_error(document_id, error.into()))?;
+
+ self.macro_event_broker
+ .send_event(&DocumentMacroEvent::content_uploaded(
+ document_id,
+ DocumentContentUploadedMetadata {
+ document_id: document_id.to_string(),
+ owner: document.owner,
+ file_type,
+ document_version_id,
+ },
+ ))
+ .map(|_| ())
+ .map_err(|error| DocumentError::Internal(error.into()))
+ }
+}
+
impl<
R: DocumentRepo,
U: PresignedUploadUrlPort,
@@ -838,16 +860,7 @@ impl<
self.repo
.get_basic_document(document_id)
.await
- .map_err(|e| {
- let err: anyhow::Error = e.into();
- if err.to_string().contains(
- "no rows returned by a query that expected to return at least one row",
- ) {
- DocumentError::NotFound(document_id.to_string())
- } else {
- DocumentError::Internal(err)
- }
- })
+ .map_err(|error| map_basic_document_error(document_id, error.into()))
}
async fn get_document_text(
@@ -1243,14 +1256,6 @@ impl<
.await
.map_err(|e| DocumentError::Internal(e.into()))?;
- // A rename changes the denormalized `document_name` on the parent doc in
- // the search index; nothing else re-indexes parent-only file types, so
- // publish a targeted refresh once the new name is committed.
- if document_name.is_some() {
- self.enqueue_name_update(&entity_access_receipt.entity().entity_id)
- .await;
- }
-
// Update project modified timestamps. args.project_id of None means "no change",
// so only move the document out of its old project when a different project (or
// "" for no project) was explicitly requested.
diff --git a/crates/documents/src/domain/service/tests.rs b/crates/documents/src/domain/service/tests.rs
index ad7cfaac59..4481f36536 100644
--- a/crates/documents/src/domain/service/tests.rs
+++ b/crates/documents/src/domain/service/tests.rs
@@ -5,11 +5,11 @@ use foreign_entity::domain::models::{
use foreign_entity::domain::ports::{ForeignEntityListQuery, ForeignEntityService};
use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker};
use macro_user_id::cowlike::CowLike;
-use model::document::DocumentMetadata;
+use model::document::{DocumentMetadata, FileType};
use std::sync::{Arc, Mutex};
use crate::domain::models::GithubPullRequest;
-use crate::domain::ports::MockDocumentRepo;
+use crate::domain::ports::{DocumentContentEventService, MockDocumentRepo};
use super::*;
@@ -343,9 +343,17 @@ struct PublishedEvent {
#[derive(Clone, Default)]
struct TestEventBroker {
published: Arc<Mutex<Vec<PublishedEvent>>>,
+ fail_send: bool,
}
impl TestEventBroker {
+ fn failing() -> Self {
+ Self {
+ fail_send: true,
+ ..Self::default()
+ }
+ }
+
fn published(&self) -> Arc<Mutex<Vec<PublishedEvent>>> {
Arc::clone(&self.published)
}
@@ -356,6 +364,10 @@ impl MacroEventBroker for TestEventBroker {
&self,
event: &E,
) -> Result<tokio::task::JoinHandle<Result<(), EventBrokerError>>, EventBrokerError> {
+ if self.fail_send {
+ return Err(EventBrokerError::Publish("test broker failure".to_string()));
+ }
+
self.published.lock().unwrap().push(PublishedEvent {
topic: event.topic(),
key: event.key().to_string(),
@@ -365,24 +377,6 @@ impl MacroEventBroker for TestEventBroker {
}
}
-#[derive(Clone, Debug, Default)]
-struct TestDocumentSearchIndexer {
- enqueued: Arc<Mutex<Vec<String>>>,
-}
-
-impl crate::domain::ports::DocumentSearchIndexer for TestDocumentSearchIndexer {
- fn enqueue_name_update(
- &self,
- document_id: String,
- ) -> std::pin::Pin<Box<dyn std::future::Future<Output = anyhow::Result<()>> + Send>> {
- let enqueued = Arc::clone(&self.enqueued);
- Box::pin(async move {
- enqueued.lock().unwrap().push(document_id);
- Ok(())
- })
- }
-}
-
type TestDocumentService = DocumentServiceImpl<
MockDocumentRepo,
TestUploadUrlPort,
@@ -449,34 +443,17 @@ fn make_test_service_with_entity_access(
(service, entity_access)
}
-/// Build a test service along with a handle to its recording search indexer.
-fn make_test_service_with_search_indexer(
+/// Build a test service along with a handle to its recording event broker.
+fn make_test_service_with_event_broker(
repo: MockDocumentRepo,
-) -> (TestDocumentService, TestDocumentSearchIndexer) {
- let search_indexer = TestDocumentSearchIndexer::default();
- let service = DocumentServiceImpl::new(
- repo,
- test_cloudfront_config(),
- sync_service_client::SyncServiceClient::new(
- "test-sync-key".to_string(),
- "http://sync-service.test".to_string(),
- ),
- TestUploadUrlPort,
- TestTaskPropertiesPort,
- TestConnectionService,
- TestEntityAccessManagementService::default(),
- TestForeignEntityService::default(),
- TestEventBroker::default(),
- )
- .with_search_indexer(Arc::new(search_indexer.clone()));
- (service, search_indexer)
+) -> (TestDocumentService, TestEventBroker) {
+ make_test_service_with_configured_event_broker(repo, TestEventBroker::default())
}
-/// Build a test service along with a handle to its recording event broker.
-fn make_test_service_with_event_broker(
+fn make_test_service_with_configured_event_broker(
repo: MockDocumentRepo,
+ event_broker: TestEventBroker,
) -> (TestDocumentService, TestEventBroker) {
- let event_broker = TestEventBroker::default();
let service = DocumentServiceImpl::new(
repo,
test_cloudfront_config(),
@@ -1537,6 +1514,92 @@ fn edit_receipt(document_id: &str) -> EntityAccessReceipt<EditAccessLevel> {
)
}
+#[tokio::test]
+async fn content_uploaded_publishes_document_event_with_owner_and_version() {
+ let mut repo = make_mock_repo();
+ repo.expect_get_basic_document()
+ .withf(|document_id| document_id == "doc-1")
+ .return_once(|_| Box::pin(std::future::ready(Ok(task_document_context("doc-1")))));
+
+ let (service, event_broker) = make_test_service_with_event_broker(repo);
+
+ service
+ .publish_content_uploaded("doc-1", FileType::Pdf, Some("convert".to_string()))
+ .await
+ .unwrap();
+
+ let published = event_broker.published();
+ let published = published.lock().unwrap();
+ assert_eq!(published.len(), 1);
+ let event = &published[0];
+ assert_eq!(event.topic, "macro.documents");
+ assert_eq!(event.key, "doc-1");
+ assert_eq!(event.payload["event_type"], "document.content_uploaded");
+ assert_eq!(event.payload["metadata"]["document_id"], "doc-1");
+ assert_eq!(event.payload["metadata"]["owner"], "macro|[email protected]");
+ assert_eq!(event.payload["metadata"]["file_type"], "pdf");
+ assert_eq!(event.payload["metadata"]["document_version_id"], "convert");
+}
+
+#[tokio::test]
+async fn content_uploaded_preserves_an_absent_version() {
+ let mut repo = make_mock_repo();
+ repo.expect_get_basic_document()
+ .return_once(|_| Box::pin(std::future::ready(Ok(task_document_context("doc-1")))));
+
+ let (service, event_broker) = make_test_service_with_event_broker(repo);
+
+ service
+ .publish_content_uploaded("doc-1", FileType::Pdf, None)
+ .await
+ .unwrap();
+
+ let published = event_broker.published();
+ let published = published.lock().unwrap();
+ assert_eq!(
+ published[0].payload["metadata"]["document_version_id"],
+ serde_json::Value::Null
+ );
+}
+
+#[tokio::test]
+async fn content_uploaded_maps_a_missing_document_to_not_found() {
+ let mut repo = make_mock_repo();
+ repo.expect_get_basic_document().return_once(|_| {
+ Box::pin(std::future::ready(Err(anyhow!(
+ "no rows returned by a query that expected to return at least one row"
+ ))))
+ });
+
+ let (service, event_broker) = make_test_service_with_event_broker(repo);
+ let result = service
+ .publish_content_uploaded("missing-doc", FileType::Pdf, None)
+ .await;
+
+ assert!(matches!(
+ result,
+ Err(DocumentError::NotFound(document_id)) if document_id == "missing-doc"
+ ));
+ assert!(event_broker.published().lock().unwrap().is_empty());
+}
+
+#[tokio::test]
+async fn content_uploaded_maps_an_immediate_broker_failure_to_internal() {
+ let mut repo = make_mock_repo();
+ repo.expect_get_basic_document()
+ .return_once(|_| Box::pin(std::future::ready(Ok(task_document_context("doc-1")))));
+ let event_broker = TestEventBroker::failing();
+ let (service, event_broker) =
+ make_test_service_with_configured_event_broker(repo, event_broker);
+
+ let result = service
+ .publish_content_uploaded("doc-1", FileType::Pdf, None)
+ .await;
+
+ assert!(matches!(result, Err(DocumentError::Internal(_))));
+ assert!(event_broker.published().lock().unwrap().is_empty());
+}
+
#[tokio::test]
async fn test_delete_document_publishes_document_deleted_event() {
let mut repo = make_mock_repo();
@@ -1675,65 +1738,6 @@ async fn test_edit_document_rename_only_keeps_project_access() {
assert!(entity_access.added_to_projects.lock().unwrap().is_empty());
}
-#[tokio::test]
-async fn test_edit_document_rename_enqueues_search_name_update() {
- let mut repo = make_mock_repo();
- repo.expect_edit_document()
- .withf(|args| args.document_id == "doc-1")
- .returning(|_| Box::pin(std::future::ready(Ok(()))));
-
- let (service, search_indexer) = make_test_service_with_search_indexer(repo);
-
- service
- .edit_document(
- edit_receipt("doc-1"),
- task_document_context("doc-1"),
- EditDocumentServiceArgs {
- document_name: Some("New name".to_string()),
- project_id: None,
- share_permission: None,
- file_type: None,
- },
- )
- .await
- .unwrap();
-
- assert_eq!(
- *search_indexer.enqueued.lock().unwrap(),
- vec!["doc-1".to_string()]
- );
-}
-
-#[tokio::test]
-async fn test_edit_document_without_rename_does_not_enqueue_search_name_update() {
- let document_id = uuid::Uuid::new_v4().to_string();
- let new_project_id = uuid::Uuid::new_v4();
-
- let mut repo = make_mock_repo();
- repo.expect_edit_document()
- .returning(|_| Box::pin(std::future::ready(Ok(()))));
- repo.expect_update_project_modified()
- .returning(|_| Box::pin(std::future::ready(Ok(()))));
-
- let (service, search_indexer) = make_test_service_with_search_indexer(repo);
-
- service
- .edit_document(
- edit_receipt(&document_id),
- task_document_context(&document_id),
- EditDocumentServiceArgs {
- document_name: None,
- project_id: Some(new_project_id.to_string()),
- share_permission: None,
- file_type: None,
- },
- )
- .await
- .unwrap();
-
- assert!(search_indexer.enqueued.lock().unwrap().is_empty());
-}
-
#[tokio::test]
async fn test_edit_document_project_change_moves_project_access() {
let document_id = uuid::Uuid::new_v4().to_string();
diff --git a/crates/documents/src/inbound/axum_router.rs b/crates/documents/src/inbound/axum_router.rs
index 66e023f864..ff7774bbfd 100644
--- a/crates/documents/src/inbound/axum_router.rs
+++ b/crates/documents/src/inbound/axum_router.rs
@@ -16,6 +16,7 @@
#[cfg(test)]
mod tests;
+pub mod content_uploaded;
pub mod copy_document;
pub mod create_document;
#[cfg(feature = "document_create")]
diff --git a/crates/documents/src/inbound/axum_router/content_uploaded.rs b/crates/documents/src/inbound/axum_router/content_uploaded.rs
new file mode 100644
index 0000000000..c0fdb8766b
--- /dev/null
+++ b/crates/documents/src/inbound/axum_router/content_uploaded.rs
@@ -0,0 +1,43 @@
+//! Handler for `POST /internal/documents/{document_id}/content-uploaded`.
+
+use axum::{
+ Json,
+ extract::{Path, State},
+ http::StatusCode,
+};
+use entity_access::domain::ports::EntityAccessService;
+use macro_authorization::{InternalOnly, MacroAuthorizationExtractor, MacroAuthorizationService};
+use model::document::FileType;
+use serde::Deserialize;
+
+use super::{DocumentRouterState, Params};
+use crate::domain::{models::DocumentError, ports::DocumentContentEventService};
+
+/// Request body describing the uploaded document content.
+#[derive(Debug, Deserialize)]
+pub struct ContentUploadedRequest {
+ /// File type of the uploaded object.
+ pub file_type: FileType,
+ /// Uploaded version or converted-file marker, when present.
+ pub document_version_id: Option<String>,
+}
+
+/// Publishes a content-uploaded event for an existing document.
+#[tracing::instrument(err, skip(state, _internal_authorization))]
+pub async fn content_uploaded_handler<
+ T: DocumentContentEventService,
+ Svc: EntityAccessService,
+ Auth: MacroAuthorizationService,
+>(
+ State(state): State<DocumentRouterState<T, Svc, Auth>>,
+ _internal_authorization: MacroAuthorizationExtractor<Auth, InternalOnly>,
+ Path(Params { document_id }): Path<Params>,
+ Json(request): Json<ContentUploadedRequest>,
+) -> Result<StatusCode, DocumentError> {
+ state
+ .service
+ .publish_content_uploaded(&document_id, request.file_type, request.document_version_id)
+ .await?;
+
+ Ok(StatusCode::OK)
+}
diff --git a/crates/documents/src/inbound/axum_router/tests.rs b/crates/documents/src/inbound/axum_router/tests.rs
index 30fc933cf2..13068befe7 100644
--- a/crates/documents/src/inbound/axum_router/tests.rs
+++ b/crates/documents/src/inbound/axum_router/tests.rs
@@ -22,7 +22,7 @@ use macro_authorization::{
};
use macro_user_id::{lowercased::Lowercase, user_id::MacroUserId, user_id::MacroUserIdStr};
use model::{
- document::{DocumentBasic, DocumentMetadata, response::DocumentResponseMetadata},
+ document::{DocumentBasic, DocumentMetadata, FileType, response::DocumentResponseMetadata},
sync_service::SyncServiceVersionID,
};
use model_entity::Entity;
@@ -42,7 +42,7 @@ use task_dedup::{
use tower::ServiceExt;
use uuid::Uuid;
-use super::{DocumentRouterState, documents_router};
+use super::{DocumentRouterState, content_uploaded::content_uploaded_handler, documents_router};
use crate::{
domain::{
content::DocumentContent,
@@ -53,7 +53,7 @@ use crate::{
DocumentTeamShareResponse, EditDocumentServiceArgs, GithubPullRequestsResponse,
LocationQueryParams, TaskBranchName,
},
- ports::{DocumentService, create::DocumentCreationService},
+ ports::{DocumentContentEventService, DocumentService, create::DocumentCreationService},
response::{
CreateDocumentResponseData, DocumentMetadataWithContent, DocumentResponse,
DocumentResponseMetadataWithContent, GetDocumentResponseData, LocationResponseV3,
@@ -89,6 +89,13 @@ struct UploadSnapshotCall {
bytes: Vec<u8>,
}
+#[derive(Clone, Debug, Eq, PartialEq)]
+struct ContentUploadedCall {
+ document_id: String,
+ file_type: FileType,
+ document_version_id: Option<String>,
+}
+
#[derive(Clone, Debug, Eq, PartialEq)]
struct TeamSlugCall {
team_id: String,
@@ -107,6 +114,7 @@ enum TeamSlugResult {
struct FakeDocumentService {
create_calls: Mutex<Vec<CreateDocumentCall>>,
upload_snapshot_calls: Mutex<Vec<UploadSnapshotCall>>,
+ content_uploaded_calls: Mutex<Vec<ContentUploadedCall>>,
internal_get_calls: Mutex<Vec<String>>,
team_slug_calls: Mutex<Vec<TeamSlugCall>>,
team_slug_result: Mutex<Option<TeamSlugResult>>,
@@ -128,6 +136,13 @@ impl FakeDocumentService {
.clone()
}
+ fn content_uploaded_calls(&self) -> Vec<ContentUploadedCall> {
+ self.content_uploaded_calls
+ .lock()
+ .expect("content uploaded calls lock poisoned")
+ .clone()
+ }
+
fn set_team_slug_result(&self, result: TeamSlugResult) {
*self
.team_slug_result
@@ -394,6 +409,25 @@ impl DocumentService for FakeDocumentService {
}
}
+impl DocumentContentEventService for FakeDocumentService {
+ async fn publish_content_uploaded(
+ &self,
+ document_id: &str,
+ file_type: FileType,
+ document_version_id: Option<String>,
+ ) -> Result<(), DocumentError> {
+ self.content_uploaded_calls
+ .lock()
+ .expect("content uploaded calls lock poisoned")
+ .push(ContentUploadedCall {
+ document_id: document_id.to_string(),
+ file_type,
+ document_version_id,
+ });
+ Ok(())
+ }
+}
+
impl DocumentCreationService for FakeDocumentService {
async fn create_document(
&self,
@@ -791,8 +825,19 @@ fn test_router() -> (
FakeDocumentService,
FakeEntityAccessService,
FakeAuthorizationService,
- (),
- >(state);
+ DocumentRouterState<FakeDocumentService, FakeEntityAccessService, FakeAuthorizationService>,
+ >(state.clone())
+ .route(
+ "/{document_id}/content-uploaded",
+ axum::routing::post(
+ content_uploaded_handler::<
+ FakeDocumentService,
+ FakeEntityAccessService,
+ FakeAuthorizationService,
+ >,
+ ),
+ )
+ .with_state(state);
(
router,
@@ -969,6 +1014,56 @@ async fn get_document_by_team_slug_preserves_domain_error_statuses() {
assert!(document_service.internal_get_calls().is_empty());
}
+#[tokio::test]
+async fn content_uploaded_requires_internal_authentication_and_forwards_request() {
+ let (router, document_service, _access_service, authorization_service) = test_router();
+ let body = || {
+ Body::from(
+ serde_json::to_vec(&json!({
+ "file_type": "pdf",
+ "document_version_id": "convert",
+ }))
+ .expect("request should serialize"),
+ )
+ };
+
+ let jwt_request = Request::post("/content-document/content-uploaded")
+ .header("authorization", format!("Bearer {JWT_TOKEN}"))
+ .header("content-type", "application/json")
+ .body(body())
+ .expect("request should build");
+ assert_eq!(
+ send_status(&router, jwt_request).await,
+ StatusCode::FORBIDDEN
+ );
+ assert!(document_service.content_uploaded_calls().is_empty());
+
+ let internal_request = Request::post("/content-document/content-uploaded")
+ .header(INTERNAL_API_KEY_HEADER, STANDARD_INTERNAL_KEY)
+ .header("content-type", "application/json")
+ .body(body())
+ .expect("request should build");
+ assert_eq!(send_status(&router, internal_request).await, StatusCode::OK);
+ assert_eq!(
+ document_service.content_uploaded_calls(),
+ [ContentUploadedCall {
+ document_id: "content-document".to_string(),
+ file_type: FileType::Pdf,
+ document_version_id: Some("convert".to_string()),
+ }]
+ );
+ assert_eq!(
+ authorization_service.calls(),
+ [
+ AuthorizationCall::Jwt(JWT_TOKEN.to_string()),
+ AuthorizationCall::Internal {
+ provided_key: STANDARD_INTERNAL_KEY.to_string(),
+ claims: InternalIdentityClaims::default(),
+ },
+ ]
+ );
+}
+
#[tokio::test]
async fn snapshot_upload_requires_internal_api_key() {
let snapshot = b"snapshot bytes";
diff --git a/crates/projects/Cargo.toml b/crates/projects/Cargo.toml
index d23d35017d..96263f337b 100644
--- a/crates/projects/Cargo.toml
+++ b/crates/projects/Cargo.toml
@@ -20,6 +20,7 @@ outbound = [
"dep:aws-sdk-s3",
"dep:base64",
"dep:document_sub_type",
+ "dep:documents",
"dep:dynamodb_client",
"dep:entity_access_db_utils",
"dep:hex",
@@ -81,16 +82,20 @@ async-recursion = { version = "1.1.1", optional = true }
aws-sdk-s3 = { workspace = true, optional = true }
base64 = { workspace = true, optional = true }
document_sub_type = { path = "../document_sub_type", optional = true }
+documents = { path = "../documents", default-features = false, optional = true }
dynamodb_client = { path = "../dynamodb_client", optional = true }
entity_access_db_utils = { path = "../entity_access_db_utils", optional = true }
hex = { workspace = true, optional = true }
macro_sha_count_client = { path = "../macro_sha_count_client", optional = true }
sqlx = { workspace = true, optional = true }
-sqs_client = { path = "../sqs_client", optional = true }
+sqs_client = { path = "../sqs_client", default-features = false, features = [
+ "document",
+], optional = true }
system_properties = { path = "../system_properties", optional = true }
workspace-hack = { version = "0.1", path = "../workspace-hack" }
[dev-dependencies]
+aws-sdk-sqs = { workspace = true }
dotenvy = { workspace = true }
http-body-util = { workspace = true }
kafka_util = { path = "../kafka_util" }
diff --git a/crates/projects/src/outbound.rs b/crates/projects/src/outbound.rs
index c49dd7e5a1..888315e3ef 100644
--- a/crates/projects/src/outbound.rs
+++ b/crates/projects/src/outbound.rs
@@ -8,7 +8,7 @@ pub mod pg_project_repo;
pub mod s3_upload_urls;
/// Redis content-hash counter adapter.
pub mod sha_counter;
-/// SQS project search-index adapter.
+/// Kafka search-cleanup and SQS document-deletion adapter.
pub mod sqs_search_indexer;
pub use dynamo_bulk_upload::DynamoBulkUploadAdapter;
diff --git a/crates/projects/src/outbound/sqs_search_indexer.rs b/crates/projects/src/outbound/sqs_search_indexer.rs
index 4455625854..cb3016ba92 100644
--- a/crates/projects/src/outbound/sqs_search_indexer.rs
+++ b/crates/projects/src/outbound/sqs_search_indexer.rs
@@ -1,35 +1,41 @@
-//! SQS adapter for document search cleanup and deletion.
+//! Kafka search-cleanup and SQS document-deletion adapter.
+
+#[cfg(test)]
+mod test;
use std::sync::Arc;
-use sqs_client::search::{SearchQueueMessage, document::DocumentId};
+use documents::domain::events::{DocumentMacroEvent, DocumentPurgedMetadata};
+use macro_event_broker::MacroEventBroker;
use crate::domain::ports::ProjectSearchIndexer;
-/// SQS-backed document search cleanup and deletion adapter.
+/// Kafka-backed document search cleanup and SQS-backed document deletion adapter.
#[derive(Clone)]
-pub struct SqsProjectSearchIndexer {
+pub struct SqsProjectSearchIndexer<B: MacroEventBroker> {
sqs: Arc<sqs_client::SQS>,
+ event_broker: B,
}
-impl SqsProjectSearchIndexer {
- /// Create a project search indexer from the shared SQS client.
- pub fn new(sqs: Arc<sqs_client::SQS>) -> Self {
- Self { sqs }
+impl<B: MacroEventBroker> SqsProjectSearchIndexer<B> {
+ /// Create a project search indexer from the shared SQS client and event broker.
+ pub fn new(sqs: Arc<sqs_client::SQS>, event_broker: B) -> Self {
+ Self { sqs, event_broker }
}
}
-impl ProjectSearchIndexer for SqsProjectSearchIndexer {
+impl<B: MacroEventBroker> ProjectSearchIndexer for SqsProjectSearchIndexer<B> {
#[tracing::instrument(skip(self), err)]
async fn remove_documents(&self, document_ids: Vec<String>) -> anyhow::Result<()> {
- let messages = document_ids
- .into_iter()
- .map(|document_id| SearchQueueMessage::RemoveDocument(DocumentId { document_id }))
- .collect();
-
- self.sqs
- .bulk_send_message_to_search_event_queue(messages)
- .await
+ for document_id in document_ids {
+ let event = DocumentMacroEvent::purged(
+ document_id.clone(),
+ DocumentPurgedMetadata { document_id },
+ );
+ drop(self.event_broker.send_event(&event)?);
+ }
+
+ Ok(())
}
#[tracing::instrument(skip(self), err)]
diff --git a/crates/projects/src/outbound/sqs_search_indexer/test.rs b/crates/projects/src/outbound/sqs_search_indexer/test.rs
new file mode 100644
index 0000000000..3c2cd879c6
--- /dev/null
+++ b/crates/projects/src/outbound/sqs_search_indexer/test.rs
@@ -0,0 +1,135 @@
+use std::sync::{Arc, Mutex};
+
+use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker};
+use serde_json::{Value, json};
+use uuid::Uuid;
+
+use super::*;
+
+#[derive(Clone, Debug, PartialEq)]
+struct PublishedEvent {
+ topic: String,
+ key: String,
+ envelope: Value,
+}
+
+#[derive(Clone, Default)]
+struct RecordingEventBroker {
+ state: Arc<Mutex<RecordingState>>,
+ fail_send: bool,
+}
+
+#[derive(Default)]
+struct RecordingState {
+ attempts: usize,
+ published: Vec<PublishedEvent>,
+}
+
+impl RecordingEventBroker {
+ fn failing() -> Self {
+ Self {
+ fail_send: true,
+ ..Self::default()
+ }
+ }
+
+ fn attempts(&self) -> usize {
+ self.state.lock().unwrap().attempts
+ }
+
+ fn published(&self) -> Vec<PublishedEvent> {
+ self.state.lock().unwrap().published.clone()
+ }
+}
+
+impl MacroEventBroker for RecordingEventBroker {
+ fn send_event<E: MacroEvent + ?Sized>(
+ &self,
+ event: &E,
+ ) -> Result<tokio::task::JoinHandle<Result<(), EventBrokerError>>, EventBrokerError> {
+ let mut state = self.state.lock().unwrap();
+ state.attempts += 1;
+
+ if self.fail_send {
+ return Err(EventBrokerError::Publish(
+ "event enqueue rejected".to_string(),
+ ));
+ }
+
+ state.published.push(PublishedEvent {
+ topic: event.topic().to_string(),
+ key: event.key().to_string(),
+ envelope: serde_json::to_value(event.event())?,
+ });
+
+ Ok(tokio::spawn(async { Ok(()) }))
+ }
+}
+
+fn test_sqs_client() -> Arc<sqs_client::SQS> {
+ let config = aws_sdk_sqs::Config::builder()
+ .behavior_version(aws_sdk_sqs::config::BehaviorVersion::latest())
+ .build();
+
+ Arc::new(sqs_client::SQS::new(aws_sdk_sqs::Client::from_conf(config)))
+}
+
+#[tokio::test]
+async fn empty_document_list_publishes_no_events() {
+ let event_broker = RecordingEventBroker::default();
+ let indexer = SqsProjectSearchIndexer::new(test_sqs_client(), event_broker.clone());
+
+ indexer.remove_documents(Vec::new()).await.unwrap();
+
+ assert_eq!(event_broker.attempts(), 0);
+ assert!(event_broker.published().is_empty());
+}
+
+#[tokio::test]
+async fn document_removals_publish_separately_keyed_purged_event_envelopes() {
+ let event_broker = RecordingEventBroker::default();
+ let indexer = SqsProjectSearchIndexer::new(test_sqs_client(), event_broker.clone());
+ let document_ids = vec!["document-one".to_string(), "document-two".to_string()];
+
+ indexer
+ .remove_documents(document_ids.clone())
+ .await
+ .unwrap();
+
+ let published = event_broker.published();
+ assert_eq!(published.len(), document_ids.len());
+
+ for (event, document_id) in published.iter().zip(document_ids) {
+ assert_eq!(event.topic, "macro.documents");
+ assert_eq!(event.key, document_id);
+
+ let event_id = event.envelope["event_id"].clone();
+ assert!(Uuid::parse_str(event_id.as_str().unwrap()).is_ok());
+ assert_eq!(
+ event.envelope,
+ json!({
+ "event_id": event_id,
+ "schema_version": 1,
+ "event_type": "document.purged",
+ "metadata": {
+ "document_id": document_id,
+ },
+ })
+ );
+ }
+}
+
+#[tokio::test]
+async fn document_removals_return_immediate_broker_failures() {
+ let event_broker = RecordingEventBroker::failing();
+ let indexer = SqsProjectSearchIndexer::new(test_sqs_client(), event_broker.clone());
+
+ let error = indexer
+ .remove_documents(vec!["document-one".to_string()])
+ .await
+ .expect_err("immediate broker failure should be returned");
+
+ assert!(format!("{error:#}").contains("event enqueue rejected"));
+ assert_eq!(event_broker.attempts(), 1);
+ assert!(event_broker.published().is_empty());
+}
diff --git a/crates/soup_realtime/src/inbound/kafka_consumer.rs b/crates/soup_realtime/src/inbound/kafka_consumer.rs
index 69996feba6..97254876e8 100644
--- a/crates/soup_realtime/src/inbound/kafka_consumer.rs
+++ b/crates/soup_realtime/src/inbound/kafka_consumer.rs
@@ -122,7 +122,10 @@ fn patches_from_document_event(event: &DocumentTopicEvent) -> Vec<SoupRealtimePa
{
push_unique_update(&mut updates, EntityType::Document, &metadata.document_id);
}
- DocumentTopicEvent::Interaction(_) => {}
+ DocumentTopicEvent::Interaction(_)
+ | DocumentTopicEvent::ContentUploaded(_)
+ | DocumentTopicEvent::SyncContentUpdated(_)
+ | DocumentTopicEvent::Purged(_) => {}
}
updates
}
diff --git a/crates/soup_realtime/src/inbound/kafka_consumer/test.rs b/crates/soup_realtime/src/inbound/kafka_consumer/test.rs
index 7184a1a51d..c44cf68b5e 100644
--- a/crates/soup_realtime/src/inbound/kafka_consumer/test.rs
+++ b/crates/soup_realtime/src/inbound/kafka_consumer/test.rs
@@ -10,7 +10,8 @@ use channels::domain::{
use chat::domain::events::{ChatMessageDeletedMetadata, ChatTopicEvent, ChatUpdatedMetadata};
use chrono::Utc;
use documents::domain::events::{
- DocumentCreatedMetadata, DocumentDeletedMetadata, DocumentInteractionMetadata,
+ DocumentContentUploadedMetadata, DocumentCreatedMetadata, DocumentDeletedMetadata,
+ DocumentInteractionMetadata, DocumentPurgedMetadata, DocumentSyncContentUpdatedMetadata,
DocumentUpdatedMetadata, InteractionReason,
};
use email::domain::events::{
@@ -178,6 +179,30 @@ fn document_edit_interactions_map_to_updated_patches() {
assert!(patches_from_document_event(&first_join).is_empty());
}
+#[test]
+fn search_only_document_events_do_not_emit_patches() {
+ let events = [
+ DocumentTopicEvent::ContentUploaded(DocumentContentUploadedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ owner: user(),
+ file_type: "pdf".parse().expect("valid file type"),
+ document_version_id: Some("convert".to_string()),
+ }),
+ DocumentTopicEvent::SyncContentUpdated(DocumentSyncContentUpdatedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ file_type: "md".parse().expect("valid file type"),
+ document_version_id: None,
+ }),
+ DocumentTopicEvent::Purged(DocumentPurgedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ }),
+ ];
+
+ for event in events {
+ assert!(patches_from_document_event(&event).is_empty());
+ }
+}
+
#[test]
fn project_deletion_maps_cascade_entities_to_deleted_patches() {
let project_id = Uuid::now_v7().to_string();
diff --git a/crates/sqs_client/src/search/document.rs b/crates/sqs_client/src/search/document.rs
index 8c6244ba6e..e827705594 100644
--- a/crates/sqs_client/src/search/document.rs
+++ b/crates/sqs_client/src/search/document.rs
@@ -2,11 +2,6 @@
use model::document::{BackfillSearchDocumentInformation, FileType};
use s3_key::CONVERTED_DOCUMENT_FILE_NAME;
-#[derive(serde::Serialize, serde::Deserialize, PartialEq, Eq, Debug)]
-pub struct DocumentId {
- pub document_id: String,
-}
-
/// A request to refresh only the indexed properties of a document (e.g. after
/// a status / assignee / tag change), without re-extracting its content.
#[derive(serde::Serialize, serde::Deserialize, PartialEq, Eq, Debug)]
diff --git a/crates/sqs_client/src/search/mod.rs b/crates/sqs_client/src/search/mod.rs
index 44fd2d6b78..59ceb16699 100644
--- a/crates/sqs_client/src/search/mod.rs
+++ b/crates/sqs_client/src/search/mod.rs
@@ -4,7 +4,7 @@ use crate::{
call::{CallRecordMessage, RemoveCallRecord},
channel::ChannelMessageUpdate,
chat::ChatMessage,
- document::{DocumentId, DocumentPropertiesUpdate, SearchExtractorMessage},
+ document::{DocumentPropertiesUpdate, SearchExtractorMessage},
email::{EmailLinkMessage, EmailMessage, EmailThreadBatchMessage, EmailThreadMessage},
project::UpsertProject,
},
@@ -70,10 +70,8 @@ pub enum Operation {
pub enum SearchQueueMessage {
// Document
ExtractDocumentText(SearchExtractorMessage),
- RemoveDocument(DocumentId),
ExtractSync(SearchExtractorMessage),
UpdateDocumentProperties(DocumentPropertiesUpdate),
- UpdateDocumentName(DocumentId),
// Chat
/// SQS backfill work-queue contract for reconciling a chat message.
ChatMessage(ChatMessage),
@@ -99,10 +97,8 @@ impl PrimaryId for SearchQueueMessage {
fn id(&self) -> String {
match self {
SearchQueueMessage::ExtractDocumentText(message) => message.document_id.clone(),
- SearchQueueMessage::RemoveDocument(message) => message.document_id.clone(),
SearchQueueMessage::ExtractSync(message) => message.document_id.clone(),
SearchQueueMessage::UpdateDocumentProperties(message) => message.document_id.clone(),
- SearchQueueMessage::UpdateDocumentName(message) => message.document_id.clone(),
// The message id keeps entries unique within an SQS batch.
SearchQueueMessage::ChatMessage(message) => message.message_id.clone(),
SearchQueueMessage::ExtractEmailMessage(message)
@@ -131,10 +127,8 @@ impl SearchQueueMessage {
match self {
// Document
SearchQueueMessage::ExtractDocumentText(_) => Operation::ExtractText,
- SearchQueueMessage::RemoveDocument(_) => Operation::Remove,
SearchQueueMessage::ExtractSync(_) => Operation::ExtractSync,
SearchQueueMessage::UpdateDocumentProperties(_) => Operation::UpdateMetadata,
- SearchQueueMessage::UpdateDocumentName(_) => Operation::UpdateMetadata,
// Chat
SearchQueueMessage::ChatMessage(_) => Operation::ExtractText,
// Email
@@ -225,31 +219,3 @@ pub async fn bulk_enqueue_search_text_extractor(
Ok(())
}
-
-#[cfg(test)]
-mod tests {
- use super::*;
- use crate::search::document::DocumentId;
-
- #[test]
- fn update_document_name_maps_to_update_metadata() {
- let message = SearchQueueMessage::UpdateDocumentName(DocumentId {
- document_id: "doc-1".to_string(),
- });
-
- assert!(matches!(message.operation(), Operation::UpdateMetadata));
- assert_eq!(message.id(), "doc-1");
- }
-
- #[test]
- fn update_document_name_round_trips() {
- let message = SearchQueueMessage::UpdateDocumentName(DocumentId {
- document_id: "doc-1".to_string(),
- });
-
- let serialized = serde_json::to_string(&message).unwrap();
- let deserialized: SearchQueueMessage = serde_json::from_str(&serialized).unwrap();
-
- assert_eq!(message, deserialized);
- }
-}
diff --git a/crates/webhook/src/domain/ingestion.rs b/crates/webhook/src/domain/ingestion.rs
index bac974c171..14999d96fe 100644
--- a/crates/webhook/src/domain/ingestion.rs
+++ b/crates/webhook/src/domain/ingestion.rs
@@ -222,7 +222,7 @@ where
fn normalized_document_event(
event: &Event<DocumentTopicEvent>,
-) -> Result<NormalizedWebhookEvent, WebhookEventIngestionError> {
+) -> Result<Option<NormalizedWebhookEvent>, WebhookEventIngestionError> {
let (event_name, entity_id) = match &event.event {
DocumentTopicEvent::Created(metadata) => ("document.created", &metadata.document_id),
DocumentTopicEvent::Updated(metadata) => ("document.updated", &metadata.document_id),
@@ -231,6 +231,9 @@ fn normalized_document_event(
DocumentTopicEvent::Interaction(metadata) => {
("document.interaction", &metadata.document_id)
}
+ DocumentTopicEvent::ContentUploaded(_)
+ | DocumentTopicEvent::SyncContentUpdated(_)
+ | DocumentTopicEvent::Purged(_) => return Ok(None),
};
if Uuid::parse_str(entity_id).is_err() {
@@ -241,14 +244,14 @@ fn normalized_document_event(
}
let broker_envelope = serde_json::to_value(event)?;
- Ok(normalized_event(
+ Ok(Some(normalized_event(
event.event_id,
event.schema_version,
event_name,
DOCUMENT_ENTITY_TYPE,
entity_id,
broker_envelope,
- ))
+ )))
}
fn normalized_channel_event(
@@ -369,7 +372,9 @@ where
&self,
event: Event<DocumentTopicEvent>,
) -> Result<(), WebhookEventIngestionError> {
- let event = normalized_document_event(&event)?;
+ let Some(event) = normalized_document_event(&event)? else {
+ return Ok(());
+ };
self.resolve_entity_access_and_enqueue(event, EntityType::Document)
.await
}
diff --git a/crates/webhook/src/domain/ingestion/test.rs b/crates/webhook/src/domain/ingestion/test.rs
index f20ae8735b..86bad88071 100644
--- a/crates/webhook/src/domain/ingestion/test.rs
+++ b/crates/webhook/src/domain/ingestion/test.rs
@@ -21,7 +21,8 @@ use channels::domain::{
};
use chrono::{DateTime, Utc};
use documents::domain::events::{
- DocumentCopiedMetadata, DocumentCreatedMetadata, DocumentDeletedMetadata,
+ DocumentContentUploadedMetadata, DocumentCopiedMetadata, DocumentCreatedMetadata,
+ DocumentDeletedMetadata, DocumentPurgedMetadata, DocumentSyncContentUpdatedMetadata,
DocumentUpdatedMetadata,
};
use entity_access::domain::models::{
@@ -493,6 +494,38 @@ fn document_event_cases() -> Vec<EventCase> {
]
}
+fn search_only_document_event_cases() -> Vec<(&'static str, Event<DocumentTopicEvent>)> {
+ vec![
+ (
+ "document.content_uploaded",
+ Event::new(DocumentTopicEvent::ContentUploaded(
+ DocumentContentUploadedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ owner: user_id("macro|[email protected]"),
+ file_type: "pdf".parse().expect("valid file type"),
+ document_version_id: Some("convert".to_string()),
+ },
+ )),
+ ),
+ (
+ "document.sync_content_updated",
+ Event::new(DocumentTopicEvent::SyncContentUpdated(
+ DocumentSyncContentUpdatedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ file_type: "md".parse().expect("valid file type"),
+ document_version_id: None,
+ },
+ )),
+ ),
+ (
+ "document.purged",
+ Event::new(DocumentTopicEvent::Purged(DocumentPurgedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ })),
+ ),
+ ]
+}
+
fn channel_event_cases() -> Vec<EventCase> {
let channel_id = Uuid::parse_str("44444444-4444-4444-4444-444444444444").unwrap();
let message_id = Uuid::parse_str("55555555-5555-5555-5555-555555555555").unwrap();
@@ -825,6 +858,45 @@ async fn normalizes_and_matches_all_fourteen_event_variants() {
}
}
+#[tokio::test]
+async fn search_only_document_events_skip_webhook_delivery() {
+ let access = MockAccessService {
+ users: vec![user_id(PERSONAL_WORKSPACE_ID)],
+ failure: Some(AccessFailure::Internal),
+ calls: Arc::default(),
+ };
+ let repository = MockRepository::new(
+ vec![PERSONAL_WORKSPACE_ID.to_string()],
+ vec![webhook("wh_match", PERSONAL_WORKSPACE_ID)],
+ );
+ {
+ let mut state = lock(&repository.state);
+ state.fail_workspace_resolution = true;
+ state.fail_matching = true;
+ }
+ let enqueuer = MockEnqueuer::default();
+ let service = service(access.clone(), repository.clone(), enqueuer.clone());
+
+ for (event_name, event) in search_only_document_event_cases() {
+ assert!(
+ normalized_document_event(&event)
+ .unwrap_or_else(|error| panic!("{event_name} normalization failed: {error}"))
+ .is_none(),
+ "{event_name} should not normalize to a public webhook event"
+ );
+ service
+ .ingest_document_event(event)
+ .await
+ .unwrap_or_else(|error| panic!("{event_name} ingestion failed: {error}"));
+ }
+
+ assert!(lock(&access.calls).is_empty());
+ let repository_state = lock(&repository.state);
+ assert!(repository_state.workspace_calls.is_empty());
+ assert!(repository_state.match_calls.is_empty());
+ assert!(lock(&enqueuer.state).attempted_messages.is_empty());
+}
+
#[tokio::test]
async fn normalizes_all_webhook_variants_with_direct_workspace_ownership() {
for event_case in webhook_event_cases() {
diff --git a/infra/stacks/deleted-item-poller/index.ts b/infra/stacks/deleted-item-poller/index.ts
index 90ff7e1e06..8e00fafd17 100644
--- a/infra/stacks/deleted-item-poller/index.ts
+++ b/infra/stacks/deleted-item-poller/index.ts
@@ -1,6 +1,6 @@
import * as aws from '@pulumi/aws';
import * as pulumi from '@pulumi/pulumi';
-import { config, getSearchEventQueue, stack } from '../../packages/shared';
+import { config, stack } from '../../packages/shared';
import { get_coparse_api_vpc } from '../../packages/vpc';
import { DeleteItemPoller } from './lambda';
@@ -42,12 +42,10 @@ const deleteChatQueueArn: pulumi.Output<string> = cloudStorageServiceStack
.getOutput('deleteChatQueueArn')
.apply((arn) => arn as string);
-const { searchEventQueueArn } = getSearchEventQueue();
-
const vpc = get_coparse_api_vpc();
const deletedItemPoller = new DeleteItemPoller('deleted-item-poller', {
- queueArns: [deleteDocumentQueueArn, deleteChatQueueArn, searchEventQueueArn],
+ queueArns: [deleteDocumentQueueArn, deleteChatQueueArn],
vpc,
envVars: {
DATABASE_URL: pulumi.interpolate`${DATABASE_URL}`,
diff --git a/infra/stacks/search-upload/index.ts b/infra/stacks/search-upload/index.ts
index 287ed92713..5afe2f32e4 100644
--- a/infra/stacks/search-upload/index.ts
+++ b/infra/stacks/search-upload/index.ts
@@ -2,7 +2,6 @@ import * as aws from '@pulumi/aws';
import * as pulumi from '@pulumi/pulumi';
import {
config,
- getSearchEventQueue,
getServiceUrl,
ServiceUrl,
stack,
@@ -15,8 +14,6 @@ const tags = {
project: 'cloud-storage-search',
};
-const { searchEventQueueArn } = getSearchEventQueue();
-
const DOCUMENT_STORAGE_SERVICE_AUTH_KEY = aws.secretsmanager
.getSecretVersionOutput({
secretId: config.require('document_storage_service_auth_key'),
@@ -36,7 +33,6 @@ const searchUploadHandler = new SearchUploadHandler(
),
DOCUMENT_STORAGE_SERVICE_AUTH_KEY: pulumi.interpolate`${DOCUMENT_STORAGE_SERVICE_AUTH_KEY}`,
},
- searchEventQueueArn,
tags,
}
);
diff --git a/infra/stacks/search-upload/search-upload-lambda.ts b/infra/stacks/search-upload/search-upload-lambda.ts
index ac442b56da..9176b4e5a0 100644
--- a/infra/stacks/search-upload/search-upload-lambda.ts
+++ b/infra/stacks/search-upload/search-upload-lambda.ts
@@ -17,7 +17,6 @@ type EnvVars = {
type Args = {
envVars: EnvVars;
- searchEventQueueArn: pulumi.Output<string> | string;
tags: { [key: string]: string };
};
@@ -31,28 +30,10 @@ export class SearchUploadHandler extends pulumi.ComponentResource {
opts?: pulumi.ComponentResourceOptions
) {
super('my:components:SearchUploadHandler', name, {}, opts);
- const { envVars, searchEventQueueArn, tags } = args;
+ const { envVars, tags } = args;
this.tags = tags;
- const sqsPolicy = new aws.iam.Policy(
- `${LAMBA_BASE_NAME}-sqs-policy`,
- {
- policy: pulumi.output({
- Version: '2012-10-17',
- Statement: [
- {
- Action: ['sqs:SendMessage'],
- Resource: [searchEventQueueArn],
- Effect: 'Allow',
- },
- ],
- }),
- tags: this.tags,
- },
- { parent: this }
- );
-
this.role = new aws.iam.Role(
`${LAMBA_BASE_NAME}-role`,
{
@@ -73,7 +54,6 @@ export class SearchUploadHandler extends pulumi.ComponentResource {
aws.iam.ManagedPolicy.AWSLambdaBasicExecutionRole,
aws.iam.ManagedPolicy.AWSLambdaRole,
aws.iam.ManagedPolicy.CloudWatchLogsFullAccess,
- sqsPolicy.arn,
],
tags: this.tags,
},
diff --git a/packages/sdk/generated/storage/index.ts b/packages/sdk/generated/storage/index.ts
index 0a4ed6ff14..97980716b0 100644
--- a/packages/sdk/generated/storage/index.ts
+++ b/packages/sdk/generated/storage/index.ts
@@ -1,4 +1,4 @@
// This file is auto-generated by @hey-api/openapi-ts
export { type Options, Sdk } from './sdk.gen';
-export type { AccessLevel, Activity, ActivityType, AddFavoriteData, AddFavoriteError, AddFavoriteErrors, AddFavoriteRequest, AddFavoriteResponse, AddFavoriteResponses, AddParticipantsData, AddParticipantsError, AddParticipantsErrors, AddParticipantsRequest, AddParticipantsResponses, AddPinHandlerData, AddPinHandlerError, AddPinHandlerErrors, AddPinHandlerResponse, AddPinHandlerResponses, AddPinRequest, Anchor, AnchorId, AnchorRequest, AnchorResponse, AnnotationIncrementalUpdate, ApiActivity, ApiAttachmentChannelReference, ApiAttachmentEntityReference, ApiAttachmentGenericReference, ApiChannelAttachment, ApiChannelAttachmentsPage, ApiChannelContextMessage, ApiChannelDetail, ApiChannelListMessage, ApiChannelListPage, ApiChannelListParticipant, ApiChannelListType, ApiChannelMessage, ApiChannelMessageKind, ApiChannelMessagesPage, ApiChannelParticipant, ApiChannelWithLatest, ApiCountedReaction, ApiEntityFilterAst, ApiGroupByField, ApiGroupMeta, ApiMessageAttachment, ApiMessageSender, ApiMessageSenderType, ApiParticipantListRole, ApiParticipantRole, ApiPropertyEntityType, ApiResolvedChannelMessage, ApiThreadInfo, ApiThreadReply, BasicDocument, BasicDocumentSubType, BomPart, Bot, BotChannel, BotChannelType, BotId, BotKind, BotOwner, BotToken, BranchNameResponse, BTreeMap, BulkWakeupRequest, BulkWakeupResponse, BulkWakeupSyncServiceDocumentsData, BulkWakeupSyncServiceDocumentsErrors, BulkWakeupSyncServiceDocumentsResponse, BulkWakeupSyncServiceDocumentsResponses, CallActiveResponse, CallFilters, CallRecord, CallRecordParticipant, CallRecordPreview, CallRecordPreviewData, CallRecordTranscriptSegment, CallStatus, CallTokenResponse, CallWebhookData, CallWebhookError, CallWebhookErrors, CallWebhookResponses, Channel, ChannelCreatedMetadata, ChannelDeletedMetadata, ChannelEventAttachment, ChannelFilters, ChannelJoinCodeResponse, ChannelMessage, ChannelMessageAttachmentCreatedMetadata, ChannelMessageAttachmentRemovedMetadata, ChannelMessageDeletedMetadata, ChannelMessageFilters, ChannelMessagePatchedMetadata, ChannelMessagePostedMetadata, ChannelParticipant, ChannelParticipantAddedMetadata, ChannelParticipantRemovedMetadata, ChannelPreview, ChannelPreviewData, ChannelSender, ChannelSharePermission, ChannelThreadFilters, ChannelTopicEvent, ChannelType, ChannelUpdatedMetadata, ChannelWebhookRequest, ChannelWebhookResponse, ChannelWithParticipants, Chat, ChatFilters, CheckActiveCallData, CheckActiveCallError, CheckActiveCallErrors, CheckActiveCallResponse, CheckActiveCallResponses, ClientOptions, CloudStorageItemType, Comment, CommentThread, CopyDocumentData, CopyDocumentError, CopyDocumentErrors, CopyDocumentQueryParams, CopyDocumentRequest, CopyDocumentResponse, CopyDocumentResponse2, CopyDocumentResponses, CreateAnchorData, CreateAnchorError, CreateAnchorErrors, CreateAnchorResponse, CreateAnchorResponses, CreateBulkDocumentResponse, CreateBulkDocumentResponseData, CreateChannelData, CreateChannelError, CreateChannelErrors, CreateChannelRequest, CreateChannelResponse, CreateChannelResponse2, CreateChannelResponses, CreateChannelScopedBotData, CreateChannelScopedBotError, CreateChannelScopedBotErrors, CreateChannelScopedBotRequest, CreateChannelScopedBotResponse, CreateChannelScopedBotResponse2, CreateChannelScopedBotResponses, CreateCommentData, CreateCommentError, CreateCommentErrors, CreateCommentRequest, CreateCommentResponse, CreateCommentResponse2, CreateCommentResponses, CreateCrmCommentData, CreateCrmCommentError, CreateCrmCommentErrors, CreateCrmCommentRequest, CreateCrmCommentResponse, CreateCrmCommentResponses, CreateCrmCompanyData, CreateCrmCompanyError, CreateCrmCompanyErrors, CreateCrmCompanyRequest, CreateCrmCompanyResponse, CreateCrmCompanyResponses, CreateCrmContactData, CreateCrmContactError, CreateCrmContactErrors, CreateCrmContactRequest, CreateCrmContactResponse, CreateCrmContactResponses, CreateDocumentData, CreateDocumentError, CreateDocumentErrors, CreateDocumentRequest, CreateDocumentResponse, CreateDocumentResponseData, CreateDocumentResponses, CreateEntityMentionData, CreateEntityMentionError, CreateEntityMentionErrors, CreateEntityMentionRequest, CreateEntityMentionResponse, CreateEntityMentionResponse2, CreateEntityMentionResponses, CreateInstructionsDocumentResponse, CreateInstructionsHandlerData, CreateInstructionsHandlerError, CreateInstructionsHandlerErrors, CreateInstructionsHandlerResponse, CreateInstructionsHandlerResponses, CreateMarkdownDocumentRequest, CreateMarkdownDocumentResponse, CreateMarkdownHandlerData, CreateMarkdownHandlerError, CreateMarkdownHandlerErrors, CreateMarkdownHandlerResponse, CreateMarkdownHandlerResponses, CreateProjectHandlerData, CreateProjectHandlerError, CreateProjectHandlerErrors, CreateProjectHandlerResponse, CreateProjectHandlerResponses, CreateProjectRequest, CreateProjectResponse, CreateSnippetHandlerData, CreateSnippetHandlerError, CreateSnippetHandlerErrors, CreateSnippetHandlerResponse, CreateSnippetHandlerResponses, CreateSnippetRequest, CreateSnippetResponse, CreateTaskHandlerData, CreateTaskHandlerError, CreateTaskHandlerErrors, CreateTaskHandlerResponse, CreateTaskHandlerResponses, CreateTaskRequest, CreateTaskResponse, CreateUnthreadedAnchorRequest, CreateUnthreadedAnchorResponse, CreateUnthreadedPdfAnchorRequest, CreateViewHandlerData, CreateViewHandlerError, CreateViewHandlerErrors, CreateViewHandlerResponse, CreateViewHandlerResponses, CreateViewRequest, CreateWebhookData, CreateWebhookError, CreateWebhookErrors, CreateWebhookRequest, CreateWebhookResponse, CreateWebhookResponse2, CreateWebhookResponses, CrmComment, CrmCommentEntityType, CrmCommentThread, CrmCompanyFilters, CrmCompanyResponse, CrmContactResponse, CrmDomainResponse, CrmPermissionRole, CrmTeamSettingsResponse, CrmThread, CustomSpeakerAssignment, DataType, DeleteAnchorData, DeleteAnchorError, DeleteAnchorErrors, DeleteAnchorInfo, DeleteAnchorResponse, DeleteAnchorResponses, DeleteCallRecordData, DeleteCallRecordError, DeleteCallRecordErrors, DeleteCallRecordResponse, DeleteCallRecordResponses, DeleteChannelData, DeleteChannelError, DeleteChannelErrors, DeleteChannelResponse, DeleteChannelResponses, DeleteCommentData, DeleteCommentError, DeleteCommentErrors, DeleteCommentRequest, DeleteCommentResponse, DeleteCommentResponse2, DeleteCommentResponses, DeleteCrmCommentData, DeleteCrmCommentError, DeleteCrmCommentErrors, DeleteCrmCommentResponse, DeleteCrmCommentResponses, DeleteCrmCommentResult, DeleteDocumentData, DeleteDocumentError, DeleteDocumentErrors, DeleteDocumentResponse, DeleteDocumentResponses, DeleteEntityMentionData, DeleteEntityMentionError, DeleteEntityMentionErrors, DeleteEntityMentionResponse, DeleteEntityMentionResponse2, DeleteEntityMentionResponses, DeleteHistoryHandlerData, DeleteHistoryHandlerError, DeleteHistoryHandlerErrors, DeleteHistoryHandlerResponse, DeleteHistoryHandlerResponses, DeleteMessageData, DeleteMessageError, DeleteMessageErrors, DeleteMessageQuery, DeleteMessageResponse, DeleteMessageResponses, DeleteProjectHandlerData, DeleteProjectHandlerError, DeleteProjectHandlerErrors, DeleteProjectHandlerResponse, DeleteProjectHandlerResponses, DeleteThreadInfo, DeleteUnthreadedAnchorRequest, DeleteUnthreadedAnchorResponse, DeleteUnthreadedPdfAnchorRequest, DeleteUserDocumentViewLocationData, DeleteUserDocumentViewLocationError, DeleteUserDocumentViewLocationErrors, DeleteUserDocumentViewLocationResponse, DeleteUserDocumentViewLocationResponses, DeleteViewHandlerData, DeleteViewHandlerError, DeleteViewHandlerErrors, DeleteViewHandlerResponses, DeleteWebhookData, DeleteWebhookError, DeleteWebhookErrors, DeleteWebhookResponse, DeleteWebhookResponses, DocumentBasic, DocumentContent, DocumentContentLocation, DocumentContentState, DocumentCopiedMetadata, DocumentCreatedMetadata, DocumentDeletedMetadata, DocumentFilters, DocumentInteractionMetadata, DocumentMetadata, DocumentMetadataWithContent, DocumentPermissionsToken, DocumentPermissionsTokenRequest, DocumentPermissionsTokenResponse, DocumentPreview, DocumentPreviewData, DocumentPreviewDataSubType, DocumentResponse, DocumentResponseMetadata, DocumentResponseMetadataWithContent, DocumentStorageServiceApiVersion, DocumentSubType, DocumentTeamShareResponse, DocumentTopicEvent, DocumentUpdatedMetadata, EditAnchorData, EditAnchorError, EditAnchorErrors, EditAnchorRequest, EditAnchorResponse, EditAnchorResponse2, EditAnchorResponses, EditCallRecordData, EditCallRecordError, EditCallRecordErrors, EditCallRecordRequest, EditCallRecordResponse, EditCallRecordResponses, EditCallTranscriptData, EditCallTranscriptError, EditCallTranscriptErrors, EditCallTranscriptRequest, EditCallTranscriptResponse, EditCallTranscriptResponses, EditCommentData, EditCommentError, EditCommentErrors, EditCommentRequest, EditCommentResponse, EditCommentResponse2, EditCommentResponses, EditCrmCommentData, EditCrmCommentError, EditCrmCommentErrors, EditCrmCommentRequest, EditCrmCommentResponse, EditCrmCommentResponses, EditDocumentData, EditDocumentError, EditDocumentErrors, EditDocumentResponse, EditDocumentResponse2, EditDocumentResponses, EditDocumentServiceArgs, EditPdfAnchorRequest, EditPdfPlaceableCommentAnchorRequest, EditProjectV2Data, EditProjectV2Error, EditProjectV2Errors, EditProjectV2Response, EditProjectV2Responses, EditThreadV2Data, EditThreadV2Error, EditThreadV2Errors, EditThreadV2Response, EditThreadV2Responses, EmailFilters, EmptyResponse, EntityFilters, EntityPermission, EntityPermissionResponse, EntityReference, EntityType, ErrorResponse, ExcludedDefaultView, ExcludeDefaultViewHandlerData, ExcludeDefaultViewHandlerError, ExcludeDefaultViewHandlerErrors, ExcludeDefaultViewHandlerResponses, ExcludeDefaultViewRequest, ExportDocumentData, ExportDocumentError, ExportDocumentErrors, ExportDocumentResponse, ExportDocumentResponse2, ExportDocumentResponses, Favorite, FavoriteEntityRef, FavoritesList, FileSystemNodeWithIds, FileType, FileTypeUpdate, FolderItem, ForeignEntity, ForeignEntityFilters, GenericErrorResponse, GenericResponse, GenericSuccessResponse, GetActivitiesResponse, GetActivityData, GetActivityError, GetActivityErrors, GetActivityResponse, GetActivityResponses, GetAttachmentReferencesData, GetAttachmentReferencesError, GetAttachmentReferencesErrors, GetAttachmentReferencesResponse, GetAttachmentReferencesResponse2, GetAttachmentReferencesResponses, GetBatchCallRecordPreviewData, GetBatchCallRecordPreviewError, GetBatchCallRecordPreviewErrors, GetBatchCallRecordPreviewRequest, GetBatchCallRecordPreviewResponse, GetBatchCallRecordPreviewResponse2, GetBatchCallRecordPreviewResponses, GetBatchChannelPreviewData, GetBatchChannelPreviewError, GetBatchChannelPreviewErrors, GetBatchChannelPreviewRequest, GetBatchChannelPreviewResponse, GetBatchChannelPreviewResponse2, GetBatchChannelPreviewResponses, GetBatchPreviewHandlerData, GetBatchPreviewHandlerError, GetBatchPreviewHandlerErrors, GetBatchPreviewHandlerResponse, GetBatchPreviewHandlerResponses, GetBatchPreviewRequest, GetBatchPreviewResponse, GetBatchProjectPreviewData, GetBatchProjectPreviewError, GetBatchProjectPreviewErrors, GetBatchProjectPreviewRequest, GetBatchProjectPreviewResponse, GetBatchProjectPreviewResponse2, GetBatchProjectPreviewResponses, GetCallRecordData, GetCallRecordError, GetCallRecordErrors, GetCallRecordResponse, GetCallRecordResponses, GetChannelAttachmentsData, GetChannelAttachmentsError, GetChannelAttachmentsErrors, GetChannelAttachmentsResponse, GetChannelAttachmentsResponses, GetChannelData, GetChannelError, GetChannelErrors, GetChannelJoinLinkData, GetChannelJoinLinkError, GetChannelJoinLinkErrors, GetChannelJoinLinkResponse, GetChannelJoinLinkResponses, GetChannelMessagesData, GetChannelMessagesError, GetChannelMessagesErrors, GetChannelMessagesResponse, GetChannelMessagesResponses, GetChannelParticipantsData, GetChannelParticipantsError, GetChannelParticipantsErrors, GetChannelParticipantsResponse, GetChannelParticipantsResponses, GetChannelResponse, GetChannelResponses, GetChannelsData, GetChannelsError, GetChannelsErrors, GetChannelsResponse, GetChannelsResponses, GetCompanyData, GetCompanyError, GetCompanyErrors, GetCompanyResponse, GetCompanyResponses, GetContactData, GetContactError, GetContactErrors, GetContactResponse, GetContactResponses, GetCrmTeamSettingsData, GetCrmTeamSettingsError, GetCrmTeamSettingsErrors, GetCrmTeamSettingsResponse, GetCrmTeamSettingsResponses, GetDocumentAnchorsData, GetDocumentAnchorsError, GetDocumentAnchorsErrors, GetDocumentAnchorsResponse, GetDocumentAnchorsResponses, GetDocumentBranchNameData, GetDocumentBranchNameError, GetDocumentBranchNameErrors, GetDocumentBranchNameResponse, GetDocumentBranchNameResponses, GetDocumentByTeamSlugData, GetDocumentByTeamSlugError, GetDocumentByTeamSlugErrors, GetDocumentByTeamSlugResponse, GetDocumentByTeamSlugResponses, GetDocumentCommentsData, GetDocumentCommentsError, GetDocumentCommentsErrors, GetDocumentCommentsResponse, GetDocumentCommentsResponses, GetDocumentData, GetDocumentError, GetDocumentErrors, GetDocumentGithubPullRequestsData, GetDocumentGithubPullRequestsError, GetDocumentGithubPullRequestsErrors, GetDocumentGithubPullRequestsResponse, GetDocumentGithubPullRequestsResponses, GetDocumentKeyResponse, GetDocumentKeyResponseData, GetDocumentListHandlerData, GetDocumentListHandlerError, GetDocumentListHandlerErrors, GetDocumentListHandlerResponse, GetDocumentListHandlerResponses, GetDocumentListResult, GetDocumentLocationV3Data, GetDocumentLocationV3Error, GetDocumentLocationV3Errors, GetDocumentLocationV3Response, GetDocumentLocationV3Responses, GetDocumentPermissionsResponseDataV2, GetDocumentPermissionsTokenData, GetDocumentPermissionsTokenError, GetDocumentPermissionsTokenErrors, GetDocumentPermissionsTokenResponse, GetDocumentPermissionsTokenResponses, GetDocumentPermissionsV2Data, GetDocumentPermissionsV2Error, GetDocumentPermissionsV2Errors, GetDocumentPermissionsV2Response, GetDocumentPermissionsV2Responses, GetDocumentProcessingResult, GetDocumentProcessingResultData, GetDocumentProcessingResultError, GetDocumentProcessingResultErrors, GetDocumentProcessingResultResponse, GetDocumentProcessingResultResponse2, GetDocumentProcessingResultResponses, GetDocumentResponse, GetDocumentResponse2, GetDocumentResponseData, GetDocumentResponses, GetDocumentSearchResponse, GetDocumentShortIdData, GetDocumentShortIdError, GetDocumentShortIdErrors, GetDocumentShortIdResponse, GetDocumentShortIdResponses, GetDocumentsResponse, GetDocumentTeamShareData, GetDocumentTeamShareError, GetDocumentTeamShareErrors, GetDocumentTeamShareResponse, GetDocumentTeamShareResponses, GetDocumentUserAccessLevelResponse, GetDocumentVersionData, GetDocumentVersionError, GetDocumentVersionErrors, GetDocumentVersionResponse, GetDocumentVersionResponses, GetDocumentViewsHandlerData, GetDocumentViewsHandlerError, GetDocumentViewsHandlerErrors, GetDocumentViewsHandlerResponse, GetDocumentViewsHandlerResponses, GetEntityPermissionData, GetEntityPermissionErrors, GetEntityPermissionResponse, GetEntityPermissionResponses, GetForeignEntityData, GetForeignEntityError, GetForeignEntityErrors, GetForeignEntityResponse, GetForeignEntityResponses, GetHistoryHandlerData, GetHistoryHandlerError, GetHistoryHandlerErrors, GetHistoryHandlerResponse, GetHistoryHandlerResponses, GetInstructionsDocumentResponse, GetInstructionsHandlerData, GetInstructionsHandlerError, GetInstructionsHandlerErrors, GetInstructionsHandlerResponse, GetInstructionsHandlerResponses, GetItemsSoupData, GetItemsSoupError, GetItemsSoupErrors, GetItemsSoupResponse, GetItemsSoupResponses, GetLocationHandlerData, GetLocationHandlerError, GetLocationHandlerErrors, GetLocationHandlerResponse, GetLocationHandlerResponses, GetMessageWithContextData, GetMessageWithContextError, GetMessageWithContextErrors, GetMessageWithContextResponse, GetMessageWithContextResponse2, GetMessageWithContextResponses, GetOrCreateAction, GetOrCreateCallData, GetOrCreateCallError, GetOrCreateCallErrors, GetOrCreateCallResponse, GetOrCreateCallResponses, GetOrCreateChannelResponse, GetOrCreateDmData, GetOrCreateDmError, GetOrCreateDmErrors, GetOrCreateDmRequest, GetOrCreateDmResponse, GetOrCreateDmResponses, GetOrCreatePrivateData, GetOrCreatePrivateError, GetOrCreatePrivateErrors, GetOrCreatePrivateRequest, GetOrCreatePrivateResponse, GetOrCreatePrivateResponses, GetPendingProjectsHandlerData, GetPendingProjectsHandlerError, GetPendingProjectsHandlerErrors, GetPendingProjectsHandlerResponse, GetPendingProjectsHandlerResponses, GetPinsHandlerData, GetPinsHandlerError, GetPinsHandlerErrors, GetPinsHandlerResponse, GetPinsHandlerResponses, GetPinsResponse, GetProjectContentHandlerData, GetProjectContentHandlerError, GetProjectContentHandlerErrors, GetProjectContentHandlerResponse, GetProjectContentHandlerResponses, GetProjectContentResponse, GetProjectHandlerData, GetProjectHandlerError, GetProjectHandlerErrors, GetProjectHandlerResponse, GetProjectHandlerResponses, GetProjectPermissionsV2Data, GetProjectPermissionsV2Error, GetProjectPermissionsV2Errors, GetProjectPermissionsV2Response, GetProjectPermissionsV2Responses, GetProjectResponse, GetProjectResponseData, GetProjectsHandlerData, GetProjectsHandlerError, GetProjectsHandlerErrors, GetProjectsHandlerResponse, GetProjectsHandlerResponses, GetProjectsResponse, GetProjectUserAccessLevelData, GetProjectUserAccessLevelError, GetProjectUserAccessLevelErrors, GetProjectUserAccessLevelResponse, GetProjectUserAccessLevelResponses, GetRecentActivityHandlerData, GetRecentActivityHandlerError, GetRecentActivityHandlerErrors, GetRecentActivityHandlerResponse, GetRecentActivityHandlerResponses, GetRingStatusData, GetRingStatusError, GetRingStatusErrors, GetRingStatusResponse, GetRingStatusResponses, GetThreadRepliesData, GetThreadRepliesError, GetThreadRepliesErrors, GetThreadRepliesResponse, GetThreadRepliesResponses, GetUserDocumentsHandlerData, GetUserDocumentsHandlerError, GetUserDocumentsHandlerErrors, GetUserDocumentsHandlerResponse, GetUserDocumentsHandlerResponses, GetUserDocumentViewLocationData, GetUserDocumentViewLocationError, GetUserDocumentViewLocationErrors, GetUserDocumentViewLocationResponse, GetUserDocumentViewLocationResponses, GetUserHistoryResponse, GetViewsHandlerData, GetViewsHandlerError, GetViewsHandlerErrors, GetViewsHandlerResponse, GetViewsHandlerResponses, GetWebhookData, GetWebhookError, GetWebhookErrors, GetWebhookResponse, GetWebhookResponses, GithubPullRequest, GithubPullRequestCheckRun, GithubPullRequestComment, GithubPullRequestsResponse, GroupedInitialParams, GroupedPageParams, GroupedSoupGroupPage, GroupedSoupInitialPage, GroupedSoupPage, GroupedSoupSort, HandlerData, HandlerError, HandlerErrors, HandlerResponse, HandlerResponses, HashMap, HealthHandlerData, HealthHandlerResponse, HealthHandlerResponses, HighlightType, IngestTranscriptData, IngestTranscriptError, IngestTranscriptErrors, IngestTranscriptResponses, InitializeUserDocumentsData, InitializeUserDocumentsError, InitializeUserDocumentsErrors, InitializeUserDocumentsResponse, InitializeUserDocumentsResponses, InstallSyncData, InteractionReason, Item, ItemWithUserAccessLevel, JobProcessingResultHandlerData, JobProcessingResultHandlerError, JobProcessingResultHandlerErrors, JobProcessingResultHandlerResponse, JobProcessingResultHandlerResponses, JoinChannelByCodeData, JoinChannelByCodeError, JoinChannelByCodeErrors, JoinChannelByCodeResponses, JoinChannelData, JoinChannelError, JoinChannelErrors, JoinChannelResponses, LatestMessage, LeaveCallResponse, LeaveChannelData, LeaveChannelError, LeaveChannelErrors, LeaveChannelResponses, LeaveOrEndCallData, LeaveOrEndCallError, LeaveOrEndCallErrors, LeaveOrEndCallResponse, LeaveOrEndCallResponses, ListBotChannelsData, ListBotChannelsError, ListBotChannelsErrors, ListBotChannelsResponse, ListBotChannelsResponses, ListCompanyContactsData, ListCompanyContactsError, ListCompanyContactsErrors, ListCompanyContactsResponse, ListCompanyContactsResponses, ListCrmCommentsData, ListCrmCommentsError, ListCrmCommentsErrors, ListCrmCommentsResponse, ListCrmCommentsResponses, ListFavoritesData, ListFavoritesError, ListFavoritesErrors, ListFavoritesResponse, ListFavoritesResponses, ListWebhooksData, ListWebhooksError, ListWebhooksErrors, ListWebhooksResponse, ListWebhooksResponse2, ListWebhooksResponses, LocationResponseData, LocationResponseV3, MacroUserIdStr, Mentions, NewChannelAttachment, NotificationFilters, Params, ParticipantRole, PatchChannelData, PatchChannelError, PatchChannelErrors, PatchChannelRequest, PatchChannelResponse, PatchChannelResponses, PatchMessageData, PatchMessageError, PatchMessageErrors, PatchMessageRequest, PatchMessageResponse, PatchMessageResponses, PatchProjectRequestV2, PatchThreadRequestV2, PatchViewHandlerData, PatchViewHandlerError, PatchViewHandlerErrors, PatchViewHandlerResponses, PatchWebhookData, PatchWebhookError, PatchWebhookErrors, PatchWebhookRequest, PatchWebhookResponse, PatchWebhookResponses, PdfAnchor, PdfAnchorId, PdfAnchorRequest, PdfHighlightAnchor, PdfHighlightAnchorRequest, PdfHighlightRect, PdfHighlightRectAnchorRequest, PdfPlaceableCommentAnchor, PdfPlaceableCommentAnchorRequest, PermanentlyDeleteDocumentData, PermanentlyDeleteDocumentError, PermanentlyDeleteDocumentErrors, PermanentlyDeleteDocumentResponse, PermanentlyDeleteDocumentResponses, PermanentlyDeleteProjectData, PermanentlyDeleteProjectError, PermanentlyDeleteProjectErrors, PermanentlyDeleteProjectResponse, PermanentlyDeleteProjectResponses, PinnedItem, PinRequest, PostActivityData, PostActivityError, PostActivityErrors, PostActivityRequest, PostActivityResponse, PostActivityResponses, PostChannelBotWebhookData, PostChannelBotWebhookError, PostChannelBotWebhookErrors, PostChannelBotWebhookResponse, PostChannelBotWebhookResponses, PostChannelMessagesData, PostChannelMessagesError, PostChannelMessagesErrors, PostChannelMessagesResponse, PostChannelMessagesResponses, PostGroupedSoupAstGroupPageRequest, PostGroupedSoupAstInitialRequest, PostGroupedSoupAstRequest, PostItemsSoupAstData, PostItemsSoupAstError, PostItemsSoupAstErrors, PostItemsSoupAstGroupedData, PostItemsSoupAstGroupedError, PostItemsSoupAstGroupedErrors, PostItemsSoupAstGroupedResponse, PostItemsSoupAstGroupedResponses, PostItemsSoupAstResponse, PostItemsSoupAstResponses, PostItemsSoupData, PostItemsSoupError, PostItemsSoupErrors, PostItemsSoupResponse, PostItemsSoupResponses, PostMessageData, PostMessageError, PostMessageErrors, PostMessageRequest, PostMessageResponse, PostMessageResponse2, PostMessageResponses, PostReactionData, PostReactionError, PostReactionErrors, PostReactionRequest, PostReactionResponse, PostReactionResponses, PostSoupAstRequest, PostSoupRequest, PostTypingData, PostTypingError, PostTypingErrors, PostTypingRequest, PostTypingResponse, PostTypingResponses, PresaveDocumentHandlerData, PresaveDocumentHandlerError, PresaveDocumentHandlerErrors, PresaveDocumentHandlerResponse, PresaveDocumentHandlerResponses, PreSaveDocumentRequest, PreSaveDocumentResponse, PreSaveDocumentResponseData, PresignedUrl, Project, ProjectDeleteResponseData, ProjectDocumentStatus, ProjectFilters, ProjectPreview, ProjectPreviewData, PropertyDefinition, PropertyFilter, PropertyInput, PropertyOwner, PropertyValue, PutCrmTeamSettingsData, PutCrmTeamSettingsError, PutCrmTeamSettingsErrors, PutCrmTeamSettingsResponse, PutCrmTeamSettingsResponses, ReactionAction, RecentlyDeletedData, RecentlyDeletedError, RecentlyDeletedErrors, RecentlyDeletedResponse, RecentlyDeletedResponseData, RecentlyDeletedResponses, RemoveBotFromChannelByBotData, RemoveBotFromChannelByBotError, RemoveBotFromChannelByBotErrors, RemoveBotFromChannelByBotResponse, RemoveBotFromChannelByBotResponses, RemoveFavoriteByEntityData, RemoveFavoriteByEntityError, RemoveFavoriteByEntityErrors, RemoveFavoriteByEntityResponses, RemoveParticipantsData, RemoveParticipantsError, RemoveParticipantsErrors, RemoveParticipantsRequest, RemoveParticipantsResponses, RemovePinHandlerData, RemovePinHandlerError, RemovePinHandlerErrors, RemovePinHandlerResponse, RemovePinHandlerResponses, ReorderFavoritesData, ReorderFavoritesError, ReorderFavoritesErrors, ReorderFavoritesRequest, ReorderFavoritesResponses, ReorderPinRequest, ReorderPinsHandlerData, ReorderPinsHandlerError, ReorderPinsHandlerErrors, ReorderPinsHandlerResponse, ReorderPinsHandlerResponses, ResolveChannelMessageData, ResolveChannelMessageError, ResolveChannelMessageErrors, ResolveChannelMessageResponse, ResolveChannelMessageResponses, RevertDeleteDocumentData, RevertDeleteDocumentError, RevertDeleteDocumentErrors, RevertDeleteDocumentResponse, RevertDeleteDocumentResponses, RevertDeleteProjectData, RevertDeleteProjectError, RevertDeleteProjectErrors, RevertDeleteProjectResponse, RevertDeleteProjectResponses, RingStatus, RingStatusResponse, S3ObjectInfo, SaveBomPart, SaveDocumentHandlerData, SaveDocumentHandlerError, SaveDocumentHandlerErrors, SaveDocumentHandlerResponse, SaveDocumentHandlerResponses, SaveDocumentRequest, SaveDocumentResponse, SaveDocumentResponseData, SetCompanyHiddenData, SetCompanyHiddenError, SetCompanyHiddenErrors, SetCompanyHiddenRequest, SetCompanyHiddenResponse, SetCompanyHiddenResponses, SetCompanyNameRequest, SetContactHiddenData, SetContactHiddenError, SetContactHiddenErrors, SetContactHiddenRequest, SetContactHiddenResponse, SetContactHiddenResponses, SetContactNameRequest, SetCrmCompanyNameData, SetCrmCompanyNameError, SetCrmCompanyNameErrors, SetCrmCompanyNameResponse, SetCrmCompanyNameResponses, SetCrmContactNameData, SetCrmContactNameError, SetCrmContactNameErrors, SetCrmContactNameResponse, SetCrmContactNameResponses, SetDocumentTeamShareData, SetDocumentTeamShareError, SetDocumentTeamShareErrors, SetDocumentTeamShareRequest, SetDocumentTeamShareResponse, SetDocumentTeamShareResponses, SetEmailSyncData, SetEmailSyncError, SetEmailSyncErrors, SetEmailSyncRequest, SetEmailSyncResponse, SetEmailSyncResponses, SetPropertyValue, SharedEmailFilter, SharePermissionV2, ShortIdResponse, SimpleMention, SimpleSaveData, SimpleSaveError, SimpleSaveErrors, SimpleSaveResponse, SimpleSaveResponses, SoupApiItem, SoupApiSort, SoupAttachment, SoupCallRecordParticipant, SoupCallRecordSoupPropertiesField, SoupChannel, SoupChannelThread, SoupChatSoupPropertiesField, SoupContact, SoupCountedReaction, SoupCrmCompanySoupPropertiesField, SoupCrmDomain, SoupDocumentSoupPropertiesField, SoupDocumentSubType, SoupEmailThreadPreview, SoupEnrichedEmailThreadPreviewSoupPropertiesField, SoupForeignEntity, SoupItem, SoupLabel, SoupLabelListVisibility, SoupLabelType, SoupMessageAttachment, SoupMessageListVisibility, SoupMessageSender, SoupMessageSenderType, SoupPage, SoupProjectSoupPropertiesField, SoupPropertiesField, SoupProperty, SoupThreadInfo, SoupThreadReply, StarterDocumentsResponse, String, SuccessResponse, SyncServiceVersionId, TagFilterMode, TaskFilters, TeamRole, Thread, ThreadResponse, ToggleShareWithTeamData, ToggleShareWithTeamError, ToggleShareWithTeamErrors, ToggleShareWithTeamResponse, ToggleShareWithTeamResponses, TranscriptSegmentRequest, TypedSuccessResponse, TypedSuccessResponseGetDocumentResponseData, TypingAction, UnthreadedPdfUuidRequest, UpdateChannelSharePermission, UpdateCrmTeamSettingsRequest, UpdateOperation, UpdateSharePermissionRequestV2, UploadDocumentStatus, UploadExtractFolderHandlerData, UploadExtractFolderHandlerError, UploadExtractFolderHandlerErrors, UploadExtractFolderHandlerResponse, UploadExtractFolderHandlerResponses, UploadExtractFolderRequest, UploadFolderHandlerData, UploadFolderHandlerError, UploadFolderHandlerErrors, UploadFolderHandlerResponse, UploadFolderHandlerResponses, UploadFolderRequest, UpsertHistoryHandlerData, UpsertHistoryHandlerError, UpsertHistoryHandlerErrors, UpsertHistoryHandlerResponse, UpsertHistoryHandlerResponses, UpsertUserDocumentViewLocationData, UpsertUserDocumentViewLocationError, UpsertUserDocumentViewLocationErrors, UpsertUserDocumentViewLocationRequest, UpsertUserDocumentViewLocationResponse, UpsertUserDocumentViewLocationResponses, UserActivitiesResponse, UserDocumentsResponse, UserDocumentViewLocation, UserDocumentViewLocationResponse, UserPinsResponse, UserViewsResponse, ValidateDocumentPermissionsTokenData, ValidateDocumentPermissionsTokenError, ValidateDocumentPermissionsTokenErrors, ValidateDocumentPermissionsTokenResponse, ValidateDocumentPermissionsTokenResponses, ValidateWebhookData, ValidateWebhookError, ValidateWebhookErrors, ValidateWebhookResponse, ValidateWebhookResponse2, ValidateWebhookResponses, Vec, View, ViewPatch, ViewsResponse, Webhook, WebhookEvent, WebhookFilter, WebhookScope, WebhookStatus, WebhookValidationTestEvent, WithCallId, WithChannelId, WithDocumentId, WithProjectId } from './types.gen';
+export type { AccessLevel, Activity, ActivityType, AddFavoriteData, AddFavoriteError, AddFavoriteErrors, AddFavoriteRequest, AddFavoriteResponse, AddFavoriteResponses, AddParticipantsData, AddParticipantsError, AddParticipantsErrors, AddParticipantsRequest, AddParticipantsResponses, AddPinHandlerData, AddPinHandlerError, AddPinHandlerErrors, AddPinHandlerResponse, AddPinHandlerResponses, AddPinRequest, Anchor, AnchorId, AnchorRequest, AnchorResponse, AnnotationIncrementalUpdate, ApiActivity, ApiAttachmentChannelReference, ApiAttachmentEntityReference, ApiAttachmentGenericReference, ApiChannelAttachment, ApiChannelAttachmentsPage, ApiChannelContextMessage, ApiChannelDetail, ApiChannelListMessage, ApiChannelListPage, ApiChannelListParticipant, ApiChannelListType, ApiChannelMessage, ApiChannelMessageKind, ApiChannelMessagesPage, ApiChannelParticipant, ApiChannelWithLatest, ApiCountedReaction, ApiEntityFilterAst, ApiGroupByField, ApiGroupMeta, ApiMessageAttachment, ApiMessageSender, ApiMessageSenderType, ApiParticipantListRole, ApiParticipantRole, ApiPropertyEntityType, ApiResolvedChannelMessage, ApiThreadInfo, ApiThreadReply, BasicDocument, BasicDocumentSubType, BomPart, Bot, BotChannel, BotChannelType, BotId, BotKind, BotOwner, BotToken, BranchNameResponse, BTreeMap, BulkWakeupRequest, BulkWakeupResponse, BulkWakeupSyncServiceDocumentsData, BulkWakeupSyncServiceDocumentsErrors, BulkWakeupSyncServiceDocumentsResponse, BulkWakeupSyncServiceDocumentsResponses, CallActiveResponse, CallFilters, CallRecord, CallRecordParticipant, CallRecordPreview, CallRecordPreviewData, CallRecordTranscriptSegment, CallStatus, CallTokenResponse, CallWebhookData, CallWebhookError, CallWebhookErrors, CallWebhookResponses, Channel, ChannelCreatedMetadata, ChannelDeletedMetadata, ChannelEventAttachment, ChannelFilters, ChannelJoinCodeResponse, ChannelMessage, ChannelMessageAttachmentCreatedMetadata, ChannelMessageAttachmentRemovedMetadata, ChannelMessageDeletedMetadata, ChannelMessageFilters, ChannelMessagePatchedMetadata, ChannelMessagePostedMetadata, ChannelParticipant, ChannelParticipantAddedMetadata, ChannelParticipantRemovedMetadata, ChannelPreview, ChannelPreviewData, ChannelSender, ChannelSharePermission, ChannelThreadFilters, ChannelTopicEvent, ChannelType, ChannelUpdatedMetadata, ChannelWebhookRequest, ChannelWebhookResponse, ChannelWithParticipants, Chat, ChatFilters, CheckActiveCallData, CheckActiveCallError, CheckActiveCallErrors, CheckActiveCallResponse, CheckActiveCallResponses, ClientOptions, CloudStorageItemType, Comment, CommentThread, CopyDocumentData, CopyDocumentError, CopyDocumentErrors, CopyDocumentQueryParams, CopyDocumentRequest, CopyDocumentResponse, CopyDocumentResponse2, CopyDocumentResponses, CreateAnchorData, CreateAnchorError, CreateAnchorErrors, CreateAnchorResponse, CreateAnchorResponses, CreateBulkDocumentResponse, CreateBulkDocumentResponseData, CreateChannelData, CreateChannelError, CreateChannelErrors, CreateChannelRequest, CreateChannelResponse, CreateChannelResponse2, CreateChannelResponses, CreateChannelScopedBotData, CreateChannelScopedBotError, CreateChannelScopedBotErrors, CreateChannelScopedBotRequest, CreateChannelScopedBotResponse, CreateChannelScopedBotResponse2, CreateChannelScopedBotResponses, CreateCommentData, CreateCommentError, CreateCommentErrors, CreateCommentRequest, CreateCommentResponse, CreateCommentResponse2, CreateCommentResponses, CreateCrmCommentData, CreateCrmCommentError, CreateCrmCommentErrors, CreateCrmCommentRequest, CreateCrmCommentResponse, CreateCrmCommentResponses, CreateCrmCompanyData, CreateCrmCompanyError, CreateCrmCompanyErrors, CreateCrmCompanyRequest, CreateCrmCompanyResponse, CreateCrmCompanyResponses, CreateCrmContactData, CreateCrmContactError, CreateCrmContactErrors, CreateCrmContactRequest, CreateCrmContactResponse, CreateCrmContactResponses, CreateDocumentData, CreateDocumentError, CreateDocumentErrors, CreateDocumentRequest, CreateDocumentResponse, CreateDocumentResponseData, CreateDocumentResponses, CreateEntityMentionData, CreateEntityMentionError, CreateEntityMentionErrors, CreateEntityMentionRequest, CreateEntityMentionResponse, CreateEntityMentionResponse2, CreateEntityMentionResponses, CreateInstructionsDocumentResponse, CreateInstructionsHandlerData, CreateInstructionsHandlerError, CreateInstructionsHandlerErrors, CreateInstructionsHandlerResponse, CreateInstructionsHandlerResponses, CreateMarkdownDocumentRequest, CreateMarkdownDocumentResponse, CreateMarkdownHandlerData, CreateMarkdownHandlerError, CreateMarkdownHandlerErrors, CreateMarkdownHandlerResponse, CreateMarkdownHandlerResponses, CreateProjectHandlerData, CreateProjectHandlerError, CreateProjectHandlerErrors, CreateProjectHandlerResponse, CreateProjectHandlerResponses, CreateProjectRequest, CreateProjectResponse, CreateSnippetHandlerData, CreateSnippetHandlerError, CreateSnippetHandlerErrors, CreateSnippetHandlerResponse, CreateSnippetHandlerResponses, CreateSnippetRequest, CreateSnippetResponse, CreateTaskHandlerData, CreateTaskHandlerError, CreateTaskHandlerErrors, CreateTaskHandlerResponse, CreateTaskHandlerResponses, CreateTaskRequest, CreateTaskResponse, CreateUnthreadedAnchorRequest, CreateUnthreadedAnchorResponse, CreateUnthreadedPdfAnchorRequest, CreateViewHandlerData, CreateViewHandlerError, CreateViewHandlerErrors, CreateViewHandlerResponse, CreateViewHandlerResponses, CreateViewRequest, CreateWebhookData, CreateWebhookError, CreateWebhookErrors, CreateWebhookRequest, CreateWebhookResponse, CreateWebhookResponse2, CreateWebhookResponses, CrmComment, CrmCommentEntityType, CrmCommentThread, CrmCompanyFilters, CrmCompanyResponse, CrmContactResponse, CrmDomainResponse, CrmPermissionRole, CrmTeamSettingsResponse, CrmThread, CustomSpeakerAssignment, DataType, DeleteAnchorData, DeleteAnchorError, DeleteAnchorErrors, DeleteAnchorInfo, DeleteAnchorResponse, DeleteAnchorResponses, DeleteCallRecordData, DeleteCallRecordError, DeleteCallRecordErrors, DeleteCallRecordResponse, DeleteCallRecordResponses, DeleteChannelData, DeleteChannelError, DeleteChannelErrors, DeleteChannelResponse, DeleteChannelResponses, DeleteCommentData, DeleteCommentError, DeleteCommentErrors, DeleteCommentRequest, DeleteCommentResponse, DeleteCommentResponse2, DeleteCommentResponses, DeleteCrmCommentData, DeleteCrmCommentError, DeleteCrmCommentErrors, DeleteCrmCommentResponse, DeleteCrmCommentResponses, DeleteCrmCommentResult, DeleteDocumentData, DeleteDocumentError, DeleteDocumentErrors, DeleteDocumentResponse, DeleteDocumentResponses, DeleteEntityMentionData, DeleteEntityMentionError, DeleteEntityMentionErrors, DeleteEntityMentionResponse, DeleteEntityMentionResponse2, DeleteEntityMentionResponses, DeleteHistoryHandlerData, DeleteHistoryHandlerError, DeleteHistoryHandlerErrors, DeleteHistoryHandlerResponse, DeleteHistoryHandlerResponses, DeleteMessageData, DeleteMessageError, DeleteMessageErrors, DeleteMessageQuery, DeleteMessageResponse, DeleteMessageResponses, DeleteProjectHandlerData, DeleteProjectHandlerError, DeleteProjectHandlerErrors, DeleteProjectHandlerResponse, DeleteProjectHandlerResponses, DeleteThreadInfo, DeleteUnthreadedAnchorRequest, DeleteUnthreadedAnchorResponse, DeleteUnthreadedPdfAnchorRequest, DeleteUserDocumentViewLocationData, DeleteUserDocumentViewLocationError, DeleteUserDocumentViewLocationErrors, DeleteUserDocumentViewLocationResponse, DeleteUserDocumentViewLocationResponses, DeleteViewHandlerData, DeleteViewHandlerError, DeleteViewHandlerErrors, DeleteViewHandlerResponses, DeleteWebhookData, DeleteWebhookError, DeleteWebhookErrors, DeleteWebhookResponse, DeleteWebhookResponses, DocumentBasic, DocumentContent, DocumentContentLocation, DocumentContentState, DocumentContentUploadedMetadata, DocumentCopiedMetadata, DocumentCreatedMetadata, DocumentDeletedMetadata, DocumentFilters, DocumentInteractionMetadata, DocumentMetadata, DocumentMetadataWithContent, DocumentPermissionsToken, DocumentPermissionsTokenRequest, DocumentPermissionsTokenResponse, DocumentPreview, DocumentPreviewData, DocumentPreviewDataSubType, DocumentPurgedMetadata, DocumentResponse, DocumentResponseMetadata, DocumentResponseMetadataWithContent, DocumentStorageServiceApiVersion, DocumentSubType, DocumentSyncContentUpdatedMetadata, DocumentTeamShareResponse, DocumentTopicEvent, DocumentUpdatedMetadata, EditAnchorData, EditAnchorError, EditAnchorErrors, EditAnchorRequest, EditAnchorResponse, EditAnchorResponse2, EditAnchorResponses, EditCallRecordData, EditCallRecordError, EditCallRecordErrors, EditCallRecordRequest, EditCallRecordResponse, EditCallRecordResponses, EditCallTranscriptData, EditCallTranscriptError, EditCallTranscriptErrors, EditCallTranscriptRequest, EditCallTranscriptResponse, EditCallTranscriptResponses, EditCommentData, EditCommentError, EditCommentErrors, EditCommentRequest, EditCommentResponse, EditCommentResponse2, EditCommentResponses, EditCrmCommentData, EditCrmCommentError, EditCrmCommentErrors, EditCrmCommentRequest, EditCrmCommentResponse, EditCrmCommentResponses, EditDocumentData, EditDocumentError, EditDocumentErrors, EditDocumentResponse, EditDocumentResponse2, EditDocumentResponses, EditDocumentServiceArgs, EditPdfAnchorRequest, EditPdfPlaceableCommentAnchorRequest, EditProjectV2Data, EditProjectV2Error, EditProjectV2Errors, EditProjectV2Response, EditProjectV2Responses, EditThreadV2Data, EditThreadV2Error, EditThreadV2Errors, EditThreadV2Response, EditThreadV2Responses, EmailFilters, EmptyResponse, EntityFilters, EntityPermission, EntityPermissionResponse, EntityReference, EntityType, ErrorResponse, ExcludedDefaultView, ExcludeDefaultViewHandlerData, ExcludeDefaultViewHandlerError, ExcludeDefaultViewHandlerErrors, ExcludeDefaultViewHandlerResponses, ExcludeDefaultViewRequest, ExportDocumentData, ExportDocumentError, ExportDocumentErrors, ExportDocumentResponse, ExportDocumentResponse2, ExportDocumentResponses, Favorite, FavoriteEntityRef, FavoritesList, FileSystemNodeWithIds, FileType, FileTypeUpdate, FolderItem, ForeignEntity, ForeignEntityFilters, GenericErrorResponse, GenericResponse, GenericSuccessResponse, GetActivitiesResponse, GetActivityData, GetActivityError, GetActivityErrors, GetActivityResponse, GetActivityResponses, GetAttachmentReferencesData, GetAttachmentReferencesError, GetAttachmentReferencesErrors, GetAttachmentReferencesResponse, GetAttachmentReferencesResponse2, GetAttachmentReferencesResponses, GetBatchCallRecordPreviewData, GetBatchCallRecordPreviewError, GetBatchCallRecordPreviewErrors, GetBatchCallRecordPreviewRequest, GetBatchCallRecordPreviewResponse, GetBatchCallRecordPreviewResponse2, GetBatchCallRecordPreviewResponses, GetBatchChannelPreviewData, GetBatchChannelPreviewError, GetBatchChannelPreviewErrors, GetBatchChannelPreviewRequest, GetBatchChannelPreviewResponse, GetBatchChannelPreviewResponse2, GetBatchChannelPreviewResponses, GetBatchPreviewHandlerData, GetBatchPreviewHandlerError, GetBatchPreviewHandlerErrors, GetBatchPreviewHandlerResponse, GetBatchPreviewHandlerResponses, GetBatchPreviewRequest, GetBatchPreviewResponse, GetBatchProjectPreviewData, GetBatchProjectPreviewError, GetBatchProjectPreviewErrors, GetBatchProjectPreviewRequest, GetBatchProjectPreviewResponse, GetBatchProjectPreviewResponse2, GetBatchProjectPreviewResponses, GetCallRecordData, GetCallRecordError, GetCallRecordErrors, GetCallRecordResponse, GetCallRecordResponses, GetChannelAttachmentsData, GetChannelAttachmentsError, GetChannelAttachmentsErrors, GetChannelAttachmentsResponse, GetChannelAttachmentsResponses, GetChannelData, GetChannelError, GetChannelErrors, GetChannelJoinLinkData, GetChannelJoinLinkError, GetChannelJoinLinkErrors, GetChannelJoinLinkResponse, GetChannelJoinLinkResponses, GetChannelMessagesData, GetChannelMessagesError, GetChannelMessagesErrors, GetChannelMessagesResponse, GetChannelMessagesResponses, GetChannelParticipantsData, GetChannelParticipantsError, GetChannelParticipantsErrors, GetChannelParticipantsResponse, GetChannelParticipantsResponses, GetChannelResponse, GetChannelResponses, GetChannelsData, GetChannelsError, GetChannelsErrors, GetChannelsResponse, GetChannelsResponses, GetCompanyData, GetCompanyError, GetCompanyErrors, GetCompanyResponse, GetCompanyResponses, GetContactData, GetContactError, GetContactErrors, GetContactResponse, GetContactResponses, GetCrmTeamSettingsData, GetCrmTeamSettingsError, GetCrmTeamSettingsErrors, GetCrmTeamSettingsResponse, GetCrmTeamSettingsResponses, GetDocumentAnchorsData, GetDocumentAnchorsError, GetDocumentAnchorsErrors, GetDocumentAnchorsResponse, GetDocumentAnchorsResponses, GetDocumentBranchNameData, GetDocumentBranchNameError, GetDocumentBranchNameErrors, GetDocumentBranchNameResponse, GetDocumentBranchNameResponses, GetDocumentByTeamSlugData, GetDocumentByTeamSlugError, GetDocumentByTeamSlugErrors, GetDocumentByTeamSlugResponse, GetDocumentByTeamSlugResponses, GetDocumentCommentsData, GetDocumentCommentsError, GetDocumentCommentsErrors, GetDocumentCommentsResponse, GetDocumentCommentsResponses, GetDocumentData, GetDocumentError, GetDocumentErrors, GetDocumentGithubPullRequestsData, GetDocumentGithubPullRequestsError, GetDocumentGithubPullRequestsErrors, GetDocumentGithubPullRequestsResponse, GetDocumentGithubPullRequestsResponses, GetDocumentKeyResponse, GetDocumentKeyResponseData, GetDocumentListHandlerData, GetDocumentListHandlerError, GetDocumentListHandlerErrors, GetDocumentListHandlerResponse, GetDocumentListHandlerResponses, GetDocumentListResult, GetDocumentLocationV3Data, GetDocumentLocationV3Error, GetDocumentLocationV3Errors, GetDocumentLocationV3Response, GetDocumentLocationV3Responses, GetDocumentPermissionsResponseDataV2, GetDocumentPermissionsTokenData, GetDocumentPermissionsTokenError, GetDocumentPermissionsTokenErrors, GetDocumentPermissionsTokenResponse, GetDocumentPermissionsTokenResponses, GetDocumentPermissionsV2Data, GetDocumentPermissionsV2Error, GetDocumentPermissionsV2Errors, GetDocumentPermissionsV2Response, GetDocumentPermissionsV2Responses, GetDocumentProcessingResult, GetDocumentProcessingResultData, GetDocumentProcessingResultError, GetDocumentProcessingResultErrors, GetDocumentProcessingResultResponse, GetDocumentProcessingResultResponse2, GetDocumentProcessingResultResponses, GetDocumentResponse, GetDocumentResponse2, GetDocumentResponseData, GetDocumentResponses, GetDocumentSearchResponse, GetDocumentShortIdData, GetDocumentShortIdError, GetDocumentShortIdErrors, GetDocumentShortIdResponse, GetDocumentShortIdResponses, GetDocumentsResponse, GetDocumentTeamShareData, GetDocumentTeamShareError, GetDocumentTeamShareErrors, GetDocumentTeamShareResponse, GetDocumentTeamShareResponses, GetDocumentUserAccessLevelResponse, GetDocumentVersionData, GetDocumentVersionError, GetDocumentVersionErrors, GetDocumentVersionResponse, GetDocumentVersionResponses, GetDocumentViewsHandlerData, GetDocumentViewsHandlerError, GetDocumentViewsHandlerErrors, GetDocumentViewsHandlerResponse, GetDocumentViewsHandlerResponses, GetEntityPermissionData, GetEntityPermissionErrors, GetEntityPermissionResponse, GetEntityPermissionResponses, GetForeignEntityData, GetForeignEntityError, GetForeignEntityErrors, GetForeignEntityResponse, GetForeignEntityResponses, GetHistoryHandlerData, GetHistoryHandlerError, GetHistoryHandlerErrors, GetHistoryHandlerResponse, GetHistoryHandlerResponses, GetInstructionsDocumentResponse, GetInstructionsHandlerData, GetInstructionsHandlerError, GetInstructionsHandlerErrors, GetInstructionsHandlerResponse, GetInstructionsHandlerResponses, GetItemsSoupData, GetItemsSoupError, GetItemsSoupErrors, GetItemsSoupResponse, GetItemsSoupResponses, GetLocationHandlerData, GetLocationHandlerError, GetLocationHandlerErrors, GetLocationHandlerResponse, GetLocationHandlerResponses, GetMessageWithContextData, GetMessageWithContextError, GetMessageWithContextErrors, GetMessageWithContextResponse, GetMessageWithContextResponse2, GetMessageWithContextResponses, GetOrCreateAction, GetOrCreateCallData, GetOrCreateCallError, GetOrCreateCallErrors, GetOrCreateCallResponse, GetOrCreateCallResponses, GetOrCreateChannelResponse, GetOrCreateDmData, GetOrCreateDmError, GetOrCreateDmErrors, GetOrCreateDmRequest, GetOrCreateDmResponse, GetOrCreateDmResponses, GetOrCreatePrivateData, GetOrCreatePrivateError, GetOrCreatePrivateErrors, GetOrCreatePrivateRequest, GetOrCreatePrivateResponse, GetOrCreatePrivateResponses, GetPendingProjectsHandlerData, GetPendingProjectsHandlerError, GetPendingProjectsHandlerErrors, GetPendingProjectsHandlerResponse, GetPendingProjectsHandlerResponses, GetPinsHandlerData, GetPinsHandlerError, GetPinsHandlerErrors, GetPinsHandlerResponse, GetPinsHandlerResponses, GetPinsResponse, GetProjectContentHandlerData, GetProjectContentHandlerError, GetProjectContentHandlerErrors, GetProjectContentHandlerResponse, GetProjectContentHandlerResponses, GetProjectContentResponse, GetProjectHandlerData, GetProjectHandlerError, GetProjectHandlerErrors, GetProjectHandlerResponse, GetProjectHandlerResponses, GetProjectPermissionsV2Data, GetProjectPermissionsV2Error, GetProjectPermissionsV2Errors, GetProjectPermissionsV2Response, GetProjectPermissionsV2Responses, GetProjectResponse, GetProjectResponseData, GetProjectsHandlerData, GetProjectsHandlerError, GetProjectsHandlerErrors, GetProjectsHandlerResponse, GetProjectsHandlerResponses, GetProjectsResponse, GetProjectUserAccessLevelData, GetProjectUserAccessLevelError, GetProjectUserAccessLevelErrors, GetProjectUserAccessLevelResponse, GetProjectUserAccessLevelResponses, GetRecentActivityHandlerData, GetRecentActivityHandlerError, GetRecentActivityHandlerErrors, GetRecentActivityHandlerResponse, GetRecentActivityHandlerResponses, GetRingStatusData, GetRingStatusError, GetRingStatusErrors, GetRingStatusResponse, GetRingStatusResponses, GetThreadRepliesData, GetThreadRepliesError, GetThreadRepliesErrors, GetThreadRepliesResponse, GetThreadRepliesResponses, GetUserDocumentsHandlerData, GetUserDocumentsHandlerError, GetUserDocumentsHandlerErrors, GetUserDocumentsHandlerResponse, GetUserDocumentsHandlerResponses, GetUserDocumentViewLocationData, GetUserDocumentViewLocationError, GetUserDocumentViewLocationErrors, GetUserDocumentViewLocationResponse, GetUserDocumentViewLocationResponses, GetUserHistoryResponse, GetViewsHandlerData, GetViewsHandlerError, GetViewsHandlerErrors, GetViewsHandlerResponse, GetViewsHandlerResponses, GetWebhookData, GetWebhookError, GetWebhookErrors, GetWebhookResponse, GetWebhookResponses, GithubPullRequest, GithubPullRequestCheckRun, GithubPullRequestComment, GithubPullRequestsResponse, GroupedInitialParams, GroupedPageParams, GroupedSoupGroupPage, GroupedSoupInitialPage, GroupedSoupPage, GroupedSoupSort, HandlerData, HandlerError, HandlerErrors, HandlerResponse, HandlerResponses, HashMap, HealthHandlerData, HealthHandlerResponse, HealthHandlerResponses, HighlightType, IngestTranscriptData, IngestTranscriptError, IngestTranscriptErrors, IngestTranscriptResponses, InitializeUserDocumentsData, InitializeUserDocumentsError, InitializeUserDocumentsErrors, InitializeUserDocumentsResponse, InitializeUserDocumentsResponses, InstallSyncData, InteractionReason, Item, ItemWithUserAccessLevel, JobProcessingResultHandlerData, JobProcessingResultHandlerError, JobProcessingResultHandlerErrors, JobProcessingResultHandlerResponse, JobProcessingResultHandlerResponses, JoinChannelByCodeData, JoinChannelByCodeError, JoinChannelByCodeErrors, JoinChannelByCodeResponses, JoinChannelData, JoinChannelError, JoinChannelErrors, JoinChannelResponses, LatestMessage, LeaveCallResponse, LeaveChannelData, LeaveChannelError, LeaveChannelErrors, LeaveChannelResponses, LeaveOrEndCallData, LeaveOrEndCallError, LeaveOrEndCallErrors, LeaveOrEndCallResponse, LeaveOrEndCallResponses, ListBotChannelsData, ListBotChannelsError, ListBotChannelsErrors, ListBotChannelsResponse, ListBotChannelsResponses, ListCompanyContactsData, ListCompanyContactsError, ListCompanyContactsErrors, ListCompanyContactsResponse, ListCompanyContactsResponses, ListCrmCommentsData, ListCrmCommentsError, ListCrmCommentsErrors, ListCrmCommentsResponse, ListCrmCommentsResponses, ListFavoritesData, ListFavoritesError, ListFavoritesErrors, ListFavoritesResponse, ListFavoritesResponses, ListWebhooksData, ListWebhooksError, ListWebhooksErrors, ListWebhooksResponse, ListWebhooksResponse2, ListWebhooksResponses, LocationResponseData, LocationResponseV3, MacroUserIdStr, Mentions, NewChannelAttachment, NotificationFilters, Params, ParticipantRole, PatchChannelData, PatchChannelError, PatchChannelErrors, PatchChannelRequest, PatchChannelResponse, PatchChannelResponses, PatchMessageData, PatchMessageError, PatchMessageErrors, PatchMessageRequest, PatchMessageResponse, PatchMessageResponses, PatchProjectRequestV2, PatchThreadRequestV2, PatchViewHandlerData, PatchViewHandlerError, PatchViewHandlerErrors, PatchViewHandlerResponses, PatchWebhookData, PatchWebhookError, PatchWebhookErrors, PatchWebhookRequest, PatchWebhookResponse, PatchWebhookResponses, PdfAnchor, PdfAnchorId, PdfAnchorRequest, PdfHighlightAnchor, PdfHighlightAnchorRequest, PdfHighlightRect, PdfHighlightRectAnchorRequest, PdfPlaceableCommentAnchor, PdfPlaceableCommentAnchorRequest, PermanentlyDeleteDocumentData, PermanentlyDeleteDocumentError, PermanentlyDeleteDocumentErrors, PermanentlyDeleteDocumentResponse, PermanentlyDeleteDocumentResponses, PermanentlyDeleteProjectData, PermanentlyDeleteProjectError, PermanentlyDeleteProjectErrors, PermanentlyDeleteProjectResponse, PermanentlyDeleteProjectResponses, PinnedItem, PinRequest, PostActivityData, PostActivityError, PostActivityErrors, PostActivityRequest, PostActivityResponse, PostActivityResponses, PostChannelBotWebhookData, PostChannelBotWebhookError, PostChannelBotWebhookErrors, PostChannelBotWebhookResponse, PostChannelBotWebhookResponses, PostChannelMessagesData, PostChannelMessagesError, PostChannelMessagesErrors, PostChannelMessagesResponse, PostChannelMessagesResponses, PostGroupedSoupAstGroupPageRequest, PostGroupedSoupAstInitialRequest, PostGroupedSoupAstRequest, PostItemsSoupAstData, PostItemsSoupAstError, PostItemsSoupAstErrors, PostItemsSoupAstGroupedData, PostItemsSoupAstGroupedError, PostItemsSoupAstGroupedErrors, PostItemsSoupAstGroupedResponse, PostItemsSoupAstGroupedResponses, PostItemsSoupAstResponse, PostItemsSoupAstResponses, PostItemsSoupData, PostItemsSoupError, PostItemsSoupErrors, PostItemsSoupResponse, PostItemsSoupResponses, PostMessageData, PostMessageError, PostMessageErrors, PostMessageRequest, PostMessageResponse, PostMessageResponse2, PostMessageResponses, PostReactionData, PostReactionError, PostReactionErrors, PostReactionRequest, PostReactionResponse, PostReactionResponses, PostSoupAstRequest, PostSoupRequest, PostTypingData, PostTypingError, PostTypingErrors, PostTypingRequest, PostTypingResponse, PostTypingResponses, PresaveDocumentHandlerData, PresaveDocumentHandlerError, PresaveDocumentHandlerErrors, PresaveDocumentHandlerResponse, PresaveDocumentHandlerResponses, PreSaveDocumentRequest, PreSaveDocumentResponse, PreSaveDocumentResponseData, PresignedUrl, Project, ProjectDeleteResponseData, ProjectDocumentStatus, ProjectFilters, ProjectPreview, ProjectPreviewData, PropertyDefinition, PropertyFilter, PropertyInput, PropertyOwner, PropertyValue, PutCrmTeamSettingsData, PutCrmTeamSettingsError, PutCrmTeamSettingsErrors, PutCrmTeamSettingsResponse, PutCrmTeamSettingsResponses, ReactionAction, RecentlyDeletedData, RecentlyDeletedError, RecentlyDeletedErrors, RecentlyDeletedResponse, RecentlyDeletedResponseData, RecentlyDeletedResponses, RemoveBotFromChannelByBotData, RemoveBotFromChannelByBotError, RemoveBotFromChannelByBotErrors, RemoveBotFromChannelByBotResponse, RemoveBotFromChannelByBotResponses, RemoveFavoriteByEntityData, RemoveFavoriteByEntityError, RemoveFavoriteByEntityErrors, RemoveFavoriteByEntityResponses, RemoveParticipantsData, RemoveParticipantsError, RemoveParticipantsErrors, RemoveParticipantsRequest, RemoveParticipantsResponses, RemovePinHandlerData, RemovePinHandlerError, RemovePinHandlerErrors, RemovePinHandlerResponse, RemovePinHandlerResponses, ReorderFavoritesData, ReorderFavoritesError, ReorderFavoritesErrors, ReorderFavoritesRequest, ReorderFavoritesResponses, ReorderPinRequest, ReorderPinsHandlerData, ReorderPinsHandlerError, ReorderPinsHandlerErrors, ReorderPinsHandlerResponse, ReorderPinsHandlerResponses, ResolveChannelMessageData, ResolveChannelMessageError, ResolveChannelMessageErrors, ResolveChannelMessageResponse, ResolveChannelMessageResponses, RevertDeleteDocumentData, RevertDeleteDocumentError, RevertDeleteDocumentErrors, RevertDeleteDocumentResponse, RevertDeleteDocumentResponses, RevertDeleteProjectData, RevertDeleteProjectError, RevertDeleteProjectErrors, RevertDeleteProjectResponse, RevertDeleteProjectResponses, RingStatus, RingStatusResponse, S3ObjectInfo, SaveBomPart, SaveDocumentHandlerData, SaveDocumentHandlerError, SaveDocumentHandlerErrors, SaveDocumentHandlerResponse, SaveDocumentHandlerResponses, SaveDocumentRequest, SaveDocumentResponse, SaveDocumentResponseData, SetCompanyHiddenData, SetCompanyHiddenError, SetCompanyHiddenErrors, SetCompanyHiddenRequest, SetCompanyHiddenResponse, SetCompanyHiddenResponses, SetCompanyNameRequest, SetContactHiddenData, SetContactHiddenError, SetContactHiddenErrors, SetContactHiddenRequest, SetContactHiddenResponse, SetContactHiddenResponses, SetContactNameRequest, SetCrmCompanyNameData, SetCrmCompanyNameError, SetCrmCompanyNameErrors, SetCrmCompanyNameResponse, SetCrmCompanyNameResponses, SetCrmContactNameData, SetCrmContactNameError, SetCrmContactNameErrors, SetCrmContactNameResponse, SetCrmContactNameResponses, SetDocumentTeamShareData, SetDocumentTeamShareError, SetDocumentTeamShareErrors, SetDocumentTeamShareRequest, SetDocumentTeamShareResponse, SetDocumentTeamShareResponses, SetEmailSyncData, SetEmailSyncError, SetEmailSyncErrors, SetEmailSyncRequest, SetEmailSyncResponse, SetEmailSyncResponses, SetPropertyValue, SharedEmailFilter, SharePermissionV2, ShortIdResponse, SimpleMention, SimpleSaveData, SimpleSaveError, SimpleSaveErrors, SimpleSaveResponse, SimpleSaveResponses, SoupApiItem, SoupApiSort, SoupAttachment, SoupCallRecordParticipant, SoupCallRecordSoupPropertiesField, SoupChannel, SoupChannelThread, SoupChatSoupPropertiesField, SoupContact, SoupCountedReaction, SoupCrmCompanySoupPropertiesField, SoupCrmDomain, SoupDocumentSoupPropertiesField, SoupDocumentSubType, SoupEmailThreadPreview, SoupEnrichedEmailThreadPreviewSoupPropertiesField, SoupForeignEntity, SoupItem, SoupLabel, SoupLabelListVisibility, SoupLabelType, SoupMessageAttachment, SoupMessageListVisibility, SoupMessageSender, SoupMessageSenderType, SoupPage, SoupProjectSoupPropertiesField, SoupPropertiesField, SoupProperty, SoupThreadInfo, SoupThreadReply, StarterDocumentsResponse, String, SuccessResponse, SyncServiceVersionId, TagFilterMode, TaskFilters, TeamRole, Thread, ThreadResponse, ToggleShareWithTeamData, ToggleShareWithTeamError, ToggleShareWithTeamErrors, ToggleShareWithTeamResponse, ToggleShareWithTeamResponses, TranscriptSegmentRequest, TypedSuccessResponse, TypedSuccessResponseGetDocumentResponseData, TypingAction, UnthreadedPdfUuidRequest, UpdateChannelSharePermission, UpdateCrmTeamSettingsRequest, UpdateOperation, UpdateSharePermissionRequestV2, UploadDocumentStatus, UploadExtractFolderHandlerData, UploadExtractFolderHandlerError, UploadExtractFolderHandlerErrors, UploadExtractFolderHandlerResponse, UploadExtractFolderHandlerResponses, UploadExtractFolderRequest, UploadFolderHandlerData, UploadFolderHandlerError, UploadFolderHandlerErrors, UploadFolderHandlerResponse, UploadFolderHandlerResponses, UploadFolderRequest, UpsertHistoryHandlerData, UpsertHistoryHandlerError, UpsertHistoryHandlerErrors, UpsertHistoryHandlerResponse, UpsertHistoryHandlerResponses, UpsertUserDocumentViewLocationData, UpsertUserDocumentViewLocationError, UpsertUserDocumentViewLocationErrors, UpsertUserDocumentViewLocationRequest, UpsertUserDocumentViewLocationResponse, UpsertUserDocumentViewLocationResponses, UserActivitiesResponse, UserDocumentsResponse, UserDocumentViewLocation, UserDocumentViewLocationResponse, UserPinsResponse, UserViewsResponse, ValidateDocumentPermissionsTokenData, ValidateDocumentPermissionsTokenError, ValidateDocumentPermissionsTokenErrors, ValidateDocumentPermissionsTokenResponse, ValidateDocumentPermissionsTokenResponses, ValidateWebhookData, ValidateWebhookError, ValidateWebhookErrors, ValidateWebhookResponse, ValidateWebhookResponse2, ValidateWebhookResponses, Vec, View, ViewPatch, ViewsResponse, Webhook, WebhookEvent, WebhookFilter, WebhookScope, WebhookStatus, WebhookValidationTestEvent, WithCallId, WithChannelId, WithDocumentId, WithProjectId } from './types.gen';
diff --git a/packages/sdk/generated/storage/types.gen.ts b/packages/sdk/generated/storage/types.gen.ts
index b2e551dfab..005cf9663b 100644
--- a/packages/sdk/generated/storage/types.gen.ts
+++ b/packages/sdk/generated/storage/types.gen.ts
@@ -3186,6 +3186,30 @@ export type DocumentContentLocation = 'object_storage' | 'sync_service' | 'docx_
*/
export type DocumentContentState = 'unknown' | 'pending' | 'ready';
+/**
+ * Metadata for [`DocumentTopicEvent::ContentUploaded`].
+ */
+export type DocumentContentUploadedMetadata = {
+ /**
+ * The id of the document whose stored bytes changed.
+ */
+ document_id: string;
+ /**
+ * Version written, or the converted-file marker; `None` for unversioned
+ * writes (mirrors `SearchExtractorMessage::document_version_id`).
+ */
+ document_version_id?: string | null;
+ /**
+ * File type of the uploaded object (may differ from the document's own
+ * type, e.g. `pdf` for the converted rendition of a docx).
+ */
+ file_type: FileType;
+ /**
+ * The owner of the document (used by the extractor to resolve S3 keys).
+ */
+ owner: MacroUserIdStr;
+};
+
/**
* Metadata for [`DocumentTopicEvent::Copied`].
*/
@@ -3491,6 +3515,16 @@ export type DocumentPreviewDataSubType = {
type: 'snippet';
};
+/**
+ * Metadata for [`DocumentTopicEvent::Purged`].
+ */
+export type DocumentPurgedMetadata = {
+ /**
+ * The id of the hard-deleted document.
+ */
+ document_id: string;
+};
+
/**
* Document response with content lifecycle metadata.
*/
@@ -3593,6 +3627,24 @@ export type DocumentStorageServiceApiVersion = 'v1' | 'v2';
*/
export type DocumentSubType = 'task' | 'snippet';
+/**
+ * Metadata for [`DocumentTopicEvent::SyncContentUpdated`].
+ */
+export type DocumentSyncContentUpdatedMetadata = {
+ /**
+ * The id of the live-collab document whose content changed.
+ */
+ document_id: string;
+ /**
+ * Version marker for the sync snapshot, when the caller supplies one.
+ */
+ document_version_id?: string | null;
+ /**
+ * File type of the sync document (markdown today).
+ */
+ file_type: FileType;
+};
+
/**
* The team-share state of a document. The team is resolved from the document
* owner's team membership.
@@ -3630,6 +3682,24 @@ export type DocumentTopicEvent = {
* A document was soft-deleted.
*/
metadata: DocumentDeletedMetadata;
+} | {
+ event_type: 'document.content_uploaded';
+ /**
+ * The document's stored bytes were (re)written to S3.
+ */
+ metadata: DocumentContentUploadedMetadata;
+} | {
+ event_type: 'document.sync_content_updated';
+ /**
+ * A live-collab (sync) document's content changed and should be re-extracted.
+ */
+ metadata: DocumentSyncContentUpdatedMetadata;
+} | {
+ event_type: 'document.purged';
+ /**
+ * The document row was permanently deleted (hard delete).
+ */
+ metadata: DocumentPurgedMetadata;
} | {
event_type: 'document.copied';
/**
diff --git a/packages/sdk/specs/storage.json b/packages/sdk/specs/storage.json
index a44f839c81..5858e9f80f 100644
--- a/packages/sdk/specs/storage.json
+++ b/packages/sdk/specs/storage.json
@@ -15046,6 +15046,29 @@
"description": "API-visible content lifecycle state derived from current document metadata.",
"enum": ["unknown", "pending", "ready"]
},
+ "DocumentContentUploadedMetadata": {
+ "type": "object",
+ "description": "Metadata for [`DocumentTopicEvent::ContentUploaded`].",
+ "required": ["document_id", "owner", "file_type"],
+ "properties": {
+ "document_id": {
+ "type": "string",
+ "description": "The id of the document whose stored bytes changed."
+ },
+ "document_version_id": {
+ "type": ["string", "null"],
+ "description": "Version written, or the converted-file marker; `None` for unversioned\nwrites (mirrors `SearchExtractorMessage::document_version_id`)."
+ },
+ "file_type": {
+ "$ref": "#/components/schemas/FileType",
+ "description": "File type of the uploaded object (may differ from the document's own\ntype, e.g. `pdf` for the converted rendition of a docx)."
+ },
+ "owner": {
+ "$ref": "#/components/schemas/MacroUserIdStr",
+ "description": "The owner of the document (used by the extractor to resolve S3 keys)."
+ }
+ }
+ },
"DocumentCopiedMetadata": {
"type": "object",
"description": "Metadata for [`DocumentTopicEvent::Copied`].",
@@ -15554,6 +15577,17 @@
],
"description": "The sub type of a document preview with associated properties.\nTask-related properties are encoded within the variant to ensure valid states."
},
+ "DocumentPurgedMetadata": {
+ "type": "object",
+ "description": "Metadata for [`DocumentTopicEvent::Purged`].",
+ "required": ["document_id"],
+ "properties": {
+ "document_id": {
+ "type": "string",
+ "description": "The id of the hard-deleted document."
+ }
+ }
+ },
"DocumentResponse": {
"type": "object",
"description": "Document response with content lifecycle metadata.",
@@ -15688,6 +15722,25 @@
"description": "The document sub type enum represents all values of document sub types.\nThese values should match the `document_sub_type_value` table in macrodb.",
"enum": ["task", "snippet"]
},
+ "DocumentSyncContentUpdatedMetadata": {
+ "type": "object",
+ "description": "Metadata for [`DocumentTopicEvent::SyncContentUpdated`].",
+ "required": ["document_id", "file_type"],
+ "properties": {
+ "document_id": {
+ "type": "string",
+ "description": "The id of the live-collab document whose content changed."
+ },
+ "document_version_id": {
+ "type": ["string", "null"],
+ "description": "Version marker for the sync snapshot, when the caller supplies one."
+ },
+ "file_type": {
+ "$ref": "#/components/schemas/FileType",
+ "description": "File type of the sync document (markdown today)."
+ }
+ }
+ },
"DocumentTeamShareResponse": {
"type": "object",
"description": "The team-share state of a document. The team is resolved from the document\nowner's team membership.",
@@ -15751,6 +15804,51 @@
}
}
},
+ {
+ "type": "object",
+ "description": "The document's stored bytes were (re)written to S3.",
+ "required": ["metadata", "event_type"],
+ "properties": {
+ "event_type": {
+ "type": "string",
+ "enum": ["document.content_uploaded"]
+ },
+ "metadata": {
+ "$ref": "#/components/schemas/DocumentContentUploadedMetadata",
+ "description": "The document's stored bytes were (re)written to S3."
+ }
+ }
+ },
+ {
+ "type": "object",
+ "description": "A live-collab (sync) document's content changed and should be re-extracted.",
+ "required": ["metadata", "event_type"],
+ "properties": {
+ "event_type": {
+ "type": "string",
+ "enum": ["document.sync_content_updated"]
+ },
+ "metadata": {
+ "$ref": "#/components/schemas/DocumentSyncContentUpdatedMetadata",
+ "description": "A live-collab (sync) document's content changed and should be re-extracted."
+ }
+ }
+ },
+ {
+ "type": "object",
+ "description": "The document row was permanently deleted (hard delete).",
+ "required": ["metadata", "event_type"],
+ "properties": {
+ "event_type": {
+ "type": "string",
+ "enum": ["document.purged"]
+ },
+ "metadata": {
+ "$ref": "#/components/schemas/DocumentPurgedMetadata",
+ "description": "The document row was permanently deleted (hard delete)."
+ }
+ }
+ },
{
"type": "object",
"description": "A document was copied.",
diff --git a/services/deleted_item_poller/Cargo.toml b/services/deleted_item_poller/Cargo.toml
index 1280e61cb1..9830e145f2 100644
--- a/services/deleted_item_poller/Cargo.toml
+++ b/services/deleted_item_poller/Cargo.toml
@@ -12,6 +12,7 @@ aws-smithy-types = { workspace = true }
aws_lambda_events = { workspace = true, features = ["eventbridge"] }
chat = { path = "../../crates/chat", default-features = false }
chrono = { workspace = true }
+documents = { path = "../../crates/documents", default-features = false }
futures = { workspace = true }
lambda_runtime = { workspace = true }
macro_db_client = { path = "../../crates/macro_db_client" }
@@ -30,7 +31,6 @@ sqlx = { workspace = true }
sqs_client = { path = "../../crates/sqs_client", default-features = false, features = [
"chat",
"document",
- "search",
] }
tokio = { workspace = true }
tracing = { workspace = true }
diff --git a/services/deleted_item_poller/src/config.rs b/services/deleted_item_poller/src/config.rs
index ee3db73e7b..0e0341b183 100644
--- a/services/deleted_item_poller/src/config.rs
+++ b/services/deleted_item_poller/src/config.rs
@@ -13,7 +13,7 @@ pub struct Config {
/// The connection URL for the Postgres database this application should use.
pub database_url: String,
- /// Comma-separated Kafka bootstrap servers for project and chat lifecycle events.
+ /// Comma-separated Kafka bootstrap servers for document, project, and chat lifecycle events.
pub kafka_brokers: KafkaBrokers,
/// The environment we are in
diff --git a/services/deleted_item_poller/src/handler.rs b/services/deleted_item_poller/src/handler.rs
index 5fe629c4cf..4c83b94f89 100644
--- a/services/deleted_item_poller/src/handler.rs
+++ b/services/deleted_item_poller/src/handler.rs
@@ -5,6 +5,7 @@ use crate::context::{self};
use anyhow::Context;
use aws_lambda_events::eventbridge::EventBridgeEvent;
use chat::domain::events::{ChatMacroEvent, ChatPermanentlyDeletedMetadata};
+use documents::domain::events::{DocumentMacroEvent, DocumentPurgedMetadata};
use futures::future::join_all;
use lambda_runtime::{
Error, LambdaEvent,
@@ -14,7 +15,6 @@ use macro_db_client::projects::ProjectToDelete;
use macro_event_broker::MacroEventBroker;
use macro_user_id::user_id::MacroUserIdStr;
use projects::domain::events::{ProjectMacroEvent, ProjectPermanentlyDeletedMetadata};
-use sqs_client::search::{SearchQueueMessage, document::DocumentId};
#[tracing::instrument(skip(ctx, _event), err)]
pub async fn handler(
@@ -30,6 +30,43 @@ pub async fn handler(
Ok(())
}
+#[tracing::instrument(skip(event_broker, document_ids), err)]
+async fn publish_document_purge_events<B: MacroEventBroker>(
+ event_broker: &B,
+ document_ids: &[String],
+) -> anyhow::Result<()> {
+ let events = document_ids
+ .iter()
+ .map(|document_id| {
+ DocumentMacroEvent::purged(
+ document_id.clone(),
+ DocumentPurgedMetadata {
+ document_id: document_id.clone(),
+ },
+ )
+ })
+ .collect::<Vec<_>>();
+
+ let publications = events
+ .iter()
+ .map(|event| event_broker.send_event(event))
+ .collect::<Vec<_>>();
+ let publication_results = join_all(publications.into_iter().map(|publication| async move {
+ let handle = publication.context("failed to enqueue document purge event")?;
+ handle
+ .await
+ .context("document purge event publication task failed")?
+ .context("failed to publish document purge event")
+ }))
+ .await;
+
+ for result in publication_results {
+ result?;
+ }
+
+ Ok(())
+}
+
#[tracing::instrument(skip(event_broker, projects_to_delete), err)]
async fn publish_project_purge_events<B: MacroEventBroker>(
event_broker: &B,
@@ -184,18 +221,9 @@ async fn handle_documents(ctx: &context::Context) -> anyhow::Result<()> {
tracing::debug!(documents_to_delete=?documents_to_delete, "documents to delete");
- ctx.sqs_client
- .bulk_send_message_to_search_event_queue(
- documents_to_delete
- .iter()
- .map(|id| {
- SearchQueueMessage::RemoveDocument(DocumentId {
- document_id: id.to_string(),
- })
- })
- .collect(),
- )
- .await?;
+ publish_document_purge_events(&ctx.macro_event_broker, &documents_to_delete)
+ .await
+ .context("unable to publish document purge events")?;
ctx.sqs_client
.bulk_enqueue_document_delete(documents_to_delete)
diff --git a/services/deleted_item_poller/src/handler/test.rs b/services/deleted_item_poller/src/handler/test.rs
index 10f77529c0..55791e2c81 100644
--- a/services/deleted_item_poller/src/handler/test.rs
+++ b/services/deleted_item_poller/src/handler/test.rs
@@ -93,10 +93,11 @@ impl MacroEventBroker for FakeEventBroker {
handle
}
DeliveryBehavior::Wait(delivery_gate) => tokio::spawn(async move {
- let _permit = delivery_gate
+ let permit = delivery_gate
.acquire_owned()
.await
.expect("delivery gate should remain open");
+ permit.forget();
Ok(())
}),
};
@@ -105,6 +106,95 @@ impl MacroEventBroker for FakeEventBroker {
}
}
+#[tokio::test]
+async fn publish_document_purge_events_publishes_separately_keyed_events() {
+ let event_broker = FakeEventBroker::default();
+ let document_ids = vec!["document-one".to_string(), "document-two".to_string()];
+
+ publish_document_purge_events(&event_broker, &document_ids)
+ .await
+ .unwrap();
+
+ let published = event_broker.published();
+ assert_eq!(published.len(), document_ids.len());
+
+ for (event, document_id) in published.iter().zip(&document_ids) {
+ assert_eq!(event.topic, "macro.documents");
+ assert_eq!(event.key, *document_id);
+ assert_eq!(event.envelope["schema_version"], json!(1));
+ assert_eq!(event.envelope["event_type"], json!("document.purged"));
+ assert_eq!(
+ event.envelope["metadata"]["document_id"],
+ json!(document_id)
+ );
+ }
+}
+
+#[tokio::test]
+async fn publish_document_purge_events_returns_immediate_send_failures() {
+ let event_broker = FakeEventBroker::failing_send();
+ let document_ids = vec!["document-one".to_string()];
+
+ let error = publish_document_purge_events(&event_broker, &document_ids)
+ .await
+ .expect_err("immediate send failure should be returned");
+
+ assert!(format!("{error:#}").contains("event enqueue rejected"));
+}
+
+#[tokio::test]
+async fn publish_document_purge_events_returns_delivery_failures() {
+ let event_broker = FakeEventBroker::failing_delivery();
+ let document_ids = vec!["document-one".to_string()];
+
+ let error = publish_document_purge_events(&event_broker, &document_ids)
+ .await
+ .expect_err("delivery failure should be returned");
+
+ assert!(format!("{error:#}").contains("publisher unavailable"));
+}
+
+#[tokio::test]
+async fn publish_document_purge_events_returns_delivery_join_failures() {
+ let event_broker = FakeEventBroker::failing_join();
+ let document_ids = vec!["document-one".to_string()];
+
+ let error = publish_document_purge_events(&event_broker, &document_ids)
+ .await
+ .expect_err("delivery join failure should be returned");
+
+ assert!(
+ format!("{error:#}").contains("document purge event publication task failed"),
+ "unexpected error: {error:#}"
+ );
+}
+
+#[tokio::test]
+async fn publish_document_purge_events_waits_for_every_delivery() {
+ let delivery_gate = Arc::new(Semaphore::new(0));
+ let event_broker = FakeEventBroker::waiting_for_delivery(delivery_gate.clone());
+ let document_ids = vec!["document-one".to_string(), "document-two".to_string()];
+
+ let publication_task = tokio::spawn({
+ let event_broker = event_broker.clone();
+ let document_ids = document_ids.clone();
+ async move { publish_document_purge_events(&event_broker, &document_ids).await }
+ });
+
+ while event_broker.published().len() < document_ids.len() {
+ tokio::task::yield_now().await;
+ }
+
+ assert!(!publication_task.is_finished());
+
+ delivery_gate.add_permits(1);
+ tokio::task::yield_now().await;
+ assert!(!publication_task.is_finished());
+
+ delivery_gate.add_permits(1);
+ publication_task.await.unwrap().unwrap();
+}
+
#[tokio::test]
async fn publish_project_purge_events_publishes_separately_keyed_events() {
let event_broker = FakeEventBroker::default();
diff --git a/services/deleted_item_poller/src/main.rs b/services/deleted_item_poller/src/main.rs
index e4022b7175..c8aa575d6a 100644
--- a/services/deleted_item_poller/src/main.rs
+++ b/services/deleted_item_poller/src/main.rs
@@ -41,13 +41,11 @@ async fn main() -> Result<(), Error> {
let document_delete_queue = macro_queues::DocumentDeleteQueue::new();
let chat_delete_queue = macro_queues::ChatDeleteQueue::new();
- let search_event_queue = macro_queues::SearchEventQueue::new();
let sqs_client = sqs_client::SQS::new(aws_sdk_sqs::Client::new(
&macro_aws_config::get_macro_aws_config().await,
))
.document_delete_queue(&document_delete_queue)
- .chat_delete_queue(&chat_delete_queue)
- .search_event_queue(&search_event_queue);
+ .chat_delete_queue(&chat_delete_queue);
let ctx = context::Context {
db,
diff --git a/services/document_storage_service/src/api/context.rs b/services/document_storage_service/src/api/context.rs
index 9f2ab31b72..c77c69d331 100644
--- a/services/document_storage_service/src/api/context.rs
+++ b/services/document_storage_service/src/api/context.rs
@@ -286,7 +286,7 @@ pub(crate) type ProjectService = ProjectServiceImpl<
DynamoBulkUploadAdapter,
ShaCountAdapter,
EntityAccessManagementService,
- SqsProjectSearchIndexer,
+ SqsProjectSearchIndexer<DssEventBroker>,
DssEventBroker,
>;
@@ -391,7 +391,7 @@ pub(crate) type DssEntityMutationService =
ProjectService,
EntityAccessService,
FavoritesServiceType,
- crate::outbound::entity_mutation::DssEntityLifecycleAdapter,
+ crate::outbound::entity_mutation::DssEntityLifecycleAdapter<DssEventBroker>,
>;
/// Type alias for the favorites service.
diff --git a/services/document_storage_service/src/api/documents/delete_document.rs b/services/document_storage_service/src/api/documents/delete_document.rs
index f61c14b713..71764120f0 100644
--- a/services/document_storage_service/src/api/documents/delete_document.rs
+++ b/services/document_storage_service/src/api/documents/delete_document.rs
@@ -1,6 +1,7 @@
use crate::api::context::ApiContext;
use crate::api::context::{AuthorizationService, EntityAccessService};
use crate::api::util::count_occurrences;
+use crate::service::document_event_publisher::publish_document_purged_event;
use axum::Json;
use axum::extract::State;
use axum::response::Response;
@@ -15,7 +16,6 @@ use model::response::{
};
use models_permissions::share_permission::access_level::OwnerAccessLevel;
use serde::Deserialize;
-use sqs_client::search::{SearchQueueMessage, document::DocumentId};
#[derive(Deserialize)]
pub struct Params {
@@ -130,22 +130,16 @@ pub async fn permanently_delete_document_handler(
.into_response()
})?;
- state
- .sqs_client
- .send_message_to_search_event_queue(SearchQueueMessage::RemoveDocument(DocumentId {
- document_id,
- }))
- .await
- .map_err(|e| {
- tracing::error!(error=?e, "unable to send message to search extractor queue");
- (
- StatusCode::INTERNAL_SERVER_ERROR,
- Json(ErrorResponse {
- message: "unable to send message to search extractor queue".into(),
- }),
- )
- .into_response()
- })?;
+ publish_document_purged_event(&state.macro_event_broker, &document_id).map_err(|e| {
+ tracing::error!(error=?e, "unable to publish document purged event");
+ (
+ StatusCode::INTERNAL_SERVER_ERROR,
+ Json(ErrorResponse {
+ message: "unable to publish document purged event".into(),
+ }),
+ )
+ .into_response()
+ })?;
let response_data = GenericSuccessResponse { success: true };
diff --git a/services/document_storage_service/src/api/internal/mod.rs b/services/document_storage_service/src/api/internal/mod.rs
index 3d3ba747d3..f1d3bd8ba6 100644
--- a/services/document_storage_service/src/api/internal/mod.rs
+++ b/services/document_storage_service/src/api/internal/mod.rs
@@ -58,6 +58,16 @@ pub fn router(state: ApiContext) -> Router<ApiContext> {
get(get_document::get_document_basic_handler)
.layer(ensure_document_exists_middleware.clone()),
)
+ .route(
+ "/documents/{document_id}/content-uploaded",
+ post(
+ documents_hex::inbound::axum_router::content_uploaded::content_uploaded_handler::<
+ DocumentService,
+ EntityAccessService,
+ AuthorizationService,
+ >,
+ ),
+ )
.route(
"/documents/{document_id}/export",
get(export_document::handler),
diff --git a/services/document_storage_service/src/main.rs b/services/document_storage_service/src/main.rs
index 13cdbf1773..ebb12c4026 100644
--- a/services/document_storage_service/src/main.rs
+++ b/services/document_storage_service/src/main.rs
@@ -470,7 +470,7 @@ async fn main() -> anyhow::Result<()> {
DynamoBulkUploadAdapter::new(dynamodb_client.clone()),
ShaCountAdapter::new(Redis::new(redis_client.clone())),
entity_access_management_service.clone(),
- SqsProjectSearchIndexer::new(Arc::new(sqs_client.clone())),
+ SqsProjectSearchIndexer::new(Arc::new(sqs_client.clone()), macro_event_broker.clone()),
if cfg!(feature = "local") {
Some(uuid::uuid!("d50676e2-0a12-4c62-bc07-4b1cb6d8e9bc"))
} else {
@@ -479,28 +479,21 @@ async fn main() -> anyhow::Result<()> {
macro_event_broker.clone(),
));
- let document_service = Arc::new(
- DocumentServiceImpl::new(
- document_repo,
- cloudfront_config,
- sync_service_client.as_ref().clone(),
- s3_upload_adapter,
- TaskPropertiesAdapter {
- system_properties: system_properties_service.clone(),
- properties: properties_service.clone(),
- entity_access_service: entity_access_service.clone(),
- },
- connection_service,
- entity_access_management_service.clone(),
- ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),
- macro_event_broker.clone(),
- )
- .with_search_indexer(Arc::new(
- crate::service::document_search_indexer::SqsDocumentSearchIndexer::new(
- sqs_client.clone(),
- ),
- )),
- );
+ let document_service = Arc::new(DocumentServiceImpl::new(
+ document_repo,
+ cloudfront_config,
+ sync_service_client.as_ref().clone(),
+ s3_upload_adapter,
+ TaskPropertiesAdapter {
+ system_properties: system_properties_service.clone(),
+ properties: properties_service.clone(),
+ entity_access_service: entity_access_service.clone(),
+ },
+ connection_service,
+ entity_access_management_service.clone(),
+ ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),
+ macro_event_broker.clone(),
+ ));
let foreign_entity_service = Arc::new(ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(
db.clone(),
@@ -1010,6 +1003,7 @@ async fn main() -> anyhow::Result<()> {
db.clone(),
redis_sha_client.clone(),
sqs_client.clone(),
+ macro_event_broker.clone(),
)),
));
diff --git a/services/document_storage_service/src/outbound/entity_mutation.rs b/services/document_storage_service/src/outbound/entity_mutation.rs
index 365cf905b4..0c6d2feac4 100644
--- a/services/document_storage_service/src/outbound/entity_mutation.rs
+++ b/services/document_storage_service/src/outbound/entity_mutation.rs
@@ -5,14 +5,17 @@ use std::sync::Arc;
use crate::{
api::util::count_occurrences,
- service::entity_mutation::{EntityLifecycleService, LifecycleError},
+ service::{
+ document_event_publisher::publish_document_purged_event,
+ entity_mutation::{EntityLifecycleService, LifecycleError},
+ },
};
use entity_mutation::EntityMutationActor;
+use macro_event_broker::MacroEventBroker;
use macro_sha_count_client::Redis;
use model_entity::{Entity, EntityType};
use models_permissions::share_permission::UpdateSharePermissionRequestV2;
use sqlx::PgPool;
-use sqs_client::search::{SearchQueueMessage, document::DocumentId};
/// Wrap a legacy client failure as an internal lifecycle error.
macro_rules! internal {
@@ -30,20 +33,26 @@ fn row_error(error: sqlx::Error) -> LifecycleError {
}
/// Production lifecycle adapter backed by the legacy persistence clients.
-pub struct DssEntityLifecycleAdapter {
+pub struct DssEntityLifecycleAdapter<B: MacroEventBroker> {
db: PgPool,
redis: Arc<Redis>,
sqs: Arc<sqs_client::SQS>,
+ event_broker: B,
}
-impl DssEntityLifecycleAdapter {
+impl<B: MacroEventBroker> DssEntityLifecycleAdapter<B> {
/// Construct the adapter from concrete outbound dependencies.
- pub fn new(db: PgPool, redis: Arc<Redis>, sqs: Arc<sqs_client::SQS>) -> Self {
- Self { db, redis, sqs }
+ pub fn new(db: PgPool, redis: Arc<Redis>, sqs: Arc<sqs_client::SQS>, event_broker: B) -> Self {
+ Self {
+ db,
+ redis,
+ sqs,
+ event_broker,
+ }
}
}
-impl EntityLifecycleService for DssEntityLifecycleAdapter {
+impl<B: MacroEventBroker> EntityLifecycleService for DssEntityLifecycleAdapter<B> {
async fn update_thread_share_policy(
&self,
_actor: &EntityMutationActor,
@@ -133,11 +142,7 @@ impl EntityLifecycleService for DssEntityLifecycleAdapter {
.enqueue_document_delete(document.owner.as_ref(), &entity.entity_id)
.await
.map_err(|error| internal!(error))?;
- self.sqs
- .send_message_to_search_event_queue(SearchQueueMessage::RemoveDocument(DocumentId {
- document_id: entity.entity_id.to_string(),
- }))
- .await
+ publish_document_purged_event(&self.event_broker, &entity.entity_id)
.map_err(|error| internal!(error))?;
Ok(document
.project_id
diff --git a/services/document_storage_service/src/service/document_event_publisher.rs b/services/document_storage_service/src/service/document_event_publisher.rs
new file mode 100644
index 0000000000..acbf87687e
--- /dev/null
+++ b/services/document_storage_service/src/service/document_event_publisher.rs
@@ -0,0 +1,19 @@
+#[cfg(test)]
+mod test;
+
+use documents_hex::domain::events::{DocumentMacroEvent, DocumentPurgedMetadata};
+use macro_event_broker::{EventBrokerError, MacroEventBroker};
+
+/// Schedules a document-purged event for asynchronous broker delivery.
+#[tracing::instrument(skip(event_broker), err)]
+pub(crate) fn publish_document_purged_event<B: MacroEventBroker>(
+ event_broker: &B,
+ document_id: &str,
+) -> Result<(), EventBrokerError> {
+ let document_id = document_id.to_owned();
+ let event =
+ DocumentMacroEvent::purged(document_id.clone(), DocumentPurgedMetadata { document_id });
+
+ drop(event_broker.send_event(&event)?);
+ Ok(())
+}
diff --git a/services/document_storage_service/src/service/document_event_publisher/test.rs b/services/document_storage_service/src/service/document_event_publisher/test.rs
new file mode 100644
index 0000000000..cb83de8086
--- /dev/null
+++ b/services/document_storage_service/src/service/document_event_publisher/test.rs
@@ -0,0 +1,79 @@
+use std::sync::Mutex;
+
+use macro_event_broker::{EventBrokerError, MacroEvent, MacroEventBroker};
+use serde_json::{Value, json};
+use uuid::Uuid;
+
+use super::*;
+
+#[derive(Debug, PartialEq)]
+struct PublishedEvent {
+ topic: String,
+ key: String,
+ envelope: Value,
+}
+
+#[derive(Default)]
+struct RecordingEventBroker {
+ published: Mutex<Vec<PublishedEvent>>,
+}
+
+impl MacroEventBroker for RecordingEventBroker {
+ fn send_event<E: MacroEvent + ?Sized>(
+ &self,
+ event: &E,
+ ) -> Result<tokio::task::JoinHandle<Result<(), EventBrokerError>>, EventBrokerError> {
+ self.published.lock().unwrap().push(PublishedEvent {
+ topic: event.topic().to_owned(),
+ key: event.key().to_owned(),
+ envelope: serde_json::to_value(event.event())?,
+ });
+
+ Ok(tokio::spawn(async { Ok(()) }))
+ }
+}
+
+struct FailingEventBroker;
+
+impl MacroEventBroker for FailingEventBroker {
+ fn send_event<E: MacroEvent + ?Sized>(
+ &self,
+ _event: &E,
+ ) -> Result<tokio::task::JoinHandle<Result<(), EventBrokerError>>, EventBrokerError> {
+ Err(EventBrokerError::Publish(
+ "event enqueue rejected".to_string(),
+ ))
+ }
+}
+
+#[tokio::test]
+async fn publishes_document_purged_event_to_documents_topic() {
+ let event_broker = RecordingEventBroker::default();
+
+ publish_document_purged_event(&event_broker, "document-one").unwrap();
+
+ let published = event_broker.published.lock().unwrap();
+ assert_eq!(published.len(), 1);
+
+ let event = &published[0];
+ assert_eq!(event.topic, "macro.documents");
+ assert_eq!(event.key, "document-one");
+ assert!(Uuid::parse_str(event.envelope["event_id"].as_str().unwrap()).is_ok());
+ assert_eq!(event.envelope["schema_version"], json!(1));
+ assert_eq!(event.envelope["event_type"], json!("document.purged"));
+ assert_eq!(
+ event.envelope["metadata"],
+ json!({ "document_id": "document-one" })
+ );
+}
+
+#[test]
+fn returns_immediate_broker_failures() {
+ let error = publish_document_purged_event(&FailingEventBroker, "document-one")
+ .expect_err("immediate broker failure should be returned");
+
+ assert!(matches!(
+ error,
+ EventBrokerError::Publish(message) if message == "event enqueue rejected"
+ ));
+}
diff --git a/services/document_storage_service/src/service/document_search_indexer.rs b/services/document_storage_service/src/service/document_search_indexer.rs
deleted file mode 100644
index 34a07193c1..0000000000
--- a/services/document_storage_service/src/service/document_search_indexer.rs
+++ /dev/null
@@ -1,36 +0,0 @@
-//! Adapter that publishes document name refreshes to the search queue.
-
-use documents_hex::domain::ports::DocumentSearchIndexer;
-use sqs_client::SQS;
-use sqs_client::search::SearchQueueMessage;
-use sqs_client::search::document::DocumentId;
-
-/// Publishes a refresh of a document's denormalized name to the shared search
-/// event queue so the search-processing service updates the indexed
-/// `document_name` after a rename.
-#[derive(Debug)]
-pub struct SqsDocumentSearchIndexer {
- sqs: SQS,
-}
-
-impl SqsDocumentSearchIndexer {
- pub fn new(sqs: SQS) -> Self {
- Self { sqs }
- }
-}
-
-impl DocumentSearchIndexer for SqsDocumentSearchIndexer {
- fn enqueue_name_update(
- &self,
- document_id: String,
- ) -> std::pin::Pin<Box<dyn std::future::Future<Output = anyhow::Result<()>> + Send>> {
- let sqs = self.sqs.clone();
- Box::pin(async move {
- sqs.send_message_to_search_event_queue(SearchQueueMessage::UpdateDocumentName(
- DocumentId { document_id },
- ))
- .await?;
- Ok(())
- })
- }
-}
diff --git a/services/document_storage_service/src/service/mod.rs b/services/document_storage_service/src/service/mod.rs
index 1d71089da3..f4d84f1703 100644
--- a/services/document_storage_service/src/service/mod.rs
+++ b/services/document_storage_service/src/service/mod.rs
@@ -1,7 +1,7 @@
pub mod conn_gateway;
#[cfg(feature = "delete_document_worker")]
pub mod delete_document_worker;
-pub mod document_search_indexer;
+pub mod document_event_publisher;
pub mod entity_mutation;
pub mod property_search_indexer;
pub mod s3;
diff --git a/services/mcp_service/src/context.rs b/services/mcp_service/src/context.rs
index aef036ace6..86c5365809 100644
--- a/services/mcp_service/src/context.rs
+++ b/services/mcp_service/src/context.rs
@@ -261,9 +261,6 @@ async fn build_tool_context(args: ToolContextBuildArgs<'_>) -> anyhow::Result<To
),
foreign_entity_service: ForeignEntityServiceImpl::new(PgForeignEntityRepo::new(db.clone())),
macro_event_broker: macro_event_broker.clone(),
- // No search event queue is configured in this context, so no
- // search-index refresh is published from here.
- search_indexer: None,
};
let lexical_client_for_tools = (*lexical_client).clone();
let document_tool_context = DocumentToolContext::new(
diff --git a/services/search_processing_service/Cargo.toml b/services/search_processing_service/Cargo.toml
index 4b3957c742..1e7075884e 100644
--- a/services/search_processing_service/Cargo.toml
+++ b/services/search_processing_service/Cargo.toml
@@ -11,6 +11,7 @@ processing = [
"dep:call",
"dep:channels",
"dep:chat",
+ "dep:documents",
"dep:kafka_util",
"dep:macro_event_broker",
"dep:macro_event_topics",
@@ -19,7 +20,13 @@ processing = [
"dep:rootcause",
"dep:tokio-retry",
]
-service = ["macro_db_client", "models_properties", "properties"]
+service = [
+ "dep:documents",
+ "dep:macro_event_broker",
+ "macro_db_client",
+ "models_properties",
+ "properties",
+]
pdf = ["dep:pdfium-render"]
[[bin]]
@@ -66,6 +73,7 @@ macro_user_id = { path = "../../crates/macro_user_id" }
macro_uuid = { path = "../../crates/macro_uuid" }
mention_utils = { path = "../../crates/mention_utils" }
document_sub_type = { path = "../../crates/document_sub_type" }
+documents = { path = "../../crates/documents", default-features = false, optional = true }
doppler-config = { workspace = true }
model = { path = "../../crates/model" }
s3_key = { path = "../../crates/s3_key" }
@@ -97,7 +105,7 @@ strum = { workspace = true }
thiserror = { workspace = true }
tokio = { workspace = true, features = ["signal", "sync", "time"] }
tokio-retry = { workspace = true, optional = true }
-tokio-util = { workspace = true }
+tokio-util = { workspace = true, features = ["rt"] }
tower = { workspace = true }
tower-http = { workspace = true }
tracing = { workspace = true }
diff --git a/services/search_processing_service/src/api/context.rs b/services/search_processing_service/src/api/context.rs
index 427b1e81ff..24b5dd085b 100644
--- a/services/search_processing_service/src/api/context.rs
+++ b/services/search_processing_service/src/api/context.rs
@@ -2,21 +2,24 @@ use axum::extract::FromRef;
use macro_authorization::{
MacroAuthorizationServiceImpl, MacroAuthorizationState, NoopMacroAuthJwtValidator,
};
+use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};
use std::sync::Arc;
+use tokio_util::task::TaskTracker;
use crate::BackfillServiceImpl;
use crate::config::Config;
use crate::domain::jobs::BackfillJobs;
pub(crate) type AuthorizationService = MacroAuthorizationServiceImpl<NoopMacroAuthJwtValidator>;
+pub(crate) type SpsEventBroker = MacroEventBrokerService<KafkaEventPublisher, TaskTracker>;
#[derive(Clone, FromRef)]
pub(crate) struct ApiContext {
pub db: sqlx::Pool<sqlx::Postgres>,
- pub sqs_client: Arc<sqs_client::SQS>,
pub opensearch_client: Arc<opensearch_client::OpensearchClient>,
pub authorization_state: MacroAuthorizationState<AuthorizationService>,
pub config: Arc<Config>,
pub backfill_service: Arc<BackfillServiceImpl>,
pub backfill_jobs: BackfillJobs,
+ pub macro_event_broker: SpsEventBroker,
}
diff --git a/services/search_processing_service/src/api/internal/extract_sync.rs b/services/search_processing_service/src/api/internal/extract_sync.rs
index 3ae6f7a95f..b7c1d3c335 100644
--- a/services/search_processing_service/src/api/internal/extract_sync.rs
+++ b/services/search_processing_service/src/api/internal/extract_sync.rs
@@ -4,10 +4,13 @@ use axum::{
http::StatusCode,
response::{IntoResponse, Response},
};
+use documents::domain::events::{DocumentMacroEvent, DocumentSyncContentUpdatedMetadata};
use macro_authorization::{InternalOnly, MacroAuthorizationExtractor};
+use macro_event_broker::MacroEventBroker as _;
use model::document::FileType;
-use sqs_client::search::{SearchQueueMessage, document::SearchExtractorMessage};
-use uuid::Uuid;
+
+#[cfg(test)]
+mod test;
#[derive(serde::Deserialize, serde::Serialize)]
pub struct SyncDocument {
@@ -23,32 +26,24 @@ pub struct ExtractSyncRequest {
pub documents: Vec<SyncDocument>,
}
-fn documents_to_messages(documents: Vec<SyncDocument>) -> Vec<SearchQueueMessage> {
+fn documents_to_events(documents: Vec<SyncDocument>) -> Vec<DocumentMacroEvent> {
documents
- .iter()
- .map(|doc| {
- // generate random UUIDv4 for DeduplicationId trait
- let document_version_id = if let Some(id) = &doc.document_version_id {
- Some(id.clone())
- } else {
- Some(Uuid::new_v4().to_string())
- };
- SearchQueueMessage::ExtractSync(SearchExtractorMessage {
- // HACK: we can leave this empty because we know that this code path doesn't use it
- // it is mainly needed for get_document_info()
- user_id: "".to_string(),
- document_id: doc.document_id.clone(),
- // NOTE: for now, we only expect markdown files from sync service. this could
- // change
- file_type: doc.file_type,
- document_version_id,
- index_override: None,
- })
+ .into_iter()
+ .map(|document| {
+ let document_id = document.document_id;
+ DocumentMacroEvent::sync_content_updated(
+ document_id.clone(),
+ DocumentSyncContentUpdatedMetadata {
+ document_id,
+ file_type: document.file_type,
+ document_version_id: document.document_version_id,
+ },
+ )
})
.collect()
}
-/// internal handler to batch queue messages to the worker queue
+/// Internal handler to publish sync-content extraction events.
#[tracing::instrument(skip(ctx, _internal_authorization, req))]
pub async fn handler(
State(ctx): State<ApiContext>,
@@ -61,87 +56,17 @@ pub async fn handler(
.map(|d| d.document_id.as_str())
.collect();
tracing::info!(?document_ids, "extract_sync received");
- let messages: Vec<SearchQueueMessage> = documents_to_messages(req.documents);
- ctx.sqs_client
- .bulk_send_message_to_search_event_queue(messages)
- .await
- .map_err(|e| {
- tracing::error!(error=?e, "failed to enqueue documents");
+ let events = documents_to_events(req.documents);
+ for event in events {
+ drop(ctx.macro_event_broker.send_event(&event).map_err(|error| {
+ tracing::error!(error=?error, "failed to publish document sync event");
(
StatusCode::INTERNAL_SERVER_ERROR,
"failed to enqueue documents",
)
.into_response()
- })?;
- Ok(StatusCode::OK.into_response())
-}
-
-#[cfg(test)]
-mod tests {
- use super::*;
-
- #[test]
- fn test_documents_to_messsages() {
- let documents: Vec<SyncDocument> = [
- ["AAA", "macro|[email protected]", "md"],
- ["CCC", "macro|[email protected]", "oops_not_a_file_format"],
- ["BBB", "macro|[email protected]", "md"],
- ]
- .iter()
- .map(|v| SyncDocument {
- document_id: v[0].to_string(),
- file_type: FileType::Md,
- document_version_id: None,
- })
- .collect();
- let messages = documents_to_messages(documents);
- assert_eq!(messages.len(), 3);
- let messages: Vec<SearchExtractorMessage> = messages
- .into_iter()
- .filter_map(|m| match m {
- SearchQueueMessage::ExtractSync(body) => Some(body),
- _ => None,
- })
- .collect();
- assert_eq!(messages.len(), 3);
- assert_eq!(&messages[0].document_id, "AAA");
- assert_eq!(&messages[2].document_id, "BBB");
+ })?);
}
- // Should be able to set it explicitly or have it auto-generated
- #[test]
- fn test_document_version_id() {
- let documents: Vec<SyncDocument> = [
- ["AAA", "macro|[email protected]", "md", "zzz"],
- ["BBB", "macro|[email protected]", "md", ""],
- ]
- .iter()
- .map(|v| SyncDocument {
- document_id: v[0].to_string(),
- //user_id: v[1].to_string(),
- file_type: FileType::Md,
- document_version_id: if v[3].is_empty() {
- None
- } else {
- Some(v[3].to_string())
- },
- })
- .collect();
- let messages = documents_to_messages(documents);
- assert_eq!(messages.len(), 2);
- let messages: Vec<SearchExtractorMessage> = messages
- .into_iter()
- .filter_map(|m| match m {
- SearchQueueMessage::ExtractSync(body) => Some(body),
- _ => None,
- })
- .collect();
- assert_eq!(messages.len(), 2);
- assert!(&messages[0].document_version_id.is_some());
- assert!(&messages[1].document_version_id.is_some());
-
- if let Some(id) = &messages[0].document_version_id {
- assert_eq!(id, "zzz");
- }
- }
+ Ok(StatusCode::OK.into_response())
}
diff --git a/services/search_processing_service/src/api/internal/extract_sync/test.rs b/services/search_processing_service/src/api/internal/extract_sync/test.rs
new file mode 100644
index 0000000000..f1a75ccc78
--- /dev/null
+++ b/services/search_processing_service/src/api/internal/extract_sync/test.rs
@@ -0,0 +1,58 @@
+use documents::domain::events::{DocumentSyncContentUpdatedMetadata, DocumentTopicEvent};
+use macro_event_broker::MacroEvent as _;
+use model::document::FileType;
+
+use super::{SyncDocument, documents_to_events};
+
+#[test]
+fn documents_map_to_events_with_matching_keys_and_exact_metadata() {
+ let events = documents_to_events(vec![
+ SyncDocument {
+ document_id: "document-a".to_string(),
+ document_version_id: Some("version-42".to_string()),
+ file_type: FileType::Md,
+ },
+ SyncDocument {
+ document_id: "document-b".to_string(),
+ document_version_id: None,
+ file_type: FileType::Pdf,
+ },
+ ]);
+
+ assert_eq!(events.len(), 2);
+ assert_eq!(events[0].key(), "document-a");
+ assert_eq!(
+ events[0].event().event,
+ DocumentTopicEvent::SyncContentUpdated(DocumentSyncContentUpdatedMetadata {
+ document_id: "document-a".to_string(),
+ file_type: FileType::Md,
+ document_version_id: Some("version-42".to_string()),
+ })
+ );
+ assert_eq!(events[1].key(), "document-b");
+ assert_eq!(
+ events[1].event().event,
+ DocumentTopicEvent::SyncContentUpdated(DocumentSyncContentUpdatedMetadata {
+ document_id: "document-b".to_string(),
+ file_type: FileType::Pdf,
+ document_version_id: None,
+ })
+ );
+}
+
+#[test]
+fn documents_map_to_one_event_each_in_input_order() {
+ let events = documents_to_events(
+ ["document-a", "document-c", "document-b"]
+ .into_iter()
+ .map(|document_id| SyncDocument {
+ document_id: document_id.to_string(),
+ document_version_id: None,
+ file_type: FileType::Md,
+ })
+ .collect(),
+ );
+
+ let event_keys: Vec<&str> = events.iter().map(|event| event.key()).collect();
+ assert_eq!(event_keys, ["document-a", "document-c", "document-b"]);
+}
diff --git a/services/search_processing_service/src/config.rs b/services/search_processing_service/src/config.rs
index 3233ba0b6f..14cb871fa3 100644
--- a/services/search_processing_service/src/config.rs
+++ b/services/search_processing_service/src/config.rs
@@ -11,7 +11,7 @@ env_vars! {
pub struct OpensearchPassword;
pub struct DocumentStorageBucket;
pub struct BackfillJobsTable;
- /// Comma-separated Kafka bootstrap servers for live call lifecycle events.
+ /// Comma-separated Kafka bootstrap servers for live search-index events.
pub struct KafkaBrokers;
}
@@ -113,7 +113,7 @@ pub struct Config {
/// The bucket where documents are stored
pub document_storage_bucket: DocumentStorageBucket,
- /// Comma-separated Kafka bootstrap servers for live call lifecycle events.
+ /// Comma-separated Kafka bootstrap servers for live search-index events.
pub kafka_brokers: KafkaBrokers,
/// The number of workers to spawn
diff --git a/services/search_processing_service/src/inbound/kafka_consumer.rs b/services/search_processing_service/src/inbound/kafka_consumer.rs
index cde317db90..a0f920a058 100644
--- a/services/search_processing_service/src/inbound/kafka_consumer.rs
+++ b/services/search_processing_service/src/inbound/kafka_consumer.rs
@@ -7,14 +7,16 @@
//! offsets are already committed.
//!
//! Per-entity event mapping and processing live in the [`call`], [`channel`],
-//! [`chat`], and [`project`] submodules; this module owns the poll loop, worker,
-//! retry policy, and commit semantics.
+//! [`chat`], [`document`], and [`project`] submodules; this module owns the poll
+//! loop, worker, retry policy, and commit semantics.
#![allow(clippy::enum_variant_names)]
mod call;
mod channel;
mod chat;
+mod context;
+mod document;
mod project;
#[cfg(test)]
mod test;
@@ -24,24 +26,25 @@ use std::{future::Future, time::Duration};
use ::call::domain::events::CallMacroEvent;
use ::chat::domain::events::ChatMacroEvent;
use channels::domain::broker_events::ChannelMacroEvent;
+use documents::domain::events::DocumentMacroEvent;
use kafka_util::{GroupName, KafkaEventConsumer};
use macro_event_broker::{KafkaConsumerAdapter, MacroEventCollection, MacroEventConsumerService};
-use opensearch_client::OpensearchClient;
use projects::domain::events::ProjectMacroEvent;
use rdkafka::{
consumer::CommitMode,
message::{BorrowedMessage, Message as _},
};
use rootcause::prelude::{Report, ResultExt as _};
-use sqlx::PgPool;
use tokio::sync::mpsc;
use tokio_retry::{Retry, strategy::ExponentialBackoff};
use self::{
call::process_call_event, channel::process_channel_event, chat::process_chat_event,
- project::process_project_event,
+ document::process_document_event, project::process_project_event,
};
+pub(crate) use self::context::KafkaProcessingContext;
+
/// Consumer group used for live search-index event offsets.
pub(crate) struct SearchProcessingConsumerGroup;
@@ -59,6 +62,7 @@ macro_event_broker::declare_topics!(
CallMacroEvent,
ChannelMacroEvent,
ChatMacroEvent,
+ DocumentMacroEvent,
ProjectMacroEvent,
);
@@ -159,14 +163,16 @@ fn attach_event_coordinates<E>(
})
}
-#[tracing::instrument(skip(db, opensearch_client, event), fields(partition, offset))]
+#[tracing::instrument(skip(context, event), fields(partition, offset))]
async fn process_event(
- db: &PgPool,
- opensearch_client: &OpensearchClient,
+ context: &KafkaProcessingContext,
event: &DeclaredMacroEvent,
partition: i32,
offset: i64,
) -> EventOutcome {
+ let db = &context.db;
+ let opensearch_client = context.opensearch_client.as_ref();
+
match event {
DeclaredMacroEvent::CallMacroEvent(event) => {
process_call_event(db, opensearch_client, event, partition, offset).await
@@ -177,6 +183,9 @@ async fn process_event(
DeclaredMacroEvent::ChatMacroEvent(event) => {
process_chat_event(db, opensearch_client, event, partition, offset).await
}
+ DeclaredMacroEvent::DocumentMacroEvent(event) => {
+ process_document_event(context, event, partition, offset).await
+ }
DeclaredMacroEvent::ProjectMacroEvent(event) => {
process_project_event(db, opensearch_client, event, partition, offset).await
}
@@ -184,14 +193,12 @@ async fn process_event(
}
async fn run_event_worker(
- db: PgPool,
- opensearch_client: OpensearchClient,
+ context: KafkaProcessingContext,
mut events: mpsc::Receiver<ReceivedEvent>,
) {
while let Some(received) = events.recv().await {
let _ = process_event(
- &db,
- &opensearch_client,
+ &context,
&received.event,
received.partition,
received.offset,
@@ -287,11 +294,10 @@ async fn poll_events(
/// channel returns an error without committing the current Kafka message. On
/// normal shutdown the sender is dropped and all buffered events are processed
/// before this function returns.
-#[tracing::instrument(skip(db, opensearch_client, shutdown), fields(brokers), err)]
+#[tracing::instrument(skip(context, shutdown), fields(brokers), err)]
pub(crate) async fn run_event_consumer(
brokers: &str,
- db: PgPool,
- opensearch_client: OpensearchClient,
+ context: KafkaProcessingContext,
shutdown: impl Future<Output = ()> + Send,
) -> Result<(), Report> {
let consumer = KafkaEventConsumer::<SearchProcessingConsumerGroup>::from_env(brokers)?;
@@ -306,7 +312,7 @@ pub(crate) async fn run_event_consumer(
);
let (events_tx, events_rx) = mpsc::channel(CHANNEL_CAPACITY);
- let worker = tokio::spawn(run_event_worker(db, opensearch_client, events_rx));
+ let worker = tokio::spawn(run_event_worker(context, events_rx));
let poll_result = poll_events(&consumer, events_tx, shutdown).await;
let worker_result = worker.await;
diff --git a/services/search_processing_service/src/inbound/kafka_consumer/context.rs b/services/search_processing_service/src/inbound/kafka_consumer/context.rs
new file mode 100644
index 0000000000..dfdc66112f
--- /dev/null
+++ b/services/search_processing_service/src/inbound/kafka_consumer/context.rs
@@ -0,0 +1,16 @@
+use std::sync::Arc;
+
+use lexical_client::LexicalClient;
+use opensearch_client::OpensearchClient;
+use s3_client::S3;
+use sqlx::PgPool;
+
+/// Shared dependencies used to process search-index events from Kafka.
+#[derive(Clone)]
+pub(crate) struct KafkaProcessingContext {
+ pub(crate) db: PgPool,
+ pub(crate) opensearch_client: Arc<OpensearchClient>,
+ pub(crate) s3_client: Arc<S3>,
+ pub(crate) document_storage_bucket: String,
+ pub(crate) lexical_client: Arc<LexicalClient>,
+}
diff --git a/services/search_processing_service/src/inbound/kafka_consumer/document.rs b/services/search_processing_service/src/inbound/kafka_consumer/document.rs
new file mode 100644
index 0000000000..c4aeaa7c70
--- /dev/null
+++ b/services/search_processing_service/src/inbound/kafka_consumer/document.rs
@@ -0,0 +1,241 @@
+//! Maps document lifecycle events to search-index actions and processes them.
+
+use documents::domain::events::{DocumentMacroEvent, DocumentTopicEvent};
+use macro_event_broker::MacroEvent as _;
+use model::document::FileType;
+use sqs_client::search::document::SearchExtractorMessage;
+
+use super::{
+ EventOutcome, KafkaProcessingContext, MAX_PROCESSING_ATTEMPTS, PROCESSING_RETRY_BASE_DELAY,
+ retry_processing,
+};
+use crate::process::document::{
+ process_extract_sync_message, process_extract_text_message, process_remove_message,
+ process_update_name_message,
+};
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(super) enum DocumentIndexAction {
+ ExtractText {
+ owner: String,
+ file_type: FileType,
+ document_version_id: Option<String>,
+ },
+ ExtractSync {
+ file_type: FileType,
+ document_version_id: Option<String>,
+ },
+ RefreshName,
+ Remove,
+ Ignore,
+}
+
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub(super) struct DocumentEventDescription {
+ pub(super) action: DocumentIndexAction,
+ pub(super) document_id: String,
+ pub(super) event_type: &'static str,
+}
+
+pub(super) fn describe_document_event(event: &DocumentTopicEvent) -> DocumentEventDescription {
+ match event {
+ DocumentTopicEvent::Created(metadata) => DocumentEventDescription {
+ action: DocumentIndexAction::Ignore,
+ document_id: metadata.document_id.clone(),
+ event_type: "document.created",
+ },
+ DocumentTopicEvent::Updated(metadata) => DocumentEventDescription {
+ action: if metadata.document_name.is_some() {
+ DocumentIndexAction::RefreshName
+ } else {
+ DocumentIndexAction::Ignore
+ },
+ document_id: metadata.document_id.clone(),
+ event_type: "document.updated",
+ },
+ DocumentTopicEvent::Deleted(metadata) => DocumentEventDescription {
+ action: DocumentIndexAction::Ignore,
+ document_id: metadata.document_id.clone(),
+ event_type: "document.deleted",
+ },
+ DocumentTopicEvent::ContentUploaded(metadata) => DocumentEventDescription {
+ action: DocumentIndexAction::ExtractText {
+ owner: metadata.owner.to_string(),
+ file_type: metadata.file_type,
+ document_version_id: metadata.document_version_id.clone(),
+ },
+ document_id: metadata.document_id.clone(),
+ event_type: "document.content_uploaded",
+ },
+ DocumentTopicEvent::SyncContentUpdated(metadata) => DocumentEventDescription {
+ action: DocumentIndexAction::ExtractSync {
+ file_type: metadata.file_type,
+ document_version_id: metadata.document_version_id.clone(),
+ },
+ document_id: metadata.document_id.clone(),
+ event_type: "document.sync_content_updated",
+ },
+ DocumentTopicEvent::Purged(metadata) => DocumentEventDescription {
+ action: DocumentIndexAction::Remove,
+ document_id: metadata.document_id.clone(),
+ event_type: "document.purged",
+ },
+ DocumentTopicEvent::Copied(metadata) => DocumentEventDescription {
+ action: DocumentIndexAction::Ignore,
+ document_id: metadata.document_id.clone(),
+ event_type: "document.copied",
+ },
+ DocumentTopicEvent::Interaction(metadata) => DocumentEventDescription {
+ action: DocumentIndexAction::Ignore,
+ document_id: metadata.document_id.clone(),
+ event_type: "document.interaction",
+ },
+ }
+}
+
+pub(super) fn stored_extractor_message(
+ document_id: &str,
+ owner: String,
+ file_type: FileType,
+ document_version_id: Option<String>,
+) -> SearchExtractorMessage {
+ SearchExtractorMessage {
+ user_id: owner,
+ document_id: document_id.to_string(),
+ file_type,
+ document_version_id,
+ index_override: None,
+ }
+}
+
+pub(super) fn sync_extractor_message(
+ document_id: &str,
+ file_type: FileType,
+ document_version_id: Option<String>,
+) -> SearchExtractorMessage {
+ SearchExtractorMessage {
+ user_id: String::new(),
+ document_id: document_id.to_string(),
+ file_type,
+ document_version_id,
+ index_override: None,
+ }
+}
+
+async fn process_document_index_action(
+ context: &KafkaProcessingContext,
+ document_id: &str,
+ action: DocumentIndexAction,
+) -> anyhow::Result<()> {
+ let opensearch_client = context.opensearch_client.as_ref();
+
+ match action {
+ DocumentIndexAction::ExtractText {
+ owner,
+ file_type,
+ document_version_id,
+ } => {
+ let message =
+ stored_extractor_message(document_id, owner, file_type, document_version_id);
+ process_extract_text_message(
+ opensearch_client,
+ &context.db,
+ context.s3_client.as_ref(),
+ &context.document_storage_bucket,
+ &message,
+ )
+ .await
+ }
+ DocumentIndexAction::ExtractSync {
+ file_type,
+ document_version_id,
+ } => {
+ let message = sync_extractor_message(document_id, file_type, document_version_id);
+ process_extract_sync_message(
+ opensearch_client,
+ &context.db,
+ context.s3_client.as_ref(),
+ &context.document_storage_bucket,
+ context.lexical_client.as_ref(),
+ &message,
+ )
+ .await
+ }
+ DocumentIndexAction::RefreshName => {
+ process_update_name_message(opensearch_client, &context.db, document_id).await
+ }
+ DocumentIndexAction::Remove => process_remove_message(opensearch_client, document_id).await,
+ DocumentIndexAction::Ignore => Ok(()),
+ }
+}
+
+pub(super) async fn process_document_event(
+ context: &KafkaProcessingContext,
+ event: &DocumentMacroEvent,
+ partition: i32,
+ offset: i64,
+) -> EventOutcome {
+ let description = describe_document_event(&event.event().event);
+ let document_id = description.document_id.as_str();
+ let event_type = description.event_type;
+
+ if description.action == DocumentIndexAction::Ignore {
+ tracing::trace!(
+ document_id,
+ event_type,
+ partition,
+ offset,
+ "ignoring document event without a search-index action"
+ );
+ return EventOutcome::Ignored;
+ }
+
+ let result = retry_processing(|attempt| {
+ let action = description.action.clone();
+ async move {
+ tracing::trace!(
+ document_id,
+ event_type,
+ partition,
+ offset,
+ attempt,
+ "processing document search-index event"
+ );
+ process_document_index_action(context, document_id, action)
+ .await
+ .inspect_err(|error| {
+ if attempt < MAX_PROCESSING_ATTEMPTS {
+ let retry_delay =
+ PROCESSING_RETRY_BASE_DELAY * 2u32.pow(attempt.saturating_sub(1));
+ tracing::warn!(
+ error = ?error,
+ document_id,
+ event_type,
+ partition,
+ offset,
+ attempt,
+ delay_secs = retry_delay.as_secs(),
+ "document search-index processing failed, retrying"
+ );
+ }
+ })
+ }
+ })
+ .await;
+
+ match result {
+ Ok(()) => EventOutcome::Indexed,
+ Err(error) => {
+ tracing::error!(
+ error = ?error,
+ document_id,
+ event_type,
+ partition,
+ offset,
+ attempts = MAX_PROCESSING_ATTEMPTS,
+ "dropping document event after processing retries were exhausted"
+ );
+ EventOutcome::Dropped
+ }
+ }
+}
diff --git a/services/search_processing_service/src/inbound/kafka_consumer/test.rs b/services/search_processing_service/src/inbound/kafka_consumer/test.rs
index b62352a43f..54336ba4cd 100644
--- a/services/search_processing_service/src/inbound/kafka_consumer/test.rs
+++ b/services/search_processing_service/src/inbound/kafka_consumer/test.rs
@@ -22,11 +22,19 @@ use channels::domain::{
models::{ChannelSender, ChannelType},
};
use chrono::Utc;
+use documents::domain::events::{
+ DocumentContentUploadedMetadata, DocumentCopiedMetadata, DocumentCreatedMetadata,
+ DocumentDeletedMetadata, DocumentInteractionMetadata, DocumentPurgedMetadata,
+ DocumentSyncContentUpdatedMetadata, DocumentTopicEvent, DocumentUpdatedMetadata,
+ InteractionReason,
+};
use macro_event_broker::{Event, EventBrokerError, MacroEvent as _, MessageParts};
use macro_event_topics::{
- MacroCallsTopic, MacroChannelsTopic, MacroChatsTopic, MacroProjectsTopic, Topic as _,
+ MacroCallsTopic, MacroChannelsTopic, MacroChatsTopic, MacroDocumentsTopic, MacroProjectsTopic,
+ Topic as _,
};
use macro_user_id::user_id::MacroUserIdStr;
+use model::document::FileType;
use projects::domain::events::{
ProjectCreatedMetadata, ProjectDeletedMetadata, ProjectPermanentlyDeletedMetadata,
ProjectRestoredMetadata, ProjectTopicEvent, ProjectUpdatedMetadata, ProjectUploadedMetadata,
@@ -37,6 +45,10 @@ use super::{
call::{CallEventDescription, CallIndexAction, describe_call_event},
channel::{ChannelEventDescription, ChannelIndexAction, describe_channel_event},
chat::{ChatEventDescription, ChatIndexAction, describe_chat_event},
+ document::{
+ DocumentEventDescription, DocumentIndexAction, describe_document_event,
+ stored_extractor_message, sync_extractor_message,
+ },
project::{
ProjectEventDescription, ProjectIndexAction, collect_project_ids, describe_project_event,
},
@@ -49,6 +61,8 @@ const MESSAGE_ID: Uuid = Uuid::from_u128(3);
const CHAT_ID: &str = "chat-id";
const SECOND_CHAT_ID: &str = "second-chat-id";
const CHAT_MESSAGE_ID: &str = "chat-message-id";
+const DOCUMENT_ID: &str = "document-id";
+const SOURCE_DOCUMENT_ID: &str = "source-document-id";
const PROJECT_ID: &str = "project-root";
const CHILD_PROJECT_ID: &str = "project-child";
const PARENT_PROJECT_ID: &str = "project-parent";
@@ -430,6 +444,145 @@ fn channel_event_cases() -> Vec<(ChannelTopicEvent, ChannelEventDescription)> {
]
}
+fn document_event_cases() -> Vec<(DocumentTopicEvent, DocumentEventDescription)> {
+ let owner = user_id();
+
+ vec![
+ (
+ DocumentTopicEvent::Created(DocumentCreatedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ owner: owner.clone(),
+ document_name: "Document".to_string(),
+ file_type: Some(FileType::Pdf),
+ project_id: Some(PROJECT_ID.to_string()),
+ sub_type: None,
+ created_at: None,
+ }),
+ DocumentEventDescription {
+ action: DocumentIndexAction::Ignore,
+ document_id: DOCUMENT_ID.to_string(),
+ event_type: "document.created",
+ },
+ ),
+ (
+ DocumentTopicEvent::Updated(DocumentUpdatedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ owner: owner.clone(),
+ actor_user_id: Some(owner.clone()),
+ document_name: Some("Renamed document".to_string()),
+ previous_project_id: Some(PROJECT_ID.to_string()),
+ project_id: None,
+ file_type: None,
+ share_permission_updated: false,
+ }),
+ DocumentEventDescription {
+ action: DocumentIndexAction::RefreshName,
+ document_id: DOCUMENT_ID.to_string(),
+ event_type: "document.updated",
+ },
+ ),
+ (
+ DocumentTopicEvent::Updated(DocumentUpdatedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ owner: owner.clone(),
+ actor_user_id: Some(owner.clone()),
+ document_name: None,
+ previous_project_id: None,
+ project_id: Some(PROJECT_ID.to_string()),
+ file_type: None,
+ share_permission_updated: true,
+ }),
+ DocumentEventDescription {
+ action: DocumentIndexAction::Ignore,
+ document_id: DOCUMENT_ID.to_string(),
+ event_type: "document.updated",
+ },
+ ),
+ (
+ DocumentTopicEvent::Deleted(DocumentDeletedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ actor_user_id: Some(owner.clone()),
+ project_id: Some(PROJECT_ID.to_string()),
+ }),
+ DocumentEventDescription {
+ action: DocumentIndexAction::Ignore,
+ document_id: DOCUMENT_ID.to_string(),
+ event_type: "document.deleted",
+ },
+ ),
+ (
+ DocumentTopicEvent::ContentUploaded(DocumentContentUploadedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ owner: owner.clone(),
+ file_type: FileType::Pdf,
+ document_version_id: Some("convert".to_string()),
+ }),
+ DocumentEventDescription {
+ action: DocumentIndexAction::ExtractText {
+ owner: owner.to_string(),
+ file_type: FileType::Pdf,
+ document_version_id: Some("convert".to_string()),
+ },
+ document_id: DOCUMENT_ID.to_string(),
+ event_type: "document.content_uploaded",
+ },
+ ),
+ (
+ DocumentTopicEvent::SyncContentUpdated(DocumentSyncContentUpdatedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ file_type: FileType::Md,
+ document_version_id: None,
+ }),
+ DocumentEventDescription {
+ action: DocumentIndexAction::ExtractSync {
+ file_type: FileType::Md,
+ document_version_id: None,
+ },
+ document_id: DOCUMENT_ID.to_string(),
+ event_type: "document.sync_content_updated",
+ },
+ ),
+ (
+ DocumentTopicEvent::Purged(DocumentPurgedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ }),
+ DocumentEventDescription {
+ action: DocumentIndexAction::Remove,
+ document_id: DOCUMENT_ID.to_string(),
+ event_type: "document.purged",
+ },
+ ),
+ (
+ DocumentTopicEvent::Copied(DocumentCopiedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ source_document_id: SOURCE_DOCUMENT_ID.to_string(),
+ source_version_id: Some(7),
+ owner,
+ document_name: "Copied document".to_string(),
+ file_type: Some(FileType::Pdf),
+ project_id: Some(PROJECT_ID.to_string()),
+ sub_type: None,
+ }),
+ DocumentEventDescription {
+ action: DocumentIndexAction::Ignore,
+ document_id: DOCUMENT_ID.to_string(),
+ event_type: "document.copied",
+ },
+ ),
+ (
+ DocumentTopicEvent::Interaction(DocumentInteractionMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ reason: InteractionReason::Edited,
+ }),
+ DocumentEventDescription {
+ action: DocumentIndexAction::Ignore,
+ document_id: DOCUMENT_ID.to_string(),
+ event_type: "document.interaction",
+ },
+ ),
+ ]
+}
+
fn project_event_cases() -> Vec<(ProjectTopicEvent, ProjectEventDescription<'static>)> {
let owner = user_id();
@@ -600,6 +753,7 @@ fn subscribes_to_declared_search_processing_topics_with_durable_group() {
assert!(topics.contains(&MacroCallsTopic::TOPIC_STR));
assert!(topics.contains(&MacroChannelsTopic::TOPIC_STR));
assert!(topics.contains(&MacroChatsTopic::TOPIC_STR));
+ assert!(topics.contains(&MacroDocumentsTopic::TOPIC_STR));
assert!(topics.contains(&MacroProjectsTopic::TOPIC_STR));
}
@@ -682,6 +836,72 @@ fn maps_all_channel_lifecycle_events_to_index_actions() {
}
}
+#[test]
+fn maps_all_document_lifecycle_events_to_index_actions() {
+ let cases = document_event_cases();
+ assert_eq!(cases.len(), 9);
+
+ for (event, expected) in cases {
+ let serialized = serde_json::to_value(&event).expect("serializable document event");
+ assert_eq!(serialized["event_type"], expected.event_type);
+ assert_eq!(describe_document_event(&event), expected);
+ }
+}
+
+#[test]
+fn document_extraction_actions_preserve_optional_versions() {
+ let content_uploaded = DocumentTopicEvent::ContentUploaded(DocumentContentUploadedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ owner: user_id(),
+ file_type: FileType::Pdf,
+ document_version_id: None,
+ });
+ assert_eq!(
+ describe_document_event(&content_uploaded).action,
+ DocumentIndexAction::ExtractText {
+ owner: "macro|[email protected]".to_string(),
+ file_type: FileType::Pdf,
+ document_version_id: None,
+ }
+ );
+
+ let sync_content_updated =
+ DocumentTopicEvent::SyncContentUpdated(DocumentSyncContentUpdatedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ file_type: FileType::Md,
+ document_version_id: Some("snapshot-7".to_string()),
+ });
+ assert_eq!(
+ describe_document_event(&sync_content_updated).action,
+ DocumentIndexAction::ExtractSync {
+ file_type: FileType::Md,
+ document_version_id: Some("snapshot-7".to_string()),
+ }
+ );
+}
+
+#[test]
+fn document_extractor_messages_disable_index_overrides_and_set_expected_users() {
+ let stored = stored_extractor_message(
+ DOCUMENT_ID,
+ "macro|[email protected]".to_string(),
+ FileType::Pdf,
+ None,
+ );
+ assert_eq!(stored.user_id, "macro|[email protected]");
+ assert_eq!(stored.document_id, DOCUMENT_ID);
+ assert_eq!(stored.file_type, FileType::Pdf);
+ assert_eq!(stored.document_version_id, None);
+ assert_eq!(stored.index_override, None);
+
+ let sync = sync_extractor_message(DOCUMENT_ID, FileType::Md, Some("snapshot-7".to_string()));
+ assert_eq!(sync.user_id, "");
+ assert_eq!(sync.document_id, DOCUMENT_ID);
+ assert_eq!(sync.file_type, FileType::Md);
+ assert_eq!(sync.document_version_id.as_deref(), Some("snapshot-7"));
+ assert_eq!(sync.index_override, None);
+}
+
#[test]
fn maps_all_project_lifecycle_events_to_reconciliation_actions() {
let cases = project_event_cases();
@@ -775,6 +995,83 @@ fn project_envelope_decodes_round_trip_with_string_key() {
assert_eq!(decoded_event.event().event, event);
}
+#[test]
+fn exact_macro_documents_envelopes_decode_into_document_events() {
+ let cases: Vec<(&[u8], Event<DocumentTopicEvent>)> = vec![
+ (
+ br#"{
+ "event_id":"00000000-0000-0000-0000-000000000001",
+ "schema_version":1,
+ "event_type":"document.content_uploaded",
+ "metadata":{
+ "document_id":"document-id",
+ "owner":"macro|[email protected]",
+ "file_type":"pdf",
+ "document_version_id":"convert"
+ }
+ }"#,
+ Event::with_event_id(
+ Uuid::from_u128(1),
+ DocumentTopicEvent::ContentUploaded(DocumentContentUploadedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ owner: user_id(),
+ file_type: FileType::Pdf,
+ document_version_id: Some("convert".to_string()),
+ }),
+ ),
+ ),
+ (
+ br#"{
+ "event_id":"00000000-0000-0000-0000-000000000002",
+ "schema_version":1,
+ "event_type":"document.sync_content_updated",
+ "metadata":{
+ "document_id":"document-id",
+ "file_type":"md",
+ "document_version_id":null
+ }
+ }"#,
+ Event::with_event_id(
+ Uuid::from_u128(2),
+ DocumentTopicEvent::SyncContentUpdated(DocumentSyncContentUpdatedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ file_type: FileType::Md,
+ document_version_id: None,
+ }),
+ ),
+ ),
+ (
+ br#"{
+ "event_id":"00000000-0000-0000-0000-000000000003",
+ "schema_version":1,
+ "event_type":"document.purged",
+ "metadata":{"document_id":"document-id"}
+ }"#,
+ Event::with_event_id(
+ Uuid::from_u128(3),
+ DocumentTopicEvent::Purged(DocumentPurgedMetadata {
+ document_id: DOCUMENT_ID.to_string(),
+ }),
+ ),
+ ),
+ ];
+
+ for (payload, expected) in cases {
+ let message = TestMessage {
+ topic: MacroDocumentsTopic::TOPIC_STR,
+ key: Some(DOCUMENT_ID.to_string()),
+ payload: Some(payload.to_vec()),
+ };
+
+ let decoded = DeclaredMacroEvent::decode(&message).expect("decodable document event");
+ let DeclaredMacroEvent::DocumentMacroEvent(decoded_event) = decoded else {
+ panic!("expected document event");
+ };
+ assert_eq!(decoded_event.key(), DOCUMENT_ID);
+ assert_eq!(decoded_event.event(), &expected);
+ }
+}
+
#[tokio::test]
async fn malformed_and_unsupported_chat_messages_are_commit_safe() {
let malformed = TestMessage {
diff --git a/services/search_processing_service/src/main.rs b/services/search_processing_service/src/main.rs
index ccd8fbf563..5bdd81317f 100644
--- a/services/search_processing_service/src/main.rs
+++ b/services/search_processing_service/src/main.rs
@@ -1,10 +1,8 @@
#![recursion_limit = "256"]
-use std::sync::Arc;
-#[cfg(feature = "processing")]
-use std::time::Duration;
+use std::{sync::Arc, time::Duration};
#[cfg(feature = "processing")]
-use crate::inbound::kafka_consumer::run_event_consumer;
+use crate::inbound::kafka_consumer::{KafkaProcessingContext, run_event_consumer};
use crate::{
api::context::{ApiContext, AuthorizationService},
config::DatabaseUrlReadonly,
@@ -17,6 +15,7 @@ use config::{Config, Environment};
use lexical_client::LexicalClient;
use macro_authorization::{InternalAuthConfig, MacroAuthorizationState, NoopMacroAuthJwtValidator};
use macro_entrypoint::MacroEntrypoint;
+use macro_event_broker::{KafkaEventPublisher, MacroEventBrokerService};
use opensearch_client::OpensearchClient;
#[cfg(feature = "pdf")]
use rust_embed::RustEmbed;
@@ -24,7 +23,7 @@ use sqlx::PgPool;
use sqlx::postgres::PgPoolOptions;
#[cfg(feature = "processing")]
use tokio_retry::{Retry, strategy::FixedInterval};
-use tokio_util::sync::CancellationToken;
+use tokio_util::{sync::CancellationToken, task::TaskTracker};
mod api;
mod config;
@@ -74,18 +73,17 @@ struct PdfiumLib;
#[cfg(feature = "processing")]
const CONSUMER_RESTART_DELAY: Duration = Duration::from_secs(5);
+const EVENT_BROKER_DRAIN_TIMEOUT: Duration = Duration::from_secs(10);
#[cfg(feature = "processing")]
async fn supervise_event_consumer(
brokers: String,
- db: PgPool,
- opensearch_client: OpensearchClient,
+ context: KafkaProcessingContext,
shutdown_token: CancellationToken,
) {
let _ = Retry::start(FixedInterval::new(CONSUMER_RESTART_DELAY), || {
let consumer_brokers = brokers.clone();
- let consumer_db = db.clone();
- let consumer_opensearch_client = opensearch_client.clone();
+ let consumer_context = context.clone();
let consumer_shutdown_token = shutdown_token.clone();
async move {
@@ -97,8 +95,7 @@ async fn supervise_event_consumer(
let consumer_result = tokio::spawn(async move {
run_event_consumer(
&consumer_brokers,
- consumer_db,
- consumer_opensearch_client,
+ consumer_context,
task_shutdown_token.cancelled_owned(),
)
.await
@@ -159,7 +156,7 @@ async fn main() -> anyhow::Result<()> {
let sqs_client = sqs_client::SQS::new(aws_sdk_sqs::Client::new(&aws_config))
.search_event_queue(&search_event_queue);
- let s3_client = s3_client::S3::new(macro_aws_config::s3_client().await);
+ let s3_client = Arc::new(s3_client::S3::new(macro_aws_config::s3_client().await));
let (min_connections, max_connections): (u32, u32) = match config.environment {
Environment::Production => (5, 50),
@@ -180,12 +177,14 @@ async fn main() -> anyhow::Result<()> {
"initialized db connection"
);
- let opensearch_client = OpensearchClient::new(
- config.opensearch_url.to_string(),
- config.opensearch_username.to_string(),
- config.opensearch_password.as_ref().to_string(),
- )
- .context("unable to create opensearch client")?;
+ let opensearch_client = Arc::new(
+ OpensearchClient::new(
+ config.opensearch_url.to_string(),
+ config.opensearch_username.to_string(),
+ config.opensearch_password.as_ref().to_string(),
+ )
+ .context("unable to create opensearch client")?,
+ );
if let Err(e) = opensearch_client.health().await {
tracing::error!(error=?e, "error connecting to opensearch");
@@ -215,11 +214,15 @@ async fn main() -> anyhow::Result<()> {
));
let shutdown_token = CancellationToken::new();
+ let event_broker_tracker = TaskTracker::new();
+ let macro_event_broker = MacroEventBrokerService::new(
+ KafkaEventPublisher::new(config.kafka_brokers.as_ref())
+ .context("failed to create kafka event publisher")?,
+ event_broker_tracker.clone(),
+ );
#[cfg(feature = "processing")]
let consumer_supervisor = {
- use std::sync::Arc;
-
// Ensures that pdfium binary exists so we can kill the container early on failure
#[cfg(feature = "pdf")]
if !std::fs::exists("./pdfium-lib/linux/libpdfium.so").expect("able to find file") {
@@ -228,10 +231,10 @@ async fn main() -> anyhow::Result<()> {
tracing::trace!("libpdfium is present");
}
- let lexical_client = LexicalClient::new(
+ let lexical_client = Arc::new(LexicalClient::new(
config.internal_api_key.to_string(),
config.lexical_service_url.clone(),
- );
+ ));
let worker = sqs_worker::SQSWorker::new(
aws_sdk_sqs::Client::new(&aws_config),
@@ -239,20 +242,26 @@ async fn main() -> anyhow::Result<()> {
config.queue_max_messages,
config.queue_wait_time_seconds,
);
- let ctx = SearchProcessingContext {
+ let search_processing_context = SearchProcessingContext {
db: db.clone(),
worker: Arc::new(worker.clone()),
document_storage_bucket: config.document_storage_bucket.to_string(),
- s3_client: Arc::new(s3_client),
- opensearch_client: Arc::new(opensearch_client.clone()),
- lexical_client: Arc::new(lexical_client),
+ s3_client: s3_client.clone(),
+ opensearch_client: opensearch_client.clone(),
+ lexical_client: lexical_client.clone(),
};
- run_search_processing_workers(ctx, config.worker_count);
+ run_search_processing_workers(search_processing_context, config.worker_count);
+ let kafka_processing_context = KafkaProcessingContext {
+ db: db.clone(),
+ opensearch_client: opensearch_client.clone(),
+ s3_client: s3_client.clone(),
+ document_storage_bucket: config.document_storage_bucket.to_string(),
+ lexical_client,
+ };
tokio::spawn(supervise_event_consumer(
config.kafka_brokers.as_ref().to_owned(),
- db.clone(),
- opensearch_client.clone(),
+ kafka_processing_context,
shutdown_token.clone(),
))
};
@@ -274,11 +283,11 @@ async fn main() -> anyhow::Result<()> {
ApiContext {
db,
authorization_state,
- sqs_client,
- opensearch_client: Arc::new(opensearch_client),
+ opensearch_client,
config: Arc::new(config),
backfill_service,
backfill_jobs,
+ macro_event_broker,
},
shutdown_token.clone(),
)
@@ -286,9 +295,24 @@ async fn main() -> anyhow::Result<()> {
shutdown_token.cancel();
#[cfg(feature = "processing")]
- consumer_supervisor
+ let consumer_result = consumer_supervisor
.await
- .context("search processing event consumer supervisor failed")?;
+ .context("search processing event consumer supervisor failed");
+ tracing::info!("waiting for event broker publishes to drain");
+ event_broker_tracker.close();
+ match tokio::time::timeout(EVENT_BROKER_DRAIN_TIMEOUT, event_broker_tracker.wait()).await {
+ Ok(()) => tracing::info!("event broker publishes drained"),
+ Err(error) => {
+ tracing::warn!(
+ error=?error,
+ timeout_seconds = EVENT_BROKER_DRAIN_TIMEOUT.as_secs(),
+ "timed out waiting for event broker publishes to drain"
+ );
+ }
+ }
+
+ #[cfg(feature = "processing")]
+ consumer_result?;
api_result
}
diff --git a/services/search_processing_service/src/process/document/mod.rs b/services/search_processing_service/src/process/document/mod.rs
index 7cfdd78bbd..52e5f03aa1 100644
--- a/services/search_processing_service/src/process/document/mod.rs
+++ b/services/search_processing_service/src/process/document/mod.rs
@@ -1,17 +1,15 @@
use anyhow::Context;
use opensearch_client::OpensearchClient;
-use sqs_client::search::document::{DocumentId, SearchExtractorMessage};
+use sqs_client::search::document::SearchExtractorMessage;
mod document_info;
mod raw_document;
pub async fn process_remove_message(
opensearch_client: &OpensearchClient,
- remove_message: &DocumentId,
+ document_id: &str,
) -> anyhow::Result<()> {
- opensearch_client
- .delete_document(remove_message.document_id.as_str(), None)
- .await?;
+ opensearch_client.delete_document(document_id, None).await?;
Ok(())
}
@@ -25,18 +23,13 @@ pub async fn process_remove_message(
pub async fn process_update_name_message(
opensearch_client: &OpensearchClient,
db: &sqlx::Pool<sqlx::Postgres>,
- update_message: &DocumentId,
+ document_id: &str,
) -> anyhow::Result<()> {
- let document = match macro_db_client::document::get_basic_document(
- db,
- update_message.document_id.as_str(),
- )
- .await
- {
+ let document = match macro_db_client::document::get_basic_document(db, document_id).await {
Ok(document) => document,
Err(sqlx::Error::RowNotFound) => {
tracing::debug!(
- document_id = %update_message.document_id,
+ document_id,
"document not found; skipping search name update"
);
return Ok(());
@@ -46,14 +39,14 @@ pub async fn process_update_name_message(
if document.deleted_at.is_some() {
tracing::debug!(
- document_id = %update_message.document_id,
+ document_id,
"document is deleted; skipping search name update"
);
return Ok(());
}
opensearch_client
- .update_document_metadata(update_message.document_id.as_str(), &document.document_name)
+ .update_document_metadata(document_id, &document.document_name)
.await
.context("failed to update document name in search index")?;
diff --git a/services/search_processing_service/src/process/document/raw_document.rs b/services/search_processing_service/src/process/document/raw_document.rs
index 4cd0c1acd2..2f27301eea 100644
--- a/services/search_processing_service/src/process/document/raw_document.rs
+++ b/services/search_processing_service/src/process/document/raw_document.rs
@@ -233,6 +233,7 @@ pub async fn update_search_with_raw_document(
};
let document_name = document_info.document_name;
+ let owner_id = document_info.owner.to_string();
let sub_type = document_info.sub_type.map(|st| st.to_string());
// TODO: this is hacky, update the search event message to use the correctly serialized
@@ -313,7 +314,7 @@ pub async fn update_search_with_raw_document(
raw_content: None,
document_name: document_name.clone(),
content: page_content.clone(),
- owner_id: search_extractor_message.user_id.clone(),
+ owner_id: owner_id.clone(),
file_type: file_type.to_string(),
updated_at_millis,
sub_type: sub_type.clone(),
@@ -342,7 +343,7 @@ pub async fn update_search_with_raw_document(
raw_content: Some(result.raw_content),
document_name: document_name.clone(),
content: result.content,
- owner_id: search_extractor_message.user_id.clone(),
+ owner_id: owner_id.clone(),
file_type: file_type.to_string(),
updated_at_millis,
sub_type: sub_type.clone(),
@@ -361,7 +362,7 @@ pub async fn update_search_with_raw_document(
raw_content: None,
document_name,
content: content.clone(),
- owner_id: search_extractor_message.user_id.clone(),
+ owner_id,
file_type: file_type.to_string(),
updated_at_millis,
sub_type: sub_type.clone(),
diff --git a/services/search_processing_service/src/process/mod.rs b/services/search_processing_service/src/process/mod.rs
index b7c1b81daa..4350fc1935 100644
--- a/services/search_processing_service/src/process/mod.rs
+++ b/services/search_processing_service/src/process/mod.rs
@@ -2,7 +2,7 @@ pub(crate) mod call;
pub(crate) mod channel;
pub(crate) mod chat;
pub mod context;
-mod document;
+pub(crate) mod document;
mod email;
pub(crate) mod project;
mod properties;
@@ -80,9 +80,6 @@ pub async fn process_message(
email::upsert::process_upsert_message(&ctx.opensearch_client, &ctx.db, &message)
.await?;
}
- SearchQueueMessage::RemoveDocument(message) => {
- document::process_remove_message(&ctx.opensearch_client, &message).await?;
- }
SearchQueueMessage::ExtractDocumentText(message) => {
document::process_extract_text_message(
&ctx.opensearch_client,
@@ -108,10 +105,6 @@ pub async fn process_message(
properties::process_entity_property_update(&ctx.opensearch_client, &ctx.db, &message)
.await?;
}
- SearchQueueMessage::UpdateDocumentName(message) => {
- document::process_update_name_message(&ctx.opensearch_client, &ctx.db, &message)
- .await?;
- }
SearchQueueMessage::ChatMessage(message) => {
chat::insert_chat_message(&ctx.opensearch_client, &ctx.db, &message).await?;
}
diff --git a/services/search_upload_handler/Cargo.toml b/services/search_upload_handler/Cargo.toml
index 7fa0776e27..d34b429af0 100644
--- a/services/search_upload_handler/Cargo.toml
+++ b/services/search_upload_handler/Cargo.toml
@@ -6,21 +6,13 @@ version = "0.1.0"
[dependencies]
anyhow = { workspace = true }
-aws-sdk-sqs = { workspace = true }
-document_storage_service_client = { path = "../../crates/document_storage_service_client" }
-macro_aws_config = { path = "../../crates/macro_aws_config" }
aws_lambda_events = { workspace = true, features = ["eventbridge"] }
+document_storage_service_client = { path = "../../crates/document_storage_service_client" }
lambda_runtime = { workspace = true }
macro_entrypoint = { path = "../../crates/macro_entrypoint" }
macro_env_var = { path = "../../crates/macro_env_var" }
-macro_queues = { path = "../../crates/macro_queues" }
+macro_service_urls = { path = "../../crates/macro_service_urls" }
s3_key = { path = "../../crates/s3_key" }
-sqs_client = { path = "../../crates/sqs_client", default-features = false, features = [
- "search",
-] }
tokio = { workspace = true }
tracing = { workspace = true }
-
-
-macro_service_urls = { path = "../../crates/macro_service_urls" }
workspace-hack = { version = "0.1", path = "../../crates/workspace-hack" }
diff --git a/services/search_upload_handler/src/handler.rs b/services/search_upload_handler/src/handler.rs
index fde704e35f..437ea37fac 100644
--- a/services/search_upload_handler/src/handler.rs
+++ b/services/search_upload_handler/src/handler.rs
@@ -6,12 +6,10 @@ use lambda_runtime::{
tracing::{self},
};
use s3_key::DocumentKey;
-use sqs_client::search::{SearchQueueMessage, document::SearchExtractorMessage};
/// Handles the Eventbridge event
-#[tracing::instrument(skip(sqs_client, dss_client), err)]
+#[tracing::instrument(skip(dss_client), err)]
pub async fn handler(
- sqs_client: &sqs_client::SQS,
dss_client: &DocumentStorageServiceClient,
event: LambdaEvent<EventBridgeEvent>,
) -> Result<(), Error> {
@@ -57,22 +55,12 @@ pub async fn handler(
}
};
- let search_extractor_message = SearchExtractorMessage {
- user_id: document_basic.owner.to_string(),
- document_id: document_id.to_string(),
- document_version_id: document_key.version_id_string(),
- file_type,
- index_override: None,
- };
-
- // All other file types are to be sent to the search text extractor queue
- let message_id = sqs_client
- .send_message_to_search_event_queue(SearchQueueMessage::ExtractDocumentText(
- search_extractor_message,
- ))
+ let document_version_id = document_key.version_id_string();
+ dss_client
+ .publish_document_content_uploaded(document_id, file_type, document_version_id)
.await?;
- tracing::info!(message_id=?message_id, "sent message to search extractor queue");
+ tracing::info!(document_id, "relayed document content upload");
Ok(())
}
diff --git a/services/search_upload_handler/src/main.rs b/services/search_upload_handler/src/main.rs
index 87e91f5e87..309f1a3f80 100644
--- a/services/search_upload_handler/src/main.rs
+++ b/services/search_upload_handler/src/main.rs
@@ -6,7 +6,6 @@ use lambda_runtime::{Error, LambdaEvent, run, service_fn, tracing};
use macro_entrypoint::MacroEntrypoint;
use macro_env_var::env_vars;
use macro_service_urls::DocumentStorageServiceUrl;
-use std::sync::Arc;
mod handler;
@@ -20,26 +19,16 @@ async fn main() -> Result<(), Error> {
tracing::trace!("initiating lambda");
- let search_event_queue = macro_queues::SearchEventQueue::new();
-
let dss_url = DocumentStorageServiceUrl::new()?.to_string();
let dss_auth_key = DocumentStorageServiceAuthKey::new()
.context("DOCUMENT_STORAGE_SERVICE_AUTH_KEY must be provided")?
.to_string();
- let sqs_client = sqs_client::SQS::new(aws_sdk_sqs::Client::new(
- &macro_aws_config::get_macro_aws_config().await,
- ))
- .search_event_queue(search_event_queue.as_ref());
-
- let shared_sqs_client = Arc::new(sqs_client);
- let shared_dss_client = Arc::new(DocumentStorageServiceClient::new(dss_auth_key, dss_url));
+ let dss_client = DocumentStorageServiceClient::new(dss_auth_key, dss_url);
let func = service_fn(move |event: LambdaEvent<EventBridgeEvent>| {
- let sqs_client = shared_sqs_client.clone();
- let dss_client = shared_dss_client.clone();
-
- async move { handler(&sqs_client, &dss_client, event).await }
+ let dss_client = dss_client.clone();
+ async move { handler(&dss_client, event).await }
});
run(func).await