Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
chore: kafka record document sync service events (#4788)
Wolf Mermelstein0a1ea464a6e3
diff --git a/crates/documents/examples/documents_consumer.rs b/crates/documents/examples/documents_consumer.rsindex 6603e2ed30..8f4e29168a 100644--- a/crates/documents/examples/documents_consumer.rs+++ b/crates/documents/examples/documents_consumer.rs@@ -146,6 +146,7 @@ fn event_type(event: &DocumentTopicEvent) -> &'static str {DocumentTopicEvent::Updated(_) => "document.updated",DocumentTopicEvent::Deleted(_) => "document.deleted",DocumentTopicEvent::Copied(_) => "document.copied",+ DocumentTopicEvent::Interaction(_) => "document.interaction",}}diff --git a/crates/documents/src/domain/events.rs b/crates/documents/src/domain/events.rsindex 9f6c576580..45bb648e94 100644--- a/crates/documents/src/domain/events.rs+++ b/crates/documents/src/domain/events.rs@@ -68,6 +68,27 @@ pub struct DocumentDeletedMetadata {pub project_id: Option<String>,}+/// Why a document interaction was reported.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]+#[serde(rename_all = "snake_case")]+pub enum InteractionReason {+ /// A periodic save of pending content changes.+ Edited,+ /// The first peer joined the document session.+ FirstJoin,+ /// The last connected peer left the document session.+ LastLeave,+}++/// Metadata for [`DocumentTopicEvent::Interaction`].+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]+pub struct DocumentInteractionMetadata {+ /// The id of the document.+ pub document_id: String,+ /// What triggered this interaction.+ pub reason: InteractionReason,+}+/// Metadata for [`DocumentTopicEvent::Copied`].#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]pub struct DocumentCopiedMetadata {@@ -105,6 +126,9 @@ pub enum DocumentTopicEvent {/// A document was copied.#[serde(rename = "document.copied")]Copied(DocumentCopiedMetadata),+ /// A peer joined, left, or a periodic save occurred.+ #[serde(rename = "document.interaction")]+ Interaction(DocumentInteractionMetadata),}impl TopicEvent for DocumentTopicEvent {@@ -142,6 +166,11 @@ impl DocumentMacroEvent {Self::new(key, DocumentTopicEvent::Copied(metadata))}+ /// Build an interaction event keyed by the document id.+ pub fn interaction(key: impl Into<String>, metadata: DocumentInteractionMetadata) -> Self {+ Self::new(key, DocumentTopicEvent::Interaction(metadata))+ }+/// Build an event from a topic-specific event variant.pub fn new(key: impl Into<String>, event: DocumentTopicEvent) -> Self {Self::with_event(key, Event::new(event))diff --git a/crates/documents/src/domain/ports/mod.rs b/crates/documents/src/domain/ports/mod.rsindex d5c100ed4e..fd024e76c4 100644--- a/crates/documents/src/domain/ports/mod.rs+++ b/crates/documents/src/domain/ports/mod.rs@@ -16,6 +16,7 @@ use macro_user_id::user_id::MacroUserIdStr;use model::document::{ContentType, DocumentBasic, DocumentMetadata};use super::content::DocumentContent;+use super::events::InteractionReason;use super::response::{CreateDocumentResponseData, DocumentResponse, GetDocumentResponseData, LocationResponseV3,};@@ -488,6 +489,14 @@ pub trait DocumentService: Send + Sync + 'static {document_id: &str,bytes: Vec<u8>,) -> impl Future<Output = anyhow::Result<()>> + Send;++ /// Publishes a `document.interaction` event: a real edit, a peer joining,+ /// or the last peer leaving.+ fn record_interaction(+ &self,+ document_id: &str,+ reason: InteractionReason,+ ) -> impl Future<Output = anyhow::Result<()>> + Send;/// Get the team-share state of a document, resolved against the owner's team.fn get_team_share(&self,diff --git a/crates/documents/src/domain/service.rs b/crates/documents/src/domain/service.rsindex c48472b200..97482c0f31 100644--- a/crates/documents/src/domain/service.rs+++ b/crates/documents/src/domain/service.rs@@ -42,8 +42,8 @@ use crate::domain::models::{use super::branch_name::{build_task_branch_name, user_branch_prefix};use super::content::{DocumentContent, DocumentContentLocation, DocumentContentState};use super::events::{- DocumentCopiedMetadata, DocumentCreatedMetadata, DocumentDeletedMetadata, DocumentMacroEvent,- DocumentUpdatedMetadata,+ DocumentCopiedMetadata, DocumentCreatedMetadata, DocumentDeletedMetadata,+ DocumentInteractionMetadata, DocumentMacroEvent, DocumentUpdatedMetadata, InteractionReason,};use super::models::{CloudFrontConfig, CommentThread, CopyDocumentRepoArgs, CreateDocumentRepoArgs,@@ -1613,7 +1613,23 @@ impl<async fn upload_snapshot(&self, document_id: &str, bytes: Vec<u8>) -> anyhow::Result<()> {self.upload_url_service.upload_snapshot(document_id, bytes)- .await+ .await?;+ Ok(())+ }++ async fn record_interaction(+ &self,+ document_id: &str,+ reason: InteractionReason,+ ) -> anyhow::Result<()> {+ self.publish_document_event(&DocumentMacroEvent::interaction(+ document_id,+ DocumentInteractionMetadata {+ document_id: document_id.to_owned(),+ reason,+ },+ ));+ Ok(())}/// Assigns the task properties to a documentdiff --git a/crates/documents/src/inbound/axum_router.rs b/crates/documents/src/inbound/axum_router.rsindex 89b78b4e37..66e023f864 100644--- a/crates/documents/src/inbound/axum_router.rs+++ b/crates/documents/src/inbound/axum_router.rs@@ -32,6 +32,7 @@ pub mod get_document_by_team_slug;pub mod get_github_pull_requests;pub mod get_location;pub mod get_short_id;+pub mod put_interaction;pub mod put_snapshot;pub mod task_duplicates;pub mod team_share;diff --git a/crates/documents/src/inbound/axum_router/put_interaction.rs b/crates/documents/src/inbound/axum_router/put_interaction.rsnew file mode 100644index 0000000000..9d5837e3f3--- /dev/null+++ b/crates/documents/src/inbound/axum_router/put_interaction.rs@@ -0,0 +1,46 @@+//! Handler for `PUT /documents/{document_id}/interaction`.++use axum::{+ Json,+ extract::{Path, State},+ http::StatusCode,+ response::IntoResponse,+};+use entity_access::domain::ports::EntityAccessService;+use macro_authorization::{InternalMacroAuthorizationExtractor, MacroAuthorizationService};+use serde::Deserialize;++use super::{DocumentRouterState, Params};+use crate::domain::events::InteractionReason;+use crate::domain::ports::DocumentService;++/// Request body for an interaction report from sync-service.+#[derive(Deserialize)]+pub struct InteractionRequest {+ reason: InteractionReason,+}++/// Records a document interaction event.+#[tracing::instrument(skip(state, body, _internal_authorization))]+pub async fn put_interaction_handler<+ T: DocumentService,+ Svc: EntityAccessService,+ Auth: MacroAuthorizationService,+>(+ State(state): State<DocumentRouterState<T, Svc, Auth>>,+ _internal_authorization: InternalMacroAuthorizationExtractor<Auth>,+ Path(Params { document_id }): Path<Params>,+ Json(body): Json<InteractionRequest>,+) -> impl IntoResponse {+ match state+ .service+ .record_interaction(&document_id, body.reason)+ .await+ {+ Ok(()) => StatusCode::OK.into_response(),+ Err(e) => {+ tracing::error!(error=?e, document_id=document_id, "failed to record interaction");+ StatusCode::INTERNAL_SERVER_ERROR.into_response()+ }+ }+}diff --git a/crates/documents/src/inbound/axum_router/tests.rs b/crates/documents/src/inbound/axum_router/tests.rsindex 2fe174763f..871cc7f5a3 100644--- a/crates/documents/src/inbound/axum_router/tests.rs+++ b/crates/documents/src/inbound/axum_router/tests.rs@@ -47,6 +47,7 @@ use crate::{domain::{content::DocumentContent,create::DocumentCreator,+ events::InteractionReason,models::{CommentThread, CreateDocumentRepoArgs, CreateTaskRequest, DocumentError,DocumentTeamShareResponse, EditDocumentServiceArgs, GithubPullRequestsResponse,@@ -369,6 +370,14 @@ impl DocumentService for FakeDocumentService {Ok(())}+ async fn record_interaction(+ &self,+ _document_id: &str,+ _reason: InteractionReason,+ ) -> anyhow::Result<()> {+ panic!("unexpected record_interaction call")+ }+async fn get_team_share(&self,_entity_access_receipt: EntityAccessReceipt<entity_access::domain::models::ViewAccessLevel>,diff --git a/crates/github/src/domain/service/sync/test.rs b/crates/github/src/domain/service/sync/test.rsindex 08545158c7..853f46f8f1 100644--- a/crates/github/src/domain/service/sync/test.rs+++ b/crates/github/src/domain/service/sync/test.rs@@ -288,6 +288,14 @@ impl DocumentService for StubDocumentService {async fn upload_snapshot(&self, _document_id: &str, _bytes: Vec<u8>) -> anyhow::Result<()> {unimplemented!()}++ async fn record_interaction(+ &self,+ _document_id: &str,+ _reason: documents::domain::events::InteractionReason,+ ) -> anyhow::Result<()> {+ unimplemented!()+ }}/// Stateful stub repo that tracks task IDs per github key.diff --git a/crates/webhook/src/domain/ingestion.rs b/crates/webhook/src/domain/ingestion.rsindex e77f1aa2e5..bac974c171 100644--- a/crates/webhook/src/domain/ingestion.rs+++ b/crates/webhook/src/domain/ingestion.rs@@ -228,6 +228,9 @@ fn normalized_document_event(DocumentTopicEvent::Updated(metadata) => ("document.updated", &metadata.document_id),DocumentTopicEvent::Deleted(metadata) => ("document.deleted", &metadata.document_id),DocumentTopicEvent::Copied(metadata) => ("document.copied", &metadata.document_id),+ DocumentTopicEvent::Interaction(metadata) => {+ ("document.interaction", &metadata.document_id)+ }};if Uuid::parse_str(entity_id).is_err() {diff --git a/docker/docker-compose.yml b/docker/docker-compose.ymlindex 662149b910..8def1a8cc5 100644--- a/docker/docker-compose.yml+++ b/docker/docker-compose.yml@@ -398,6 +398,7 @@ services:environment:- INTERNAL_API_SECRET_KEY=${INTERNAL_API_SECRET_KEY}- DOCUMENT_PERMISSIONS_SECRET=${DOCUMENT_PERMISSION_JWT}+ - DSS_INTERNAL_AUTH_KEY=${DOCUMENT_STORAGE_SERVICE_AUTH_KEY}networks:services:aliases:diff --git a/docker/sync-service.Dockerfile b/docker/sync-service.Dockerfileindex 9025b5b953..0d450eb6d2 100644--- a/docker/sync-service.Dockerfile+++ b/docker/sync-service.Dockerfile@@ -58,4 +58,4 @@ EXPOSE 8787# Generate .dev.vars from environment variables so wrangler can resolve secret bindings,# apply local D1 migrations, then start.-CMD sh -c 'printf "LOCAL_API_KEY=%s\nDOCUMENT_PERMISSIONS_SECRET=%s\nSPS_API_SECRET_KEY=%s\n" "$INTERNAL_API_SECRET_KEY" "$DOCUMENT_PERMISSIONS_SECRET" "$INTERNAL_API_SECRET_KEY" > .dev.vars && CI=true npx wrangler d1 migrations apply USER_PEER_MAPPING --local --config wrangler.docker.toml && exec npx wrangler dev --local --ip 0.0.0.0 --port 8787 --config wrangler.docker.toml'+CMD sh -c 'printf "LOCAL_API_KEY=%s\nDOCUMENT_PERMISSIONS_SECRET=%s\nSPS_API_SECRET_KEY=%s\nDSS_INTERNAL_AUTH_KEY=%s\n" "$INTERNAL_API_SECRET_KEY" "$DOCUMENT_PERMISSIONS_SECRET" "$INTERNAL_API_SECRET_KEY" "$DSS_INTERNAL_AUTH_KEY" > .dev.vars && CI=true npx wrangler d1 migrations apply USER_PEER_MAPPING --local --config wrangler.docker.toml && exec npx wrangler dev --local --ip 0.0.0.0 --port 8787 --config wrangler.docker.toml'diff --git a/services/document_storage_service/src/api/internal/mod.rs b/services/document_storage_service/src/api/internal/mod.rsindex 1907552b19..e6bbfe4fcf 100644--- a/services/document_storage_service/src/api/internal/mod.rs+++ b/services/document_storage_service/src/api/internal/mod.rs@@ -136,6 +136,16 @@ pub fn router(state: ApiContext) -> Router<ApiContext> {>,),)+ .route(+ "/documents/{document_id}/interaction",+ put(+ documents_hex::inbound::axum_router::put_interaction::put_interaction_handler::<+ DocumentService,+ EntityAccessService,+ AuthorizationService,+ >,+ ),+ ).route("/documents/metadata", post(get_documents_metadata::handler))// History routes.route(diff --git a/services/sync-service/src/dss_internal.rs b/services/sync-service/src/dss_internal.rsindex f8db6156b6..54ae81762d 100644--- a/services/sync-service/src/dss_internal.rs+++ b/services/sync-service/src/dss_internal.rs@@ -1,14 +1,41 @@+use serde::Serialize;use tracing::error;use worker::{Env, Fetch, Method, Request, RequestInit};use crate::constants::header_names::MACRO_DOCUMENT_STORAGE_SERVICE_AUTH_HEADER_KEY;+/// Why a document interaction was reported to DSS.+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]+#[serde(rename_all = "snake_case")]+pub enum InteractionReason {+ /// A periodic save of pending content changes.+ Edited,+ /// The first peer joined the document session.+ FirstJoin,+ /// The last connected peer left the document session.+ LastLeave,+}++#[derive(Serialize)]+struct InteractionRequest {+ reason: InteractionReason,+}+pub trait DssInternal {+ /// Uploads the raw snapshot bytes for storage, and publishes a+ /// `document.edited` event.async fn publish_shallow_snapshot(&self,document_id: &str,snapshot: &[u8],) -> worker::Result<()>;++ /// Reports an interaction (join/leave/periodic edit).+ async fn publish_interaction(+ &self,+ document_id: &str,+ reason: InteractionReason,+ ) -> worker::Result<()>;}pub struct DssInternalClient<'a> {@@ -69,4 +96,46 @@ impl DssInternal for DssInternalClient<'_> {tracing::trace!("snapshot uploaded to DSS");Ok(())}++ async fn publish_interaction(+ &self,+ document_id: &str,+ reason: InteractionReason,+ ) -> worker::Result<()> {+ let url = format!(+ "{}/internal/documents/{}/interaction",+ self.dss_url()?,+ document_id+ );+ let auth_key = self.internal_auth_key()?;++ let body = serde_json::to_vec(&InteractionRequest { reason })+ .map_err(|e| worker::Error::from(format!("failed to serialize interaction: {e}")))?;++ let mut req = Request::new_with_init(+ &url,+ RequestInit::new()+ .with_method(Method::Put)+ .with_body(Some(body.into())),+ )?;+ req.headers_mut()?+ .set(MACRO_DOCUMENT_STORAGE_SERVICE_AUTH_HEADER_KEY, &auth_key)?;+ req.headers_mut()?.set("Content-Type", "application/json")?;++ let resp = Fetch::Request(req).send().await?;+ if resp.status_code() != 200 {+ error!(+ document_id = document_id,+ status = resp.status_code(),+ "DSS interaction upload failed"+ );+ return Err(worker::Error::from(format!(+ "DSS interaction upload returned {}",+ resp.status_code()+ )));+ }++ tracing::trace!("interaction uploaded to DSS");+ Ok(())+ }}diff --git a/services/sync-service/src/durable_object.rs b/services/sync-service/src/durable_object.rsindex 652d654e53..6c7d8fd3de 100644--- a/services/sync-service/src/durable_object.rs+++ b/services/sync-service/src/durable_object.rs@@ -21,7 +21,7 @@ use crate::{auth::{AccessLevel, TokenFrom, decode_jwt},constants::USER_PEER_D1_BINDING,d1::{PeerWithUserId, get_user_id_from_peer_id, insert_user_mapping},- dss_internal::{DssInternal, DssInternalClient},+ dss_internal::{DssInternal, DssInternalClient, InteractionReason},error::ResultExt,generated::schema::InitializeFromSnapshotRequest,keepalive::{DEFAULT_TIME_TO_LIVE, keepalive},@@ -287,6 +287,16 @@ async fn report_new_doc_state(document_id: &str, snapshot: &[u8], env: &Env) {}}+/// Report an interaction (join/leave/periodic edit) to DSS.+async fn report_interaction(document_id: &str, env: &Env, reason: InteractionReason) {+ if let Err(err) = DssInternalClient::new(env)+ .publish_interaction(document_id, reason)+ .await+ {+ warn!(error=?err, "failed to push interaction to DSS");+ }+}+/// Schedule an alarm 5 seconds from now.async fn bump_alarm(state: &State) -> Result<()> {let current_alarm = state.storage().get_alarm().await?;@@ -640,6 +650,10 @@ impl DocumentSyncSession {// Below is websocket stuff only i.e connectlet pair = WebSocketPair::new().context("failed to create websocket pair")?;+ // Whether this peer is the first to join the session (used to+ // decide whether to report a `FirstJoin` interaction below).+ let is_first_join = self.state.get_websockets().is_empty();+// create tag for ws and store itlet ws_id = new_ws_id();trace!(ws_id = ws_id, "websocket connect");@@ -681,6 +695,18 @@ impl DocumentSyncSession {);}+ // This is the single source of truth for `FirstJoin`: whenever+ // the peer count genuinely transitions 0 -> 1, regardless of+ // whether the document already has content.+ if is_first_join {+ let document_id_owned = document_id.to_string();+ let env = self.env.clone();+ self.state.wait_until(async move {+ report_interaction(&document_id_owned, &env, InteractionReason::FirstJoin)+ .await;+ });+ }+Response::from_websocket(pair.client).context("failed to create websocket response")?});@@ -999,6 +1025,7 @@ impl DurableObject for DocumentSyncSession {&& let Ok(snapshot) = doc_state.export_shallow_snapshot(){report_new_doc_state(&document_id, &snapshot, &env).await;+ report_interaction(&document_id, &env, InteractionReason::Edited).await;}});}@@ -1051,6 +1078,7 @@ impl DurableObject for DocumentSyncSession {let env = self.env.clone();self.state.wait_until(async move {report_new_doc_state(&document_id, &snapshot, &env).await;+ report_interaction(&document_id, &env, InteractionReason::LastLeave).await;});}Ok(())