Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

chore: kafka record document sync service events (#4788)

Wolf Mermelstein0a1ea464a6e3
diff --git a/crates/documents/examples/documents_consumer.rs b/crates/documents/examples/documents_consumer.rs
index 6603e2ed30..8f4e29168a 100644
--- a/crates/documents/examples/documents_consumer.rs
+++ b/crates/documents/examples/documents_consumer.rs
@@ -146,6 +146,7 @@ fn event_type(event: &DocumentTopicEvent) -> &'static str {
DocumentTopicEvent::Updated(_) => "document.updated",
DocumentTopicEvent::Deleted(_) => "document.deleted",
DocumentTopicEvent::Copied(_) => "document.copied",
+ DocumentTopicEvent::Interaction(_) => "document.interaction",
}
}
diff --git a/crates/documents/src/domain/events.rs b/crates/documents/src/domain/events.rs
index 9f6c576580..45bb648e94 100644
--- a/crates/documents/src/domain/events.rs
+++ b/crates/documents/src/domain/events.rs
@@ -68,6 +68,27 @@ pub struct DocumentDeletedMetadata {
pub project_id: Option<String>,
}
+/// Why a document interaction was reported.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum InteractionReason {
+ /// A periodic save of pending content changes.
+ Edited,
+ /// The first peer joined the document session.
+ FirstJoin,
+ /// The last connected peer left the document session.
+ LastLeave,
+}
+
+/// Metadata for [`DocumentTopicEvent::Interaction`].
+#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
+pub struct DocumentInteractionMetadata {
+ /// The id of the document.
+ pub document_id: String,
+ /// What triggered this interaction.
+ pub reason: InteractionReason,
+}
+
/// Metadata for [`DocumentTopicEvent::Copied`].
#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)]
pub struct DocumentCopiedMetadata {
@@ -105,6 +126,9 @@ pub enum DocumentTopicEvent {
/// A document was copied.
#[serde(rename = "document.copied")]
Copied(DocumentCopiedMetadata),
+ /// A peer joined, left, or a periodic save occurred.
+ #[serde(rename = "document.interaction")]
+ Interaction(DocumentInteractionMetadata),
}
impl TopicEvent for DocumentTopicEvent {
@@ -142,6 +166,11 @@ impl DocumentMacroEvent {
Self::new(key, DocumentTopicEvent::Copied(metadata))
}
+ /// Build an interaction event keyed by the document id.
+ pub fn interaction(key: impl Into<String>, metadata: DocumentInteractionMetadata) -> Self {
+ Self::new(key, DocumentTopicEvent::Interaction(metadata))
+ }
+
/// Build an event from a topic-specific event variant.
pub fn new(key: impl Into<String>, event: DocumentTopicEvent) -> Self {
Self::with_event(key, Event::new(event))
diff --git a/crates/documents/src/domain/ports/mod.rs b/crates/documents/src/domain/ports/mod.rs
index d5c100ed4e..fd024e76c4 100644
--- a/crates/documents/src/domain/ports/mod.rs
+++ b/crates/documents/src/domain/ports/mod.rs
@@ -16,6 +16,7 @@ use macro_user_id::user_id::MacroUserIdStr;
use model::document::{ContentType, DocumentBasic, DocumentMetadata};
use super::content::DocumentContent;
+use super::events::InteractionReason;
use super::response::{
CreateDocumentResponseData, DocumentResponse, GetDocumentResponseData, LocationResponseV3,
};
@@ -488,6 +489,14 @@ pub trait DocumentService: Send + Sync + 'static {
document_id: &str,
bytes: Vec<u8>,
) -> impl Future<Output = anyhow::Result<()>> + Send;
+
+ /// Publishes a `document.interaction` event: a real edit, a peer joining,
+ /// or the last peer leaving.
+ fn record_interaction(
+ &self,
+ document_id: &str,
+ reason: InteractionReason,
+ ) -> impl Future<Output = anyhow::Result<()>> + Send;
/// Get the team-share state of a document, resolved against the owner's team.
fn get_team_share(
&self,
diff --git a/crates/documents/src/domain/service.rs b/crates/documents/src/domain/service.rs
index c48472b200..97482c0f31 100644
--- a/crates/documents/src/domain/service.rs
+++ b/crates/documents/src/domain/service.rs
@@ -42,8 +42,8 @@ use crate::domain::models::{
use super::branch_name::{build_task_branch_name, user_branch_prefix};
use super::content::{DocumentContent, DocumentContentLocation, DocumentContentState};
use super::events::{
- DocumentCopiedMetadata, DocumentCreatedMetadata, DocumentDeletedMetadata, DocumentMacroEvent,
- DocumentUpdatedMetadata,
+ DocumentCopiedMetadata, DocumentCreatedMetadata, DocumentDeletedMetadata,
+ DocumentInteractionMetadata, DocumentMacroEvent, DocumentUpdatedMetadata, InteractionReason,
};
use super::models::{
CloudFrontConfig, CommentThread, CopyDocumentRepoArgs, CreateDocumentRepoArgs,
@@ -1613,7 +1613,23 @@ impl<
async fn upload_snapshot(&self, document_id: &str, bytes: Vec<u8>) -> anyhow::Result<()> {
self.upload_url_service
.upload_snapshot(document_id, bytes)
- .await
+ .await?;
+ Ok(())
+ }
+
+ async fn record_interaction(
+ &self,
+ document_id: &str,
+ reason: InteractionReason,
+ ) -> anyhow::Result<()> {
+ self.publish_document_event(&DocumentMacroEvent::interaction(
+ document_id,
+ DocumentInteractionMetadata {
+ document_id: document_id.to_owned(),
+ reason,
+ },
+ ));
+ Ok(())
}
/// Assigns the task properties to a document
diff --git a/crates/documents/src/inbound/axum_router.rs b/crates/documents/src/inbound/axum_router.rs
index 89b78b4e37..66e023f864 100644
--- a/crates/documents/src/inbound/axum_router.rs
+++ b/crates/documents/src/inbound/axum_router.rs
@@ -32,6 +32,7 @@ pub mod get_document_by_team_slug;
pub mod get_github_pull_requests;
pub mod get_location;
pub mod get_short_id;
+pub mod put_interaction;
pub mod put_snapshot;
pub mod task_duplicates;
pub mod team_share;
diff --git a/crates/documents/src/inbound/axum_router/put_interaction.rs b/crates/documents/src/inbound/axum_router/put_interaction.rs
new file mode 100644
index 0000000000..9d5837e3f3
--- /dev/null
+++ b/crates/documents/src/inbound/axum_router/put_interaction.rs
@@ -0,0 +1,46 @@
+//! Handler for `PUT /documents/{document_id}/interaction`.
+
+use axum::{
+ Json,
+ extract::{Path, State},
+ http::StatusCode,
+ response::IntoResponse,
+};
+use entity_access::domain::ports::EntityAccessService;
+use macro_authorization::{InternalMacroAuthorizationExtractor, MacroAuthorizationService};
+use serde::Deserialize;
+
+use super::{DocumentRouterState, Params};
+use crate::domain::events::InteractionReason;
+use crate::domain::ports::DocumentService;
+
+/// Request body for an interaction report from sync-service.
+#[derive(Deserialize)]
+pub struct InteractionRequest {
+ reason: InteractionReason,
+}
+
+/// Records a document interaction event.
+#[tracing::instrument(skip(state, body, _internal_authorization))]
+pub async fn put_interaction_handler<
+ T: DocumentService,
+ Svc: EntityAccessService,
+ Auth: MacroAuthorizationService,
+>(
+ State(state): State<DocumentRouterState<T, Svc, Auth>>,
+ _internal_authorization: InternalMacroAuthorizationExtractor<Auth>,
+ Path(Params { document_id }): Path<Params>,
+ Json(body): Json<InteractionRequest>,
+) -> impl IntoResponse {
+ match state
+ .service
+ .record_interaction(&document_id, body.reason)
+ .await
+ {
+ Ok(()) => StatusCode::OK.into_response(),
+ Err(e) => {
+ tracing::error!(error=?e, document_id=document_id, "failed to record interaction");
+ StatusCode::INTERNAL_SERVER_ERROR.into_response()
+ }
+ }
+}
diff --git a/crates/documents/src/inbound/axum_router/tests.rs b/crates/documents/src/inbound/axum_router/tests.rs
index 2fe174763f..871cc7f5a3 100644
--- a/crates/documents/src/inbound/axum_router/tests.rs
+++ b/crates/documents/src/inbound/axum_router/tests.rs
@@ -47,6 +47,7 @@ use crate::{
domain::{
content::DocumentContent,
create::DocumentCreator,
+ events::InteractionReason,
models::{
CommentThread, CreateDocumentRepoArgs, CreateTaskRequest, DocumentError,
DocumentTeamShareResponse, EditDocumentServiceArgs, GithubPullRequestsResponse,
@@ -369,6 +370,14 @@ impl DocumentService for FakeDocumentService {
Ok(())
}
+ async fn record_interaction(
+ &self,
+ _document_id: &str,
+ _reason: InteractionReason,
+ ) -> anyhow::Result<()> {
+ panic!("unexpected record_interaction call")
+ }
+
async fn get_team_share(
&self,
_entity_access_receipt: EntityAccessReceipt<entity_access::domain::models::ViewAccessLevel>,
diff --git a/crates/github/src/domain/service/sync/test.rs b/crates/github/src/domain/service/sync/test.rs
index 08545158c7..853f46f8f1 100644
--- a/crates/github/src/domain/service/sync/test.rs
+++ b/crates/github/src/domain/service/sync/test.rs
@@ -288,6 +288,14 @@ impl DocumentService for StubDocumentService {
async fn upload_snapshot(&self, _document_id: &str, _bytes: Vec<u8>) -> anyhow::Result<()> {
unimplemented!()
}
+
+ async fn record_interaction(
+ &self,
+ _document_id: &str,
+ _reason: documents::domain::events::InteractionReason,
+ ) -> anyhow::Result<()> {
+ unimplemented!()
+ }
}
/// Stateful stub repo that tracks task IDs per github key.
diff --git a/crates/webhook/src/domain/ingestion.rs b/crates/webhook/src/domain/ingestion.rs
index e77f1aa2e5..bac974c171 100644
--- a/crates/webhook/src/domain/ingestion.rs
+++ b/crates/webhook/src/domain/ingestion.rs
@@ -228,6 +228,9 @@ fn normalized_document_event(
DocumentTopicEvent::Updated(metadata) => ("document.updated", &metadata.document_id),
DocumentTopicEvent::Deleted(metadata) => ("document.deleted", &metadata.document_id),
DocumentTopicEvent::Copied(metadata) => ("document.copied", &metadata.document_id),
+ DocumentTopicEvent::Interaction(metadata) => {
+ ("document.interaction", &metadata.document_id)
+ }
};
if Uuid::parse_str(entity_id).is_err() {
diff --git a/docker/docker-compose.yml b/docker/docker-compose.yml
index 662149b910..8def1a8cc5 100644
--- a/docker/docker-compose.yml
+++ b/docker/docker-compose.yml
@@ -398,6 +398,7 @@ services:
environment:
- INTERNAL_API_SECRET_KEY=${INTERNAL_API_SECRET_KEY}
- DOCUMENT_PERMISSIONS_SECRET=${DOCUMENT_PERMISSION_JWT}
+ - DSS_INTERNAL_AUTH_KEY=${DOCUMENT_STORAGE_SERVICE_AUTH_KEY}
networks:
services:
aliases:
diff --git a/docker/sync-service.Dockerfile b/docker/sync-service.Dockerfile
index 9025b5b953..0d450eb6d2 100644
--- a/docker/sync-service.Dockerfile
+++ b/docker/sync-service.Dockerfile
@@ -58,4 +58,4 @@ EXPOSE 8787
# Generate .dev.vars from environment variables so wrangler can resolve secret bindings,
# apply local D1 migrations, then start.
-CMD sh -c 'printf "LOCAL_API_KEY=%s\nDOCUMENT_PERMISSIONS_SECRET=%s\nSPS_API_SECRET_KEY=%s\n" "$INTERNAL_API_SECRET_KEY" "$DOCUMENT_PERMISSIONS_SECRET" "$INTERNAL_API_SECRET_KEY" > .dev.vars && CI=true npx wrangler d1 migrations apply USER_PEER_MAPPING --local --config wrangler.docker.toml && exec npx wrangler dev --local --ip 0.0.0.0 --port 8787 --config wrangler.docker.toml'
+CMD sh -c 'printf "LOCAL_API_KEY=%s\nDOCUMENT_PERMISSIONS_SECRET=%s\nSPS_API_SECRET_KEY=%s\nDSS_INTERNAL_AUTH_KEY=%s\n" "$INTERNAL_API_SECRET_KEY" "$DOCUMENT_PERMISSIONS_SECRET" "$INTERNAL_API_SECRET_KEY" "$DSS_INTERNAL_AUTH_KEY" > .dev.vars && CI=true npx wrangler d1 migrations apply USER_PEER_MAPPING --local --config wrangler.docker.toml && exec npx wrangler dev --local --ip 0.0.0.0 --port 8787 --config wrangler.docker.toml'
diff --git a/services/document_storage_service/src/api/internal/mod.rs b/services/document_storage_service/src/api/internal/mod.rs
index 1907552b19..e6bbfe4fcf 100644
--- a/services/document_storage_service/src/api/internal/mod.rs
+++ b/services/document_storage_service/src/api/internal/mod.rs
@@ -136,6 +136,16 @@ pub fn router(state: ApiContext) -> Router<ApiContext> {
>,
),
)
+ .route(
+ "/documents/{document_id}/interaction",
+ put(
+ documents_hex::inbound::axum_router::put_interaction::put_interaction_handler::<
+ DocumentService,
+ EntityAccessService,
+ AuthorizationService,
+ >,
+ ),
+ )
.route("/documents/metadata", post(get_documents_metadata::handler))
// History routes
.route(
diff --git a/services/sync-service/src/dss_internal.rs b/services/sync-service/src/dss_internal.rs
index f8db6156b6..54ae81762d 100644
--- a/services/sync-service/src/dss_internal.rs
+++ b/services/sync-service/src/dss_internal.rs
@@ -1,14 +1,41 @@
+use serde::Serialize;
use tracing::error;
use worker::{Env, Fetch, Method, Request, RequestInit};
use crate::constants::header_names::MACRO_DOCUMENT_STORAGE_SERVICE_AUTH_HEADER_KEY;
+/// Why a document interaction was reported to DSS.
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize)]
+#[serde(rename_all = "snake_case")]
+pub enum InteractionReason {
+ /// A periodic save of pending content changes.
+ Edited,
+ /// The first peer joined the document session.
+ FirstJoin,
+ /// The last connected peer left the document session.
+ LastLeave,
+}
+
+#[derive(Serialize)]
+struct InteractionRequest {
+ reason: InteractionReason,
+}
+
pub trait DssInternal {
+ /// Uploads the raw snapshot bytes for storage, and publishes a
+ /// `document.edited` event.
async fn publish_shallow_snapshot(
&self,
document_id: &str,
snapshot: &[u8],
) -> worker::Result<()>;
+
+ /// Reports an interaction (join/leave/periodic edit).
+ async fn publish_interaction(
+ &self,
+ document_id: &str,
+ reason: InteractionReason,
+ ) -> worker::Result<()>;
}
pub struct DssInternalClient<'a> {
@@ -69,4 +96,46 @@ impl DssInternal for DssInternalClient<'_> {
tracing::trace!("snapshot uploaded to DSS");
Ok(())
}
+
+ async fn publish_interaction(
+ &self,
+ document_id: &str,
+ reason: InteractionReason,
+ ) -> worker::Result<()> {
+ let url = format!(
+ "{}/internal/documents/{}/interaction",
+ self.dss_url()?,
+ document_id
+ );
+ let auth_key = self.internal_auth_key()?;
+
+ let body = serde_json::to_vec(&InteractionRequest { reason })
+ .map_err(|e| worker::Error::from(format!("failed to serialize interaction: {e}")))?;
+
+ let mut req = Request::new_with_init(
+ &url,
+ RequestInit::new()
+ .with_method(Method::Put)
+ .with_body(Some(body.into())),
+ )?;
+ req.headers_mut()?
+ .set(MACRO_DOCUMENT_STORAGE_SERVICE_AUTH_HEADER_KEY, &auth_key)?;
+ req.headers_mut()?.set("Content-Type", "application/json")?;
+
+ let resp = Fetch::Request(req).send().await?;
+ if resp.status_code() != 200 {
+ error!(
+ document_id = document_id,
+ status = resp.status_code(),
+ "DSS interaction upload failed"
+ );
+ return Err(worker::Error::from(format!(
+ "DSS interaction upload returned {}",
+ resp.status_code()
+ )));
+ }
+
+ tracing::trace!("interaction uploaded to DSS");
+ Ok(())
+ }
}
diff --git a/services/sync-service/src/durable_object.rs b/services/sync-service/src/durable_object.rs
index 652d654e53..6c7d8fd3de 100644
--- a/services/sync-service/src/durable_object.rs
+++ b/services/sync-service/src/durable_object.rs
@@ -21,7 +21,7 @@ use crate::{
auth::{AccessLevel, TokenFrom, decode_jwt},
constants::USER_PEER_D1_BINDING,
d1::{PeerWithUserId, get_user_id_from_peer_id, insert_user_mapping},
- dss_internal::{DssInternal, DssInternalClient},
+ dss_internal::{DssInternal, DssInternalClient, InteractionReason},
error::ResultExt,
generated::schema::InitializeFromSnapshotRequest,
keepalive::{DEFAULT_TIME_TO_LIVE, keepalive},
@@ -287,6 +287,16 @@ async fn report_new_doc_state(document_id: &str, snapshot: &[u8], env: &Env) {
}
}
+/// Report an interaction (join/leave/periodic edit) to DSS.
+async fn report_interaction(document_id: &str, env: &Env, reason: InteractionReason) {
+ if let Err(err) = DssInternalClient::new(env)
+ .publish_interaction(document_id, reason)
+ .await
+ {
+ warn!(error=?err, "failed to push interaction to DSS");
+ }
+}
+
/// Schedule an alarm 5 seconds from now.
async fn bump_alarm(state: &State) -> Result<()> {
let current_alarm = state.storage().get_alarm().await?;
@@ -640,6 +650,10 @@ impl DocumentSyncSession {
// Below is websocket stuff only i.e connect
let pair = WebSocketPair::new().context("failed to create websocket pair")?;
+ // Whether this peer is the first to join the session (used to
+ // decide whether to report a `FirstJoin` interaction below).
+ let is_first_join = self.state.get_websockets().is_empty();
+
// create tag for ws and store it
let ws_id = new_ws_id();
trace!(ws_id = ws_id, "websocket connect");
@@ -681,6 +695,18 @@ impl DocumentSyncSession {
);
}
+ // This is the single source of truth for `FirstJoin`: whenever
+ // the peer count genuinely transitions 0 -> 1, regardless of
+ // whether the document already has content.
+ if is_first_join {
+ let document_id_owned = document_id.to_string();
+ let env = self.env.clone();
+ self.state.wait_until(async move {
+ report_interaction(&document_id_owned, &env, InteractionReason::FirstJoin)
+ .await;
+ });
+ }
+
Response::from_websocket(pair.client).context("failed to create websocket response")?
});
@@ -999,6 +1025,7 @@ impl DurableObject for DocumentSyncSession {
&& let Ok(snapshot) = doc_state.export_shallow_snapshot()
{
report_new_doc_state(&document_id, &snapshot, &env).await;
+ report_interaction(&document_id, &env, InteractionReason::Edited).await;
}
});
}
@@ -1051,6 +1078,7 @@ impl DurableObject for DocumentSyncSession {
let env = self.env.clone();
self.state.wait_until(async move {
report_new_doc_state(&document_id, &snapshot, &env).await;
+ report_interaction(&document_id, &env, InteractionReason::LastLeave).await;
});
}
Ok(())