Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(documents): await sync-service init when finalizing markdown uploads (#7710)

Wolf Mermelsteine8423300a504
diff --git a/crates/documents/src/inbound/toolset.rs b/crates/documents/src/inbound/toolset.rs
index 49698939cb..18739589b7 100644
--- a/crates/documents/src/inbound/toolset.rs
+++ b/crates/documents/src/inbound/toolset.rs
@@ -203,7 +203,7 @@ impl<
let sync_service_client = Arc::new(sync_service_client);
let creator = DocumentCreator::new(
service.clone(),
- LexicalSyncMarkdownInitializer::new(
+ LexicalSyncMarkdownInitializer::detached(
lexical_client.as_ref().clone(),
sync_service_client.as_ref().clone(),
),
diff --git a/crates/documents/src/outbound/markdown_init.rs b/crates/documents/src/outbound/markdown_init.rs
index 1f05c5f1a4..3bd1af0b1e 100644
--- a/crates/documents/src/outbound/markdown_init.rs
+++ b/crates/documents/src/outbound/markdown_init.rs
@@ -9,8 +9,8 @@ use std::sync::Arc;
use std::time::Duration;
use lexical_client::LexicalClient;
-use sync_service_client::SyncServiceClient;
-use tokio_retry::{Retry, strategy::FixedInterval};
+use sync_service_client::{SyncServiceClient, initialize::SnapshotAlreadyExists};
+use tokio_retry::{RetryIf, strategy::FixedInterval};
use crate::domain::models::DocumentError;
use crate::domain::ports::markdown::{
@@ -36,6 +36,17 @@ impl SyncInitializeSnapshotPort for SyncServiceClient {
}
}
+/// When the sync-service snapshot write finishes relative to the caller.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum SyncInitialization {
+ /// Return only after sync-service holds the snapshot.
+ Awaited,
+ /// Return immediately and write the snapshot in a background task. Only
+ /// sound in a long-lived process: a Lambda freezes, and a CLI exits, with
+ /// the task still pending, leaving a document that never loads.
+ Detached,
+}
+
/// Markdown initializer backed by lexical-service and sync-service clients.
///
/// Generic over the two port traits so tests can substitute mocks; defaults
@@ -44,14 +55,26 @@ impl SyncInitializeSnapshotPort for SyncServiceClient {
pub struct LexicalSyncMarkdownInitializer<L = LexicalClient, S = SyncServiceClient> {
lexical_client: L,
sync_service_client: Arc<S>,
+ sync_initialization: SyncInitialization,
}
impl<L, S> LexicalSyncMarkdownInitializer<L, S> {
- /// Construct a lexical/sync-backed markdown initializer.
+ /// Construct an initializer that returns once sync-service holds the snapshot.
pub fn new(lexical_client: L, sync_service_client: S) -> Self {
Self {
lexical_client,
sync_service_client: Arc::new(sync_service_client),
+ sync_initialization: SyncInitialization::Awaited,
+ }
+ }
+
+ /// Construct an initializer that writes the snapshot in a background task,
+ /// so optimistic document creation can return before sync-service does.
+ /// Only for long-lived servers.
+ pub fn detached(lexical_client: L, sync_service_client: S) -> Self {
+ Self {
+ sync_initialization: SyncInitialization::Detached,
+ ..Self::new(lexical_client, sync_service_client)
}
}
}
@@ -67,7 +90,7 @@ where
document_id: &str,
markdown: &str,
) -> Result<Vec<u8>, DocumentError> {
- let loro_snapshot = if markdown.is_empty() {
+ let loro_snapshot: Vec<u8> = if markdown.is_empty() {
MARKDOWN_GOLDEN_SNAPSHOT.into()
} else {
self.lexical_client
@@ -76,46 +99,68 @@ where
.map_err(DocumentError::Internal)?
};
- let sync_service_client = self.sync_service_client.clone();
- let document_id = document_id.to_owned();
- let initial_snapshot = loro_snapshot.clone();
- tokio::spawn(async move {
- const MAX_ATTEMPTS: usize = 3;
- const RETRY_DELAY: Duration = Duration::from_secs(1);
-
- let loro_snapshot: Arc<[u8]> = loro_snapshot.into();
- let mut attempt = 0usize;
- let result = Retry::start(
- FixedInterval::new(RETRY_DELAY).take(MAX_ATTEMPTS - 1),
- || {
- attempt += 1;
- let sync_service_client = Arc::clone(&sync_service_client);
- let document_id = document_id.clone();
- let loro_snapshot = Arc::clone(&loro_snapshot);
- async move {
- let result = sync_service_client
- .initialize_from_snapshot(&document_id, &loro_snapshot)
- .await;
- if let Err(error) = &result
- && attempt < MAX_ATTEMPTS
- {
- tracing::warn!(error=?error, attempt, "failed to initialize sync service from snapshot, retrying in 1s");
- }
- result
+ match self.sync_initialization {
+ SyncInitialization::Awaited => {
+ initialize_with_retry(&*self.sync_service_client, document_id, &loro_snapshot)
+ .await
+ .map_err(DocumentError::Internal)?;
+ }
+ SyncInitialization::Detached => {
+ let sync_service_client = Arc::clone(&self.sync_service_client);
+ let document_id = document_id.to_owned();
+ let snapshot = loro_snapshot.clone();
+ tokio::spawn(async move {
+ if let Err(error) =
+ initialize_with_retry(&*sync_service_client, &document_id, &snapshot).await
+ {
+ tracing::error!(error=?error, %document_id, "failed to initialize sync service from snapshot");
}
- },
- )
- .await;
-
- if let Err(error) = result {
- tracing::error!(error=?error, "failed to initialize sync service from snapshot after {MAX_ATTEMPTS} attempts");
+ });
}
- });
+ }
- Ok(initial_snapshot)
+ Ok(loro_snapshot)
}
}
+/// POST the snapshot to sync-service, retrying transient failures. A 409 is
+/// returned as-is: retrying cannot change it, and callers decide whether an
+/// existing snapshot is acceptable.
+async fn initialize_with_retry<S: SyncInitializeSnapshotPort>(
+ sync_service_client: &S,
+ document_id: &str,
+ snapshot: &[u8],
+) -> anyhow::Result<()> {
+ const MAX_ATTEMPTS: usize = 3;
+ const RETRY_DELAY: Duration = Duration::from_secs(1);
+
+ let mut attempt = 0usize;
+ RetryIf::start(
+ FixedInterval::new(RETRY_DELAY).take(MAX_ATTEMPTS - 1),
+ || {
+ attempt += 1;
+ async move {
+ let result = sync_service_client
+ .initialize_from_snapshot(document_id, snapshot)
+ .await;
+ if let Err(error) = &result
+ && attempt < MAX_ATTEMPTS
+ && !is_snapshot_already_exists(error)
+ {
+ tracing::warn!(error=?error, attempt, "failed to initialize sync service from snapshot, retrying in 1s");
+ }
+ result
+ }
+ },
+ |error: &anyhow::Error| !is_snapshot_already_exists(error),
+ )
+ .await
+}
+
+fn is_snapshot_already_exists(error: &anyhow::Error) -> bool {
+ error.is::<SnapshotAlreadyExists>()
+}
+
#[cfg(test)]
mod tests {
use super::*;
@@ -162,4 +207,66 @@ mod tests {
.unwrap();
assert_eq!(snapshot, [1, 2, 3]);
}
+
+ #[tokio::test(start_paused = true)]
+ async fn awaited_returns_sync_failure_after_retrying() {
+ let mut lexical = MockLexicalSnapshotPort::new();
+ let mut sync = MockSyncInitializeSnapshotPort::new();
+
+ lexical
+ .expect_markdown_to_loro_snapshot()
+ .times(1)
+ .returning(|_| Box::pin(async { Ok(vec![1, 2, 3]) }));
+ sync.expect_initialize_from_snapshot()
+ .times(3)
+ .returning(|_, _| Box::pin(async { Err(anyhow::anyhow!("sync-service unavailable")) }));
+
+ let initializer = LexicalSyncMarkdownInitializer::new(lexical, sync);
+ let error = initializer
+ .initialize_existing_markdown("doc3", "# hi")
+ .await
+ .unwrap_err();
+ assert_eq!(error.to_string(), "sync-service unavailable");
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn awaited_does_not_retry_an_existing_snapshot() {
+ let mut lexical = MockLexicalSnapshotPort::new();
+ let mut sync = MockSyncInitializeSnapshotPort::new();
+
+ lexical
+ .expect_markdown_to_loro_snapshot()
+ .times(1)
+ .returning(|_| Box::pin(async { Ok(vec![1, 2, 3]) }));
+ sync.expect_initialize_from_snapshot()
+ .times(1)
+ .returning(|_, _| Box::pin(async { Err(SnapshotAlreadyExists.into()) }));
+
+ let initializer = LexicalSyncMarkdownInitializer::new(lexical, sync);
+ let error = initializer
+ .initialize_existing_markdown("doc4", "# hi")
+ .await
+ .unwrap_err();
+ assert_eq!(error.to_string(), "snapshot already exists");
+ }
+
+ #[tokio::test(start_paused = true)]
+ async fn detached_returns_the_snapshot_without_waiting_for_sync() {
+ let mut lexical = MockLexicalSnapshotPort::new();
+ let mut sync = MockSyncInitializeSnapshotPort::new();
+
+ lexical
+ .expect_markdown_to_loro_snapshot()
+ .times(1)
+ .returning(|_| Box::pin(async { Ok(vec![1, 2, 3]) }));
+ sync.expect_initialize_from_snapshot()
+ .returning(|_, _| Box::pin(async { Err(anyhow::anyhow!("sync-service unavailable")) }));
+
+ let initializer = LexicalSyncMarkdownInitializer::detached(lexical, sync);
+ let snapshot = initializer
+ .initialize_existing_markdown("doc5", "# hi")
+ .await
+ .unwrap();
+ assert_eq!(snapshot, [1, 2, 3]);
+ }
}
diff --git a/services/document_storage_service/src/server.rs b/services/document_storage_service/src/server.rs
index dd9038d39d..39ab28cea4 100644
--- a/services/document_storage_service/src/server.rs
+++ b/services/document_storage_service/src/server.rs
@@ -510,7 +510,7 @@ pub async fn run() -> anyhow::Result<()> {
config.docx_document_upload_bucket.as_ref(),
);
let markdown_initializer =
- documents_hex::outbound::markdown_init::LexicalSyncMarkdownInitializer::new(
+ documents_hex::outbound::markdown_init::LexicalSyncMarkdownInitializer::detached(
lexical_client.as_ref().clone(),
sync_service_client.as_ref().clone(),
);