Skip to content

World's first open-source enterprise world model.

Reconnect notification streams after broadcast lag to recover missed updates (#1080)

Daniel229b8469fceb
diff --git a/crates/utopia-server/src/api/alerts_routes.rs b/crates/utopia-server/src/api/alerts_routes.rs
index 07c1acd3..5683a052 100644
--- a/crates/utopia-server/src/api/alerts_routes.rs
+++ b/crates/utopia-server/src/api/alerts_routes.rs
@@ -16,7 +16,7 @@ use uuid::Uuid;
use crate::auth::AuthUser;
use crate::error::ApiResult;
-use crate::state::AppState;
+use crate::state::{AppEvent, AppState};
/// 一页几组。弹窗里放得下的量——再多就该翻页,而不是让人滚一屏。
const PAGE: i64 = 8;
@@ -139,7 +139,12 @@ pub async fn stream(
State(state): State<AppState>,
AuthUser(_user): AuthUser,
) -> ApiResult<Sse<impl Stream<Item = Result<Event, Infallible>>>> {
- let mut rx = state.events.subscribe();
+ Ok(alert_event_stream(state.events.subscribe()))
+}
+
+fn alert_event_stream(
+ mut rx: broadcast::Receiver<AppEvent>,
+) -> Sse<impl Stream<Item = Result<Event, Infallible>>> {
let stream = async_stream::stream! {
loop {
match rx.recv().await {
@@ -147,10 +152,111 @@ pub async fn stream(
yield Ok(Event::default().event("alert").data("{}"));
}
Ok(_) => continue,
- Err(broadcast::error::RecvError::Lagged(_)) => continue,
+ // 唯一的告警可能已被其他类型的通知挤掉。结束响应,让 EventSource
+ // 重连后通过 onRecover 补刷,即使之后再没有新告警。
+ Err(broadcast::error::RecvError::Lagged(_)) => return,
Err(broadcast::error::RecvError::Closed) => return,
}
}
};
- Ok(Sse::new(stream).keep_alive(KeepAlive::default()))
+ Sse::new(stream).keep_alive(KeepAlive::default())
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+ use axum::response::IntoResponse;
+ use axum::{routing::get, Router};
+ use futures_util::{FutureExt, StreamExt};
+ use std::time::Duration;
+
+ #[tokio::test]
+ async fn lagged_alert_stream_ends_when_non_alert_events_displace_its_only_alert() {
+ let (sender, _) = broadcast::channel(256);
+ let route_sender = sender.clone();
+ let app = Router::new().route(
+ "/events",
+ get(move || {
+ let sender = route_sender.clone();
+ async move {
+ let receiver = sender.subscribe();
+ sender
+ .send(AppEvent {
+ kb_id: None,
+ kind: "alert",
+ document_id: None,
+ })
+ .unwrap();
+ for _ in 0..257 {
+ sender
+ .send(AppEvent {
+ kb_id: Some(Uuid::now_v7()),
+ kind: "document",
+ document_id: None,
+ })
+ .unwrap();
+ }
+ alert_event_stream(receiver)
+ }
+ }),
+ );
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let address = listener.local_addr().unwrap();
+ let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
+ let client = reqwest::Client::builder().no_proxy().build().unwrap();
+ let request = async {
+ let response = client
+ .get(format!("http://{address}/events"))
+ .send()
+ .await
+ .unwrap();
+ assert_eq!(response.status(), reqwest::StatusCode::OK);
+ assert_eq!(response.headers()["content-type"], "text/event-stream");
+ response.text().await.unwrap()
+ };
+ // 保留 sender:被过滤的剩余事件不能让漏掉告警的连接一直开着。
+ let result = tokio::time::timeout(Duration::from_secs(2), request).await;
+ server.abort();
+ assert!(result
+ .expect("a lagged alert SSE response must end")
+ .is_empty());
+ }
+
+ #[tokio::test]
+ async fn alert_stream_filters_non_alerts_and_never_exposes_alert_details() {
+ let (sender, receiver) = broadcast::channel(256);
+ let response = alert_event_stream(receiver).into_response();
+ assert_eq!(response.headers()["content-type"], "text/event-stream");
+ let mut body = response.into_body().into_data_stream();
+ assert!(body.next().now_or_never().is_none());
+ sender
+ .send(AppEvent {
+ kb_id: Some(Uuid::now_v7()),
+ kind: "document",
+ document_id: None,
+ })
+ .unwrap();
+ assert!(body.next().now_or_never().is_none());
+ sender
+ .send(AppEvent {
+ kb_id: Some(Uuid::now_v7()),
+ kind: "alert",
+ document_id: Some(Uuid::now_v7()),
+ })
+ .unwrap();
+ let frame = body.next().await.unwrap().unwrap();
+ assert_eq!(frame.as_ref(), b"event: alert\ndata: {}\n\n");
+ assert!(body.next().now_or_never().is_none());
+ drop(sender);
+ assert!(body.next().await.is_none());
+ }
+
+ #[tokio::test]
+ async fn dropping_an_alert_response_releases_its_subscription() {
+ let (sender, receiver) = broadcast::channel(256);
+ let response = alert_event_stream(receiver).into_response();
+ assert_eq!(sender.receiver_count(), 1);
+ drop(response);
+ assert_eq!(sender.receiver_count(), 0);
+ }
}
diff --git a/crates/utopia-server/src/api/events_routes.rs b/crates/utopia-server/src/api/events_routes.rs
index ad2ffe9d..aa99b28d 100644
--- a/crates/utopia-server/src/api/events_routes.rs
+++ b/crates/utopia-server/src/api/events_routes.rs
@@ -24,7 +24,13 @@ pub async fn kb_events(
) -> ApiResult<Sse<impl Stream<Item = Result<Event, Infallible>>>> {
utopia_store::access::require_kb(&state.pool, &user, kb_id, Role::Viewer).await?;
- let mut rx = state.events.subscribe();
+ Ok(kb_event_stream(state.events.subscribe(), kb_id))
+}
+
+fn kb_event_stream(
+ mut rx: broadcast::Receiver<AppEvent>,
+ kb_id: Uuid,
+) -> Sse<impl Stream<Item = Result<Event, Infallible>>> {
let stream = async_stream::stream! {
loop {
match rx.recv().await {
@@ -35,13 +41,14 @@ pub async fn kb_events(
.data(serde_json::to_string(&ev).unwrap_or_else(|_| "{}".into()))),
None => continue,
},
- // 消费落后被跳帧:无所谓,事件只是"该刷新了"的信号
- Err(broadcast::error::RecvError::Lagged(_)) => continue,
+ // 丢掉的可能是这个库或告警的唯一通知;余下事件可能全部被过滤。
+ // 结束响应,让 EventSource 重连后通过 onRecover 补刷。
+ Err(broadcast::error::RecvError::Lagged(_)) => return,
Err(broadcast::error::RecvError::Closed) => return,
}
}
};
- Ok(Sse::new(stream).keep_alive(KeepAlive::default()))
+ Sse::new(stream).keep_alive(KeepAlive::default())
}
#[derive(Debug, PartialEq)]
@@ -66,6 +73,125 @@ fn relay(ev: &AppEvent, kb_id: Uuid) -> Option<Relay> {
#[cfg(test)]
mod tests {
use super::*;
+ use axum::response::IntoResponse;
+ use axum::{routing::get, Router};
+ use futures_util::{FutureExt, StreamExt};
+ use std::time::Duration;
+
+ #[tokio::test]
+ async fn lagged_kb_stream_ends_when_its_only_document_or_global_alert_is_lost() {
+ let (here, elsewhere) = (Uuid::now_v7(), Uuid::now_v7());
+ for (kb_id, kind) in [(Some(here), "document"), (None, "alert")] {
+ let (sender, _) = broadcast::channel(256);
+ let route_sender = sender.clone();
+ let app = Router::new().route(
+ "/events",
+ get(move || {
+ let sender = route_sender.clone();
+ async move {
+ let receiver = sender.subscribe();
+ sender
+ .send(AppEvent {
+ kb_id,
+ kind,
+ document_id: None,
+ })
+ .unwrap();
+ // 首次 poll 前确定性地挤掉唯一有效通知,剩下的全部会被过滤。
+ for _ in 0..257 {
+ sender
+ .send(AppEvent {
+ kb_id: Some(elsewhere),
+ kind: "document",
+ document_id: None,
+ })
+ .unwrap();
+ }
+ kb_event_stream(receiver, here)
+ }
+ }),
+ );
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();
+ let address = listener.local_addr().unwrap();
+ let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });
+ let client = reqwest::Client::builder().no_proxy().build().unwrap();
+ let request = async {
+ let response = client
+ .get(format!("http://{address}/events"))
+ .send()
+ .await
+ .unwrap();
+ assert_eq!(response.status(), reqwest::StatusCode::OK);
+ assert_eq!(response.headers()["content-type"], "text/event-stream");
+ response.text().await.unwrap()
+ };
+ // 保留 sender,确保真实 HTTP 响应结束来自 Lagged,而不是通道关闭。
+ let result = tokio::time::timeout(Duration::from_secs(2), request).await;
+ server.abort();
+ assert!(result
+ .expect("a lagged KB SSE response must end")
+ .is_empty());
+ }
+ }
+
+ #[tokio::test]
+ async fn kb_stream_keeps_filtering_and_alert_payloads_until_the_channel_closes() {
+ let (here, elsewhere) = (Uuid::now_v7(), Uuid::now_v7());
+ let (sender, receiver) = broadcast::channel(256);
+ let response = kb_event_stream(receiver, here).into_response();
+ assert_eq!(response.headers()["content-type"], "text/event-stream");
+ let mut body = response.into_body().into_data_stream();
+ // 初次连接不凭空发刷新事件,也不结束响应。
+ assert!(body.next().now_or_never().is_none());
+ sender
+ .send(AppEvent {
+ kb_id: Some(elsewhere),
+ kind: "document",
+ document_id: None,
+ })
+ .unwrap();
+ assert!(body.next().now_or_never().is_none());
+
+ let document = AppEvent {
+ kb_id: Some(here),
+ kind: "document",
+ document_id: Some(Uuid::now_v7()),
+ };
+ sender.send(document.clone()).unwrap();
+ let frame = body.next().await.unwrap().unwrap();
+ assert_eq!(
+ frame.as_ref(),
+ format!(
+ "event: document\ndata: {}\n\n",
+ serde_json::to_string(&document).unwrap()
+ )
+ .as_bytes()
+ );
+
+ for kb_id in [None, Some(here), Some(elsewhere)] {
+ sender
+ .send(AppEvent {
+ kb_id,
+ kind: "alert",
+ document_id: Some(Uuid::now_v7()),
+ })
+ .unwrap();
+ let frame = body.next().await.unwrap().unwrap();
+ assert_eq!(frame.as_ref(), b"event: alert\ndata: {}\n\n");
+ }
+ assert!(body.next().now_or_never().is_none());
+ drop(sender);
+ assert!(body.next().await.is_none());
+ }
+
+ #[tokio::test]
+ async fn dropping_a_kb_response_releases_its_subscription() {
+ let (sender, receiver) = broadcast::channel(256);
+ let response = kb_event_stream(receiver, Uuid::now_v7()).into_response();
+ assert_eq!(sender.receiver_count(), 1);
+ drop(response);
+ assert_eq!(sender.receiver_count(), 0);
+ }
#[test]
fn a_kb_stream_carries_its_own_events_and_every_alert() {
diff --git a/web/src/useEventReconnect.test.ts b/web/src/useEventReconnect.test.ts
index 33fbf9c8..2e37fe5c 100644
--- a/web/src/useEventReconnect.test.ts
+++ b/web/src/useEventReconnect.test.ts
@@ -128,9 +128,12 @@ afterEach(() => {
});
describe("KB stream reconnect", () => {
- it("refetches missed changes immediately for every stream key in the current KB", async () => {
+ it("refetches every current KB stream key and alerts after reconnect without new business events", async () => {
const client = createClient();
const current = await Promise.all(STREAM_KEYS.map((head) => watch(client, [head, "kb-a", "detail"])));
+ const badge = await watch(client, ["alerts", "unread"]);
+ const list = await watch(client, ["alerts", "list", "", 0]);
+ const recoveredQueries = [...current, badge, list];
const otherKb = await watch(client, ["graph", "kb-b"]);
const unrelated = await watch(client, ["health"]);
const inactiveKey = ["documents", "kb-a", "unmounted"];
@@ -138,20 +141,23 @@ describe("KB stream reconnect", () => {
const { source } = mount(() => useKbEvents("kb-a"));
expect(source.url).toBe("/api/v1/kbs/kb-a/events");
source.emit("open");
- for (const query of current) expect(query.fetch).toHaveBeenCalledTimes(1);
+ for (const query of recoveredQueries) expect(query.fetch).toHaveBeenCalledTimes(1);
source.emit("error");
- for (const query of current) query.changeServer();
+ for (const query of recoveredQueries) query.changeServer();
// staleTime 到期只让缓存变旧,不会自动请求;断线期间完全没有业务事件。
await vi.advanceTimersByTimeAsync(STREAM_STALE_MS + 1);
- for (const query of current) expect(query.fetch).toHaveBeenCalledTimes(1);
+ for (const query of recoveredQueries) expect(query.fetch).toHaveBeenCalledTimes(1);
source.emit("open");
// 重连不等普通事件的 300ms 合并窗口。
- for (const query of current) expect(query.fetch).toHaveBeenCalledTimes(2);
+ for (const query of recoveredQueries) expect(query.fetch).toHaveBeenCalledTimes(2);
await vi.waitFor(() => {
- for (const query of current) expect(query.observer.getCurrentResult().data).toBe("after disconnect");
+ for (const query of recoveredQueries) expect(query.observer.getCurrentResult().data).toBe("after disconnect");
});
+ source.emit("open");
+ await vi.advanceTimersByTimeAsync(SETTLE_MS);
+ for (const query of recoveredQueries) expect(query.fetch).toHaveBeenCalledTimes(2);
expect(client.getQueryState(inactiveKey)?.isInvalidated).toBe(true);
expect(otherKb.fetch).toHaveBeenCalledTimes(1);
expect(unrelated.fetch).toHaveBeenCalledTimes(1);
@@ -173,17 +179,25 @@ describe("KB stream reconnect", () => {
const client = createClient();
const graph = await watch(client, ["graph", "kb-a"]);
const documents = await watch(client, ["documents", "kb-a"]);
+ const badge = await watch(client, ["alerts", "unread"]);
+ const list = await watch(client, ["alerts", "list", "", 0]);
const { source } = mount(() => useKbEvents("kb-a"));
source.emit("open");
source.emit("document");
source.emit("graph");
+ source.emit("alert");
+ source.emit("alert");
source.emit("error");
source.emit("open");
expect(graph.fetch).toHaveBeenCalledTimes(2);
expect(documents.fetch).toHaveBeenCalledTimes(2);
+ expect(badge.fetch).toHaveBeenCalledTimes(2);
+ expect(list.fetch).toHaveBeenCalledTimes(2);
await vi.advanceTimersByTimeAsync(SETTLE_MS);
expect(graph.fetch).toHaveBeenCalledTimes(2);
expect(documents.fetch).toHaveBeenCalledTimes(2);
+ expect(badge.fetch).toHaveBeenCalledTimes(2);
+ expect(list.fetch).toHaveBeenCalledTimes(2);
expect(vi.getTimerCount()).toBe(0);
});
@@ -209,15 +223,18 @@ describe("KB stream reconnect", () => {
const client = createClient();
const oldKb = await watch(client, ["graph", "kb-a"]);
const newKb = await watch(client, ["graph", "kb-b"]);
+ const badge = await watch(client, ["alerts", "unread"]);
+ const list = await watch(client, ["alerts", "list", "", 0]);
const old = mount(() => useKbEvents("kb-a"));
old.source.emit("open");
old.source.emit("error");
const lateOpen = [...old.source.listeners.get("open") ?? []];
const lateGraph = [...old.source.listeners.get("graph") ?? []];
+ const lateAlert = [...old.source.listeners.get("alert") ?? []];
old.unmount();
const next = mount(() => useKbEvents("kb-b"));
next.source.emit("open");
- for (const callback of [...lateOpen, ...lateGraph]) callback();
+ for (const callback of [...lateOpen, ...lateGraph, ...lateAlert]) callback();
old.source.emit("open");
old.source.emit("document");
await vi.advanceTimersByTimeAsync(SETTLE_MS);
@@ -226,10 +243,14 @@ describe("KB stream reconnect", () => {
expect(vi.getTimerCount()).toBe(0);
expect(oldKb.fetch).toHaveBeenCalledTimes(1);
expect(newKb.fetch).toHaveBeenCalledTimes(1);
+ expect(badge.fetch).toHaveBeenCalledTimes(1);
+ expect(list.fetch).toHaveBeenCalledTimes(1);
next.source.emit("error");
next.source.emit("open");
expect(oldKb.fetch).toHaveBeenCalledTimes(1);
expect(newKb.fetch).toHaveBeenCalledTimes(2);
+ expect(badge.fetch).toHaveBeenCalledTimes(2);
+ expect(list.fetch).toHaveBeenCalledTimes(2);
});
it("does not subscribe without a selected KB", () => {