World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Reconnect notification streams after broadcast lag to recover missed updates (#1080)
Daniel229b8469fceb
diff --git a/crates/utopia-server/src/api/alerts_routes.rs b/crates/utopia-server/src/api/alerts_routes.rsindex 07c1acd3..5683a052 100644--- a/crates/utopia-server/src/api/alerts_routes.rs+++ b/crates/utopia-server/src/api/alerts_routes.rs@@ -16,7 +16,7 @@ use uuid::Uuid;use crate::auth::AuthUser;use crate::error::ApiResult;-use crate::state::AppState;+use crate::state::{AppEvent, AppState};/// 一页几组。弹窗里放得下的量——再多就该翻页,而不是让人滚一屏。const PAGE: i64 = 8;@@ -139,7 +139,12 @@ pub async fn stream(State(state): State<AppState>,AuthUser(_user): AuthUser,) -> ApiResult<Sse<impl Stream<Item = Result<Event, Infallible>>>> {- let mut rx = state.events.subscribe();+ Ok(alert_event_stream(state.events.subscribe()))+}++fn alert_event_stream(+ mut rx: broadcast::Receiver<AppEvent>,+) -> Sse<impl Stream<Item = Result<Event, Infallible>>> {let stream = async_stream::stream! {loop {match rx.recv().await {@@ -147,10 +152,111 @@ pub async fn stream(yield Ok(Event::default().event("alert").data("{}"));}Ok(_) => continue,- Err(broadcast::error::RecvError::Lagged(_)) => continue,+ // 唯一的告警可能已被其他类型的通知挤掉。结束响应,让 EventSource+ // 重连后通过 onRecover 补刷,即使之后再没有新告警。+ Err(broadcast::error::RecvError::Lagged(_)) => return,Err(broadcast::error::RecvError::Closed) => return,}}};- Ok(Sse::new(stream).keep_alive(KeepAlive::default()))+ Sse::new(stream).keep_alive(KeepAlive::default())+}++#[cfg(test)]+mod tests {+ use super::*;+ use axum::response::IntoResponse;+ use axum::{routing::get, Router};+ use futures_util::{FutureExt, StreamExt};+ use std::time::Duration;++ #[tokio::test]+ async fn lagged_alert_stream_ends_when_non_alert_events_displace_its_only_alert() {+ let (sender, _) = broadcast::channel(256);+ let route_sender = sender.clone();+ let app = Router::new().route(+ "/events",+ get(move || {+ let sender = route_sender.clone();+ async move {+ let receiver = sender.subscribe();+ sender+ .send(AppEvent {+ kb_id: None,+ kind: "alert",+ document_id: None,+ })+ .unwrap();+ for _ in 0..257 {+ sender+ .send(AppEvent {+ kb_id: Some(Uuid::now_v7()),+ kind: "document",+ document_id: None,+ })+ .unwrap();+ }+ alert_event_stream(receiver)+ }+ }),+ );+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();+ let address = listener.local_addr().unwrap();+ let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });+ let client = reqwest::Client::builder().no_proxy().build().unwrap();+ let request = async {+ let response = client+ .get(format!("http://{address}/events"))+ .send()+ .await+ .unwrap();+ assert_eq!(response.status(), reqwest::StatusCode::OK);+ assert_eq!(response.headers()["content-type"], "text/event-stream");+ response.text().await.unwrap()+ };+ // 保留 sender:被过滤的剩余事件不能让漏掉告警的连接一直开着。+ let result = tokio::time::timeout(Duration::from_secs(2), request).await;+ server.abort();+ assert!(result+ .expect("a lagged alert SSE response must end")+ .is_empty());+ }++ #[tokio::test]+ async fn alert_stream_filters_non_alerts_and_never_exposes_alert_details() {+ let (sender, receiver) = broadcast::channel(256);+ let response = alert_event_stream(receiver).into_response();+ assert_eq!(response.headers()["content-type"], "text/event-stream");+ let mut body = response.into_body().into_data_stream();+ assert!(body.next().now_or_never().is_none());+ sender+ .send(AppEvent {+ kb_id: Some(Uuid::now_v7()),+ kind: "document",+ document_id: None,+ })+ .unwrap();+ assert!(body.next().now_or_never().is_none());+ sender+ .send(AppEvent {+ kb_id: Some(Uuid::now_v7()),+ kind: "alert",+ document_id: Some(Uuid::now_v7()),+ })+ .unwrap();+ let frame = body.next().await.unwrap().unwrap();+ assert_eq!(frame.as_ref(), b"event: alert\ndata: {}\n\n");+ assert!(body.next().now_or_never().is_none());+ drop(sender);+ assert!(body.next().await.is_none());+ }++ #[tokio::test]+ async fn dropping_an_alert_response_releases_its_subscription() {+ let (sender, receiver) = broadcast::channel(256);+ let response = alert_event_stream(receiver).into_response();+ assert_eq!(sender.receiver_count(), 1);+ drop(response);+ assert_eq!(sender.receiver_count(), 0);+ }}diff --git a/crates/utopia-server/src/api/events_routes.rs b/crates/utopia-server/src/api/events_routes.rsindex ad2ffe9d..aa99b28d 100644--- a/crates/utopia-server/src/api/events_routes.rs+++ b/crates/utopia-server/src/api/events_routes.rs@@ -24,7 +24,13 @@ pub async fn kb_events() -> ApiResult<Sse<impl Stream<Item = Result<Event, Infallible>>>> {utopia_store::access::require_kb(&state.pool, &user, kb_id, Role::Viewer).await?;- let mut rx = state.events.subscribe();+ Ok(kb_event_stream(state.events.subscribe(), kb_id))+}++fn kb_event_stream(+ mut rx: broadcast::Receiver<AppEvent>,+ kb_id: Uuid,+) -> Sse<impl Stream<Item = Result<Event, Infallible>>> {let stream = async_stream::stream! {loop {match rx.recv().await {@@ -35,13 +41,14 @@ pub async fn kb_events(.data(serde_json::to_string(&ev).unwrap_or_else(|_| "{}".into()))),None => continue,},- // 消费落后被跳帧:无所谓,事件只是"该刷新了"的信号- Err(broadcast::error::RecvError::Lagged(_)) => continue,+ // 丢掉的可能是这个库或告警的唯一通知;余下事件可能全部被过滤。+ // 结束响应,让 EventSource 重连后通过 onRecover 补刷。+ Err(broadcast::error::RecvError::Lagged(_)) => return,Err(broadcast::error::RecvError::Closed) => return,}}};- Ok(Sse::new(stream).keep_alive(KeepAlive::default()))+ Sse::new(stream).keep_alive(KeepAlive::default())}#[derive(Debug, PartialEq)]@@ -66,6 +73,125 @@ fn relay(ev: &AppEvent, kb_id: Uuid) -> Option<Relay> {#[cfg(test)]mod tests {use super::*;+ use axum::response::IntoResponse;+ use axum::{routing::get, Router};+ use futures_util::{FutureExt, StreamExt};+ use std::time::Duration;++ #[tokio::test]+ async fn lagged_kb_stream_ends_when_its_only_document_or_global_alert_is_lost() {+ let (here, elsewhere) = (Uuid::now_v7(), Uuid::now_v7());+ for (kb_id, kind) in [(Some(here), "document"), (None, "alert")] {+ let (sender, _) = broadcast::channel(256);+ let route_sender = sender.clone();+ let app = Router::new().route(+ "/events",+ get(move || {+ let sender = route_sender.clone();+ async move {+ let receiver = sender.subscribe();+ sender+ .send(AppEvent {+ kb_id,+ kind,+ document_id: None,+ })+ .unwrap();+ // 首次 poll 前确定性地挤掉唯一有效通知,剩下的全部会被过滤。+ for _ in 0..257 {+ sender+ .send(AppEvent {+ kb_id: Some(elsewhere),+ kind: "document",+ document_id: None,+ })+ .unwrap();+ }+ kb_event_stream(receiver, here)+ }+ }),+ );+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap();+ let address = listener.local_addr().unwrap();+ let server = tokio::spawn(async move { axum::serve(listener, app).await.unwrap() });+ let client = reqwest::Client::builder().no_proxy().build().unwrap();+ let request = async {+ let response = client+ .get(format!("http://{address}/events"))+ .send()+ .await+ .unwrap();+ assert_eq!(response.status(), reqwest::StatusCode::OK);+ assert_eq!(response.headers()["content-type"], "text/event-stream");+ response.text().await.unwrap()+ };+ // 保留 sender,确保真实 HTTP 响应结束来自 Lagged,而不是通道关闭。+ let result = tokio::time::timeout(Duration::from_secs(2), request).await;+ server.abort();+ assert!(result+ .expect("a lagged KB SSE response must end")+ .is_empty());+ }+ }++ #[tokio::test]+ async fn kb_stream_keeps_filtering_and_alert_payloads_until_the_channel_closes() {+ let (here, elsewhere) = (Uuid::now_v7(), Uuid::now_v7());+ let (sender, receiver) = broadcast::channel(256);+ let response = kb_event_stream(receiver, here).into_response();+ assert_eq!(response.headers()["content-type"], "text/event-stream");+ let mut body = response.into_body().into_data_stream();+ // 初次连接不凭空发刷新事件,也不结束响应。+ assert!(body.next().now_or_never().is_none());+ sender+ .send(AppEvent {+ kb_id: Some(elsewhere),+ kind: "document",+ document_id: None,+ })+ .unwrap();+ assert!(body.next().now_or_never().is_none());++ let document = AppEvent {+ kb_id: Some(here),+ kind: "document",+ document_id: Some(Uuid::now_v7()),+ };+ sender.send(document.clone()).unwrap();+ let frame = body.next().await.unwrap().unwrap();+ assert_eq!(+ frame.as_ref(),+ format!(+ "event: document\ndata: {}\n\n",+ serde_json::to_string(&document).unwrap()+ )+ .as_bytes()+ );++ for kb_id in [None, Some(here), Some(elsewhere)] {+ sender+ .send(AppEvent {+ kb_id,+ kind: "alert",+ document_id: Some(Uuid::now_v7()),+ })+ .unwrap();+ let frame = body.next().await.unwrap().unwrap();+ assert_eq!(frame.as_ref(), b"event: alert\ndata: {}\n\n");+ }+ assert!(body.next().now_or_never().is_none());+ drop(sender);+ assert!(body.next().await.is_none());+ }++ #[tokio::test]+ async fn dropping_a_kb_response_releases_its_subscription() {+ let (sender, receiver) = broadcast::channel(256);+ let response = kb_event_stream(receiver, Uuid::now_v7()).into_response();+ assert_eq!(sender.receiver_count(), 1);+ drop(response);+ assert_eq!(sender.receiver_count(), 0);+ }#[test]fn a_kb_stream_carries_its_own_events_and_every_alert() {diff --git a/web/src/useEventReconnect.test.ts b/web/src/useEventReconnect.test.tsindex 33fbf9c8..2e37fe5c 100644--- a/web/src/useEventReconnect.test.ts+++ b/web/src/useEventReconnect.test.ts@@ -128,9 +128,12 @@ afterEach(() => {});describe("KB stream reconnect", () => {- it("refetches missed changes immediately for every stream key in the current KB", async () => {+ it("refetches every current KB stream key and alerts after reconnect without new business events", async () => {const client = createClient();const current = await Promise.all(STREAM_KEYS.map((head) => watch(client, [head, "kb-a", "detail"])));+ const badge = await watch(client, ["alerts", "unread"]);+ const list = await watch(client, ["alerts", "list", "", 0]);+ const recoveredQueries = [...current, badge, list];const otherKb = await watch(client, ["graph", "kb-b"]);const unrelated = await watch(client, ["health"]);const inactiveKey = ["documents", "kb-a", "unmounted"];@@ -138,20 +141,23 @@ describe("KB stream reconnect", () => {const { source } = mount(() => useKbEvents("kb-a"));expect(source.url).toBe("/api/v1/kbs/kb-a/events");source.emit("open");- for (const query of current) expect(query.fetch).toHaveBeenCalledTimes(1);+ for (const query of recoveredQueries) expect(query.fetch).toHaveBeenCalledTimes(1);source.emit("error");- for (const query of current) query.changeServer();+ for (const query of recoveredQueries) query.changeServer();// staleTime 到期只让缓存变旧,不会自动请求;断线期间完全没有业务事件。await vi.advanceTimersByTimeAsync(STREAM_STALE_MS + 1);- for (const query of current) expect(query.fetch).toHaveBeenCalledTimes(1);+ for (const query of recoveredQueries) expect(query.fetch).toHaveBeenCalledTimes(1);source.emit("open");// 重连不等普通事件的 300ms 合并窗口。- for (const query of current) expect(query.fetch).toHaveBeenCalledTimes(2);+ for (const query of recoveredQueries) expect(query.fetch).toHaveBeenCalledTimes(2);await vi.waitFor(() => {- for (const query of current) expect(query.observer.getCurrentResult().data).toBe("after disconnect");+ for (const query of recoveredQueries) expect(query.observer.getCurrentResult().data).toBe("after disconnect");});+ source.emit("open");+ await vi.advanceTimersByTimeAsync(SETTLE_MS);+ for (const query of recoveredQueries) expect(query.fetch).toHaveBeenCalledTimes(2);expect(client.getQueryState(inactiveKey)?.isInvalidated).toBe(true);expect(otherKb.fetch).toHaveBeenCalledTimes(1);expect(unrelated.fetch).toHaveBeenCalledTimes(1);@@ -173,17 +179,25 @@ describe("KB stream reconnect", () => {const client = createClient();const graph = await watch(client, ["graph", "kb-a"]);const documents = await watch(client, ["documents", "kb-a"]);+ const badge = await watch(client, ["alerts", "unread"]);+ const list = await watch(client, ["alerts", "list", "", 0]);const { source } = mount(() => useKbEvents("kb-a"));source.emit("open");source.emit("document");source.emit("graph");+ source.emit("alert");+ source.emit("alert");source.emit("error");source.emit("open");expect(graph.fetch).toHaveBeenCalledTimes(2);expect(documents.fetch).toHaveBeenCalledTimes(2);+ expect(badge.fetch).toHaveBeenCalledTimes(2);+ expect(list.fetch).toHaveBeenCalledTimes(2);await vi.advanceTimersByTimeAsync(SETTLE_MS);expect(graph.fetch).toHaveBeenCalledTimes(2);expect(documents.fetch).toHaveBeenCalledTimes(2);+ expect(badge.fetch).toHaveBeenCalledTimes(2);+ expect(list.fetch).toHaveBeenCalledTimes(2);expect(vi.getTimerCount()).toBe(0);});@@ -209,15 +223,18 @@ describe("KB stream reconnect", () => {const client = createClient();const oldKb = await watch(client, ["graph", "kb-a"]);const newKb = await watch(client, ["graph", "kb-b"]);+ const badge = await watch(client, ["alerts", "unread"]);+ const list = await watch(client, ["alerts", "list", "", 0]);const old = mount(() => useKbEvents("kb-a"));old.source.emit("open");old.source.emit("error");const lateOpen = [...old.source.listeners.get("open") ?? []];const lateGraph = [...old.source.listeners.get("graph") ?? []];+ const lateAlert = [...old.source.listeners.get("alert") ?? []];old.unmount();const next = mount(() => useKbEvents("kb-b"));next.source.emit("open");- for (const callback of [...lateOpen, ...lateGraph]) callback();+ for (const callback of [...lateOpen, ...lateGraph, ...lateAlert]) callback();old.source.emit("open");old.source.emit("document");await vi.advanceTimersByTimeAsync(SETTLE_MS);@@ -226,10 +243,14 @@ describe("KB stream reconnect", () => {expect(vi.getTimerCount()).toBe(0);expect(oldKb.fetch).toHaveBeenCalledTimes(1);expect(newKb.fetch).toHaveBeenCalledTimes(1);+ expect(badge.fetch).toHaveBeenCalledTimes(1);+ expect(list.fetch).toHaveBeenCalledTimes(1);next.source.emit("error");next.source.emit("open");expect(oldKb.fetch).toHaveBeenCalledTimes(1);expect(newKb.fetch).toHaveBeenCalledTimes(2);+ expect(badge.fetch).toHaveBeenCalledTimes(2);+ expect(list.fetch).toHaveBeenCalledTimes(2);});it("does not subscribe without a selected KB", () => {