Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(connection-gateway): fix connectivity bug (#9282)
teo25e31075da1b
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.jsonindex e455121bfb..4625f57ed9 100644--- a/.github/workspace-dep-closures.json+++ b/.github/workspace-dep-closures.json@@ -4255,8 +4255,10 @@"crates/connection_gateway_client","crates/connection_gateway_models","crates/cowlike",+ "crates/database_env_vars","crates/document_sub_type","crates/ensure_exists",+ "crates/entity_access","crates/filter_ast","crates/frecency","crates/genai_telemetry",@@ -4282,6 +4284,7 @@"crates/model_owner","crates/model_user","crates/models_bulk_upload",+ "crates/models_entity_access_management","crates/models_pagination","crates/models_permissions","crates/non_empty",diff --git a/Cargo.lock b/Cargo.lockindex e21ec02430..8855038919 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -4902,6 +4902,7 @@ dependencies = ["dashmap","doppler-config","ensure_exists",+ "entity_access","frecency","futures","http-body-util",diff --git a/services/connection_gateway/AGENTS.md b/services/connection_gateway/AGENTS.mdindex 7ff9dc868e..adbe845def 100644--- a/services/connection_gateway/AGENTS.md+++ b/services/connection_gateway/AGENTS.md@@ -15,19 +15,23 @@ the socket themselves.backed by DynamoDB (tracks which entity is on which instance).- `src/service/sender.rs` + `src/service/redis.rs`: cross-instance messagerelay (Redis pub/sub) so a send reaches a connection on another instance.+- `src/service/entity_grants.rs`: which entities a connection may subscribe+ to (`track_entity`, `stream_events`), checked through `entity_access`.- `scripts/stale_connections.rs` (`stale_connections` bin): scans DynamoDB forconnections past the 60s timeout; `README.md` has usage.## Scope- Owns: WebSocket connection tracking and message delivery to a connected- entity, across instances.+ entity, across instances, and authorizing which entities a connection may+ subscribe to.- Not here: the message/event payloads themselves, or who is authorized tosend to whom — callers decide that before calling in.## Boundaries-- Depends on: `connection_gateway_models`, `macro_authorization`.+- Depends on: `connection_gateway_models`, `macro_authorization`,+ `entity_access`.- Used by: nearly every other service, through `connection_gateway_client`(document_storage_service, document_cognition_service, email_service,calendar_service, agent_harness_service, mcp_service, scheduled_action,@@ -50,6 +54,11 @@ the socket themselves.- A send can target a connection on a different instance — it always goesthrough the Redis relay in `src/service/redis.rs`, never a direct in-processlookup; don't bypass it when adding a new send path.+- Every subscription frame is authorized: a `user` entity only for that+ user, any other type for any permission `EntityAccessService` grants on it.+ Types it has no check for (`static_file`, `skill`, `channel_message`) are+ refused, so a new entity type clients track needs an access check first.+ Grants are cached per connection and rechecked every 5 minutes.- Presence (`user_tracking_change`) counts an entity row as active for 60safter its last `open` or `ping`, compared across instances' clocks. A `ping`for a row that is gone re-creates it, so a lost `open` heals on the nextdiff --git a/services/connection_gateway/Cargo.toml b/services/connection_gateway/Cargo.tomlindex f4e6d6486e..a1e8d4a4f2 100644--- a/services/connection_gateway/Cargo.toml+++ b/services/connection_gateway/Cargo.toml@@ -40,6 +40,7 @@ connection_gateway_models = { path = "../../crates/connection_gateway_models" }cowlike = { path = "../../crates/cowlike" }dashmap = { workspace = true }ensure_exists = { path = "../../crates/ensure_exists", features = ["dynamodb"] }+entity_access = { path = "../../crates/entity_access", default-features = false, features = ["outbound"] }frecency = { path = "../../crates/frecency", features = ["inbound","outbound",diff --git a/services/connection_gateway/src/api/connection.rs b/services/connection_gateway/src/api/connection.rsindex 8b180879a4..43b72cfdae 100644--- a/services/connection_gateway/src/api/connection.rs+++ b/services/connection_gateway/src/api/connection.rs@@ -7,6 +7,7 @@ use crate::{message::{OutgoingMessage, record_span_error},tracking::EntityConnectionExt,},+ service::entity_grants::EntityGrants,};use anyhow::Result;use axum::{@@ -105,7 +106,12 @@ async fn handle_websocket_connection(crate::service::fanout::connected(&connection_context, ¯o_user_id).await;- let receiver_task = handle_websocket_stream(connection_context, stream, sender.clone()).fuse();+ let grants = EntityGrants::new(+ macro_user_id.clone(),+ user_context.organization_id.map(i64::from),+ );+ let receiver_task =+ handle_websocket_stream(connection_context, stream, sender.clone(), grants).fuse();tokio::select! {res = sender_task => {diff --git a/services/connection_gateway/src/api/connection/messages.rs b/services/connection_gateway/src/api/connection/messages.rsindex 506ab63d0e..bc90ac0847 100644--- a/services/connection_gateway/src/api/connection/messages.rs+++ b/services/connection_gateway/src/api/connection/messages.rs@@ -1,43 +1,68 @@use crate::{+ constants::GRANT_REVALIDATION_INTERVAL,model::{connection::ConnectionContext,message::OutgoingMessage,- tracking::{EntityConnectionExt, TrackingData},+ tracking::{EntityConnectionExt, TrackAction, TrackingData},websocket::ToWebsocketMessage,},- service::{fanout, stream_event::handle_stream_events, tracker},+ service::{+ entity_grants::{Admission, EntityGrants},+ fanout,+ stream_event::handle_stream_events,+ tracker,+ },};use anyhow::{Context, Result};use axum::extract::ws::{Message, WebSocket};use cowlike::CowLike;use futures::{StreamExt, stream::SplitStream};use macro_user_id::user_id::MacroUserIdStr;-use tokio::sync::mpsc::Sender;+use model_entity::Entity;+use tokio::{+ sync::mpsc::Sender,+ time::{Instant, MissedTickBehavior},+};pub async fn handle_websocket_stream(connection_context: ConnectionContext<'_>,mut stream: SplitStream<WebSocket>,sender: Sender<OutgoingMessage>,+ mut grants: EntityGrants,) -> Result<()> {- while let Some(msg) = stream.next().await {- match msg {- Ok(msg) => {- if let Err(e) = handle_message(connection_context, msg, &sender).await {- tracing::warn!(error=?e, "error handling message");+ let mut revalidation = tokio::time::interval_at(+ Instant::now() + GRANT_REVALIDATION_INTERVAL,+ GRANT_REVALIDATION_INTERVAL,+ );+ revalidation.set_missed_tick_behavior(MissedTickBehavior::Delay);++ loop {+ tokio::select! {+ msg = stream.next() => match msg {+ Some(Ok(msg)) => {+ if let Err(e) =+ handle_message(connection_context, msg, &sender, &mut grants).await+ {+ tracing::warn!(error=?e, "error handling message");+ }}+ // A read error means the client went away without a close frame+ // (connection reset, broken pipe, no closing handshake).+ Some(Err(err)) => {+ tracing::info!(+ error = ?err,+ connection_id = %connection_context.connection_id,+ user_id = %connection_context.user_context.user_id,+ "web-socket closed with error",+ );+ break;+ }+ None => break,+ },+ _ = revalidation.tick() => {+ revoke_lost_grants(connection_context, &sender, &mut grants).await;}- // A read error means the client went away without a close frame- // (connection reset, broken pipe, no closing handshake).- Err(err) => {- tracing::info!(- error = ?err,- connection_id = %connection_context.connection_id,- user_id = %connection_context.user_context.user_id,- "web-socket closed with error",- );- break;- }- };+ }}Ok(())@@ -49,6 +74,7 @@ pub async fn handle_message(connection_context: ConnectionContext<'_>,message: Message,sender: &Sender<OutgoingMessage>,+ grants: &mut EntityGrants,) -> Result<()> {// Every client frame is republished to the fanout channel, unparsed,// before any in-gateway handling. Backend services consume what they care@@ -98,30 +124,95 @@ pub async fn handle_message(let parsed_message = serde_json::from_str::<ToWebsocketMessage>(&text_message).context("unable to parse message")?;+ let access = connection_context.api_context.entity_access.as_ref();match parsed_message {ToWebsocketMessage::TrackEntityMessage(message) => {- let entity_id = message.extra.entity_id.to_string();-- tracker::track_entity(- sender,- connection_context,- TrackingData {- entity: message- .extra- .entity_type- .with_entity_str(&entity_id)- .with_connection_str(connection_context.connection_id)- .with_user_str(&connection_context.user_context.user_id),- action: message.action,- },- )- .await- .ok();+ let entity = message.extra;+ match grants.admit(access, &entity, message.action).await {+ Ok(Admission::Admitted) => {+ track(connection_context, sender, &entity, message.action).await;+ }+ Ok(Admission::Refused) => {+ log_refused(connection_context, &entity, message.action.into());+ }+ Ok(Admission::Revoked) => {+ log_refused(connection_context, &entity, message.action.into());+ track(connection_context, sender, &entity, TrackAction::Close).await;+ }+ Err(error) => log_check_failed(connection_context, &entity, &error),+ }}ToWebsocketMessage::StreamEvents(message) => {- handle_stream_events(connection_context, sender.clone(), message)+ match grants.permits(access, &message.entity).await {+ Ok(true) => {+ let entity = message.entity.clone();+ let listener =+ handle_stream_events(connection_context, sender.clone(), message);+ grants.hold_stream_listener(&entity, listener);+ }+ Ok(false) => log_refused(connection_context, &message.entity, "stream_events"),+ Err(error) => log_check_failed(connection_context, &message.entity, &error),+ }}};Ok(())}++async fn track(+ connection_context: ConnectionContext<'_>,+ sender: &Sender<OutgoingMessage>,+ entity: &Entity<'_>,+ action: TrackAction,+) {+ tracker::track_entity(+ sender,+ connection_context,+ TrackingData {+ entity: entity+ .copied()+ .with_connection_str(connection_context.connection_id)+ .with_user_str(&connection_context.user_context.user_id),+ action,+ },+ )+ .await+ .ok();+}++/// Tears down subscriptions to entities the user can no longer view.+async fn revoke_lost_grants(+ connection_context: ConnectionContext<'_>,+ sender: &Sender<OutgoingMessage>,+ grants: &mut EntityGrants,+) {+ let access = connection_context.api_context.entity_access.as_ref();+ for entity in grants.revalidate(access).await {+ log_refused(connection_context, &entity, "revalidation");+ track(connection_context, sender, &entity, TrackAction::Close).await;+ }+}++fn log_refused(connection_context: ConnectionContext<'_>, entity: &Entity<'_>, frame: &str) {+ tracing::warn!(+ connection_id = %connection_context.connection_id,+ entity_type = %entity.entity_type,+ entity_id = %entity.entity_id,+ frame,+ "refused an entity subscription the user cannot view"+ );+}++fn log_check_failed(+ connection_context: ConnectionContext<'_>,+ entity: &Entity<'_>,+ error: &entity_access::domain::models::AccessError,+) {+ tracing::error!(+ error = ?error,+ connection_id = %connection_context.connection_id,+ entity_type = %entity.entity_type,+ entity_id = %entity.entity_id,+ "could not check entity access; dropped the subscription frame"+ );+}diff --git a/services/connection_gateway/src/constants.rs b/services/connection_gateway/src/constants.rsindex dc4d1d9f97..be7b8111e5 100644--- a/services/connection_gateway/src/constants.rs+++ b/services/connection_gateway/src/constants.rs@@ -32,3 +32,7 @@ pub const DEFAULT_TIMEOUT_THRESHOLD: u64 = 60_000;/// Emit diagnostics while a websocket queue or write remains blocked this long.pub(crate) const SLOW_WEBSOCKET_OPERATION_THRESHOLD: Duration = Duration::from_secs(1);++/// How often a connection re-checks that its user can still view each entity+/// it is subscribed to.+pub(crate) const GRANT_REVALIDATION_INTERVAL: Duration = Duration::from_secs(300);diff --git a/services/connection_gateway/src/context.rs b/services/connection_gateway/src/context.rsindex 2b48abdd89..7da17b86ec 100644--- a/services/connection_gateway/src/context.rs+++ b/services/connection_gateway/src/context.rs@@ -1,5 +1,6 @@use crate::config::Config;use axum::extract::FromRef;+use entity_access::{domain::service::EntityAccessServiceImpl, outbound::PgAccessRepository};use frecency::{domain::services::EventIngestorImpl,inbound::polling_aggregator::FrecencyAggregatorWorkerHandle,@@ -15,9 +16,13 @@ use stream::domain::StreamManager;pub type AuthorizationService = MacroAuthorizationServiceImpl<MacroAuthJwtValidator>;+pub type EntityAccess = EntityAccessServiceImpl<PgAccessRepository>;+#[derive(Clone, FromRef)]pub struct ApiContext {pub connection_manager: crate::service::connection::ConnectionManager,+ /// Decides which entities a connection may subscribe to.+ pub entity_access: Arc<EntityAccess>,/// Boot-unique id of this gateway instance, used to tag fanout/// messages and name this instance's outbound fanout channel.pub fanout_gateway_id: Arc<str>,diff --git a/services/connection_gateway/src/main.rs b/services/connection_gateway/src/main.rsindex 2ef4f22f89..3e691249ec 100644--- a/services/connection_gateway/src/main.rs+++ b/services/connection_gateway/src/main.rs@@ -18,6 +18,7 @@ use axum::http::{};use config::Config;use constants::ORIGINS;+use entity_access::{domain::service::EntityAccessServiceImpl, outbound::PgAccessRepository};use frecency::{domain::services::{EventIngestorImpl, PullAggregatorImpl},inbound::polling_aggregator::FrecencyAggregatorWorkerHandle,@@ -117,6 +118,9 @@ async fn main() -> Result<()> {let fanout_gateway_id: Arc<str> = Arc::from(uuid::Uuid::new_v4().to_string());let context = context::ApiContext {connection_manager,+ entity_access: Arc::new(EntityAccessServiceImpl::new(PgAccessRepository::new(+ pgpool.clone(),+ ))),fanout_gateway_id: Arc::clone(&fanout_gateway_id),redis_client: Arc::clone(&redis_client),redis_connection,diff --git a/services/connection_gateway/src/service/entity_grants.rs b/services/connection_gateway/src/service/entity_grants.rsnew file mode 100644index 0000000000..3f7d25b095--- /dev/null+++ b/services/connection_gateway/src/service/entity_grants.rs@@ -0,0 +1,238 @@+//! Which entities a websocket connection may receive realtime events for.+//!+//! A connection subscribes to an entity by tracking it (`track_entity`) or by+//! asking for its stream events (`stream_events`). Services that send to an+//! entity, or read its trackers to pick recipients, deliver to every+//! subscriber, so a connection may only subscribe to entities its user can+//! view, and loses the subscription when that access goes away.++use std::collections::{HashMap, HashSet};++use entity_access::domain::{+ models::{AccessError, AnyEntityPermission},+ ports::{AccessRepository, EntityAccessService},+ service::EntityAccessServiceImpl,+};+use macro_user_id::user_id::MacroUserIdStr;+use model_entity::{Entity, EntityType};+use tokio::task::AbortHandle;++use crate::model::tracking::TrackAction;++#[cfg(test)]+mod test;++/// Answers whether a user can view an entity.+pub trait EntityViewAccess: Send + Sync {+ /// `Ok(false)` when the user cannot view the entity, including when it+ /// does not exist or its type has no access check; `Err` only when access+ /// could not be determined.+ fn can_view(+ &self,+ user_id: &MacroUserIdStr<'_>,+ organization_id: Option<i64>,+ entity: &Entity<'_>,+ ) -> impl Future<Output = Result<bool, AccessError>> + Send;+}++impl<R: AccessRepository> EntityViewAccess for EntityAccessServiceImpl<R> {+ // Any permission at all: the lowest access level is view, and channels and+ // teams answer with a role rather than an access level.+ async fn can_view(+ &self,+ user_id: &MacroUserIdStr<'_>,+ organization_id: Option<i64>,+ entity: &Entity<'_>,+ ) -> Result<bool, AccessError> {+ viewable(+ self.generate_entity_access_receipt::<AnyEntityPermission>(+ &user_id.0,+ organization_id,+ &entity.entity_id,+ entity.entity_type,+ )+ .await,+ )+ }+}++/// Reads an access check's result as a yes/no, keeping only the errors that+/// mean the answer is unknown.+fn viewable<T>(result: Result<T, AccessError>) -> Result<bool, AccessError> {+ match result {+ Ok(_) => Ok(true),+ Err(+ AccessError::Unauthorized+ | AccessError::UnauthorizedWithMessage(_)+ | AccessError::NotFound(_)+ | AccessError::BadRequest(_),+ ) => Ok(false),+ Err(error @ (AccessError::Unavailable(_) | AccessError::Internal(_))) => Err(error),+ }+}++/// What to do with a `track_entity` frame.+#[derive(Debug, Clone, Copy, PartialEq, Eq)]+pub enum Admission {+ /// Handle the frame as sent.+ Admitted,+ /// Drop the frame.+ Refused,+ /// Drop the frame and tear down the connection's existing subscription:+ /// it was granted earlier, and the user can no longer view the entity.+ Revoked,+}++type GrantKey = (EntityType, String);++fn key(entity: &Entity<'_>) -> GrantKey {+ (entity.entity_type, entity.entity_id.to_string())+}++/// The entities one connection is subscribed to, for the connection's user.+pub struct EntityGrants {+ user_id: MacroUserIdStr<'static>,+ organization_id: Option<i64>,+ /// Entities tracked with `track_entity`.+ granted: HashSet<GrantKey>,+ /// Listeners started by `stream_events`, stopped when access is lost.+ stream_listeners: HashMap<GrantKey, AbortHandle>,+}++impl EntityGrants {+ pub fn new(user_id: MacroUserIdStr<'static>, organization_id: Option<i64>) -> Self {+ Self {+ user_id,+ organization_id,+ granted: HashSet::new(),+ stream_listeners: HashMap::new(),+ }+ }++ /// Decides a `track_entity` frame and records the grant it creates or+ /// ends. An `Err` means access could not be checked; drop the frame.+ pub async fn admit(+ &mut self,+ access: &impl EntityViewAccess,+ entity: &Entity<'_>,+ action: TrackAction,+ ) -> Result<Admission, AccessError> {+ let key = key(entity);+ match action {+ // A close removes only this connection's row, but it also records+ // frecency and announces presence, so it needs a prior grant.+ TrackAction::Close => Ok(if self.granted.remove(&key) {+ Admission::Admitted+ } else {+ Admission::Refused+ }),+ // An open already checked this entity; its pings stay cheap.+ TrackAction::Ping if self.granted.contains(&key) => Ok(Admission::Admitted),+ // A ping with no grant re-creates the row, the same as an open.+ TrackAction::Open | TrackAction::Ping => {+ if self.can_view(access, entity).await? {+ self.granted.insert(key);+ Ok(Admission::Admitted)+ } else if self.granted.remove(&key) {+ Ok(Admission::Revoked)+ } else {+ Ok(Admission::Refused)+ }+ }+ }+ }++ /// Whether the user may receive `entity`'s stream events.+ pub async fn permits(+ &self,+ access: &impl EntityViewAccess,+ entity: &Entity<'_>,+ ) -> Result<bool, AccessError> {+ self.can_view(access, entity).await+ }++ /// Keeps the listener a permitted `stream_events` frame started, so it+ /// can be stopped when access is lost, replacing any earlier one.+ pub fn hold_stream_listener(&mut self, entity: &Entity<'_>, listener: AbortHandle) {+ if let Some(replaced) = self.stream_listeners.insert(key(entity), listener) {+ replaced.abort();+ }+ }++ /// Re-checks every grant and drops the ones the user can no longer view:+ /// stops their stream listeners, and returns the tracked ones so their+ /// subscriptions can be torn down. A grant whose check fails is kept until+ /// the next pass. Checks run one at a time to keep a connection's share of+ /// the database pool small.+ pub async fn revalidate(&mut self, access: &impl EntityViewAccess) -> Vec<Entity<'static>> {+ let held: HashSet<GrantKey> = self+ .granted+ .iter()+ .chain(self.stream_listeners.keys())+ .cloned()+ .collect();+ let mut revoked = Vec::new();+ for (entity_type, entity_id) in held {+ let entity = entity_type.with_entity_string(entity_id);+ let result = self.can_view(access, &entity).await;+ match result {+ Ok(true) => {}+ Ok(false) => {+ let key = key(&entity);+ if let Some(listener) = self.stream_listeners.remove(&key) {+ listener.abort();+ }+ if self.granted.remove(&key) {+ revoked.push(entity);+ }+ }+ Err(error) => tracing::warn!(+ error = ?error,+ entity_type = %entity.entity_type,+ entity_id = %entity.entity_id,+ "could not recheck an entity subscription; keeping it until the next pass"+ ),+ }+ }+ revoked+ }++ #[tracing::instrument(+ name = "connection_gateway.authorize_subscription",+ skip(self, access),+ fields(+ entity_type = %entity.entity_type,+ entity_id = %entity.entity_id,+ allowed = tracing::field::Empty,+ ),+ err+ )]+ async fn can_view(+ &self,+ access: &impl EntityViewAccess,+ entity: &Entity<'_>,+ ) -> Result<bool, AccessError> {+ let allowed = match entity.entity_type {+ // A user entity carries everything sent to that user, so only the+ // user themselves may subscribe to it.+ EntityType::User => MacroUserIdStr::parse_from_str(&entity.entity_id)+ .is_ok_and(|requested| requested.0.as_ref() == self.user_id.0.as_ref()),+ _ => {+ access+ .can_view(&self.user_id, self.organization_id, entity)+ .await?+ }+ };+ tracing::Span::current().record("allowed", allowed);+ Ok(allowed)+ }+}++impl Drop for EntityGrants {+ /// The connection is gone; its stream listeners have no one to serve.+ fn drop(&mut self) {+ for listener in self.stream_listeners.values() {+ listener.abort();+ }+ }+}diff --git a/services/connection_gateway/src/service/entity_grants/test.rs b/services/connection_gateway/src/service/entity_grants/test.rsnew file mode 100644index 0000000000..98433b185c--- /dev/null+++ b/services/connection_gateway/src/service/entity_grants/test.rs@@ -0,0 +1,428 @@+use std::{+ collections::HashSet,+ sync::{+ Mutex,+ atomic::{AtomicUsize, Ordering},+ },+};++use entity_access::domain::models::AccessError;+use macro_user_id::user_id::MacroUserIdStr;+use model_entity::{Entity, EntityType};++use super::{Admission, EntityGrants, EntityViewAccess, viewable};+use crate::model::tracking::TrackAction;++const VIEWER: &str = "macro|[email protected]";+const OTHER_USER: &str = "macro|[email protected]";+const SHARED_DOCUMENT: &str = "0198f7a2-0000-7000-8000-000000000001";+const PRIVATE_CHANNEL: &str = "0198f7a2-0000-7000-8000-000000000002";+const BROKEN_CHAT: &str = "0198f7a2-0000-7000-8000-000000000003";++/// Grants view access to a fixed set of entities and fails for `BROKEN_CHAT`.+#[derive(Default)]+struct FakeAccess {+ viewable: Mutex<HashSet<(EntityType, String)>>,+ lookups: AtomicUsize,+}++impl FakeAccess {+ fn with(entities: &[(EntityType, &str)]) -> Self {+ let access = Self::default();+ for (entity_type, entity_id) in entities {+ access.grant(*entity_type, entity_id);+ }+ access+ }++ fn grant(&self, entity_type: EntityType, entity_id: &str) {+ self.viewable+ .lock()+ .unwrap()+ .insert((entity_type, entity_id.to_string()));+ }++ fn revoke(&self, entity_type: EntityType, entity_id: &str) {+ self.viewable+ .lock()+ .unwrap()+ .remove(&(entity_type, entity_id.to_string()));+ }++ fn lookups(&self) -> usize {+ self.lookups.load(Ordering::SeqCst)+ }+}++impl EntityViewAccess for FakeAccess {+ async fn can_view(+ &self,+ user_id: &MacroUserIdStr<'_>,+ _organization_id: Option<i64>,+ entity: &Entity<'_>,+ ) -> Result<bool, AccessError> {+ self.lookups.fetch_add(1, Ordering::SeqCst);+ assert_eq!(user_id.0.as_ref(), VIEWER);+ if entity.entity_id == BROKEN_CHAT {+ return Err(AccessError::internal("database unavailable"));+ }+ Ok(self+ .viewable+ .lock()+ .unwrap()+ .contains(&(entity.entity_type, entity.entity_id.to_string())))+ }+}++fn grants() -> EntityGrants {+ EntityGrants::new(MacroUserIdStr::parse_from_str(VIEWER).unwrap(), None)+}++fn entity(entity_type: EntityType, entity_id: &str) -> Entity<'_> {+ entity_type.with_entity_str(entity_id)+}++#[tokio::test]+async fn only_the_connected_user_may_track_their_user_entity() {+ let access = FakeAccess::default();+ let mut grants = grants();++ assert_eq!(+ grants+ .admit(+ &access,+ &entity(EntityType::User, OTHER_USER),+ TrackAction::Open+ )+ .await+ .unwrap(),+ Admission::Refused+ );+ assert_eq!(+ grants+ .admit(+ &access,+ &entity(EntityType::User, OTHER_USER),+ TrackAction::Ping+ )+ .await+ .unwrap(),+ Admission::Refused+ );+ assert!(+ !grants+ .permits(&access, &entity(EntityType::User, OTHER_USER))+ .await+ .unwrap()+ );+ assert_eq!(+ grants+ .admit(+ &access,+ &entity(EntityType::User, "macro|[email protected]"),+ TrackAction::Open+ )+ .await+ .unwrap(),+ Admission::Admitted+ );+ assert_eq!(+ grants+ .admit(+ &access,+ &entity(EntityType::User, "not a user id"),+ TrackAction::Open+ )+ .await+ .unwrap(),+ Admission::Refused+ );+ assert_eq!(+ access.lookups(),+ 0,+ "user entities never reach entity access"+ );+}++#[tokio::test]+async fn open_is_admitted_only_for_viewable_entities() {+ let access = FakeAccess::with(&[(EntityType::Document, SHARED_DOCUMENT)]);+ let mut grants = grants();++ assert_eq!(+ grants+ .admit(+ &access,+ &entity(EntityType::Document, SHARED_DOCUMENT),+ TrackAction::Open+ )+ .await+ .unwrap(),+ Admission::Admitted+ );+ assert_eq!(+ grants+ .admit(+ &access,+ &entity(EntityType::Channel, PRIVATE_CHANNEL),+ TrackAction::Open+ )+ .await+ .unwrap(),+ Admission::Refused+ );+ // The id alone is not enough: the type is part of what was checked.+ assert_eq!(+ grants+ .admit(+ &access,+ &entity(EntityType::Chat, SHARED_DOCUMENT),+ TrackAction::Open+ )+ .await+ .unwrap(),+ Admission::Refused+ );+}++#[tokio::test]+async fn ping_without_a_grant_is_checked_like_an_open() {+ let access = FakeAccess::with(&[(EntityType::Document, SHARED_DOCUMENT)]);+ let mut grants = grants();++ assert_eq!(+ grants+ .admit(+ &access,+ &entity(EntityType::Channel, PRIVATE_CHANNEL),+ TrackAction::Ping+ )+ .await+ .unwrap(),+ Admission::Refused+ );+ assert_eq!(+ grants+ .admit(+ &access,+ &entity(EntityType::Document, SHARED_DOCUMENT),+ TrackAction::Ping+ )+ .await+ .unwrap(),+ Admission::Admitted+ );+ assert_eq!(access.lookups(), 2);++ // Granted now, so later pings skip the lookup.+ for _ in 0..3 {+ assert_eq!(+ grants+ .admit(+ &access,+ &entity(EntityType::Document, SHARED_DOCUMENT),+ TrackAction::Ping+ )+ .await+ .unwrap(),+ Admission::Admitted+ );+ }+ assert_eq!(access.lookups(), 2);+}++#[tokio::test]+async fn close_needs_a_grant_and_ends_it() {+ let access = FakeAccess::with(&[(EntityType::Document, SHARED_DOCUMENT)]);+ let mut grants = grants();+ let document = entity(EntityType::Document, SHARED_DOCUMENT);++ assert_eq!(+ grants+ .admit(+ &access,+ &entity(EntityType::Channel, PRIVATE_CHANNEL),+ TrackAction::Close+ )+ .await+ .unwrap(),+ Admission::Refused+ );+ assert_eq!(+ grants+ .admit(&access, &document, TrackAction::Close)+ .await+ .unwrap(),+ Admission::Refused,+ "never opened on this connection"+ );++ grants+ .admit(&access, &document, TrackAction::Open)+ .await+ .unwrap();+ assert_eq!(+ grants+ .admit(&access, &document, TrackAction::Close)+ .await+ .unwrap(),+ Admission::Admitted+ );++ let lookups = access.lookups();+ grants+ .admit(&access, &document, TrackAction::Ping)+ .await+ .unwrap();+ assert_eq!(access.lookups(), lookups + 1, "the close dropped the grant");+}++#[tokio::test]+async fn reopening_an_entity_whose_access_was_removed_revokes_it() {+ let access = FakeAccess::with(&[(EntityType::Document, SHARED_DOCUMENT)]);+ let mut grants = grants();+ let document = entity(EntityType::Document, SHARED_DOCUMENT);++ grants+ .admit(&access, &document, TrackAction::Open)+ .await+ .unwrap();+ access.revoke(EntityType::Document, SHARED_DOCUMENT);++ assert_eq!(+ grants+ .admit(&access, &document, TrackAction::Open)+ .await+ .unwrap(),+ Admission::Revoked+ );+ assert_eq!(+ grants+ .admit(&access, &document, TrackAction::Close)+ .await+ .unwrap(),+ Admission::Refused+ );+}++#[tokio::test]+async fn a_failed_check_is_an_error_and_grants_nothing() {+ let access = FakeAccess::default();+ let mut grants = grants();+ let chat = entity(EntityType::Chat, BROKEN_CHAT);++ assert!(+ grants+ .admit(&access, &chat, TrackAction::Open)+ .await+ .is_err()+ );+ assert!(grants.permits(&access, &chat).await.is_err());+ assert_eq!(+ grants+ .admit(&access, &chat, TrackAction::Close)+ .await+ .unwrap(),+ Admission::Refused+ );+}++#[tokio::test]+async fn revalidation_revokes_lost_access_and_keeps_the_rest() {+ let access = FakeAccess::with(&[+ (EntityType::Document, SHARED_DOCUMENT),+ (EntityType::Channel, PRIVATE_CHANNEL),+ (EntityType::Chat, BROKEN_CHAT),+ ]);+ let mut grants = grants();+ for granted in [+ entity(EntityType::Document, SHARED_DOCUMENT),+ entity(EntityType::Channel, PRIVATE_CHANNEL),+ ] {+ grants+ .admit(&access, &granted, TrackAction::Open)+ .await+ .unwrap();+ }+ // A grant whose recheck will fail.+ grants+ .granted+ .insert((EntityType::Chat, BROKEN_CHAT.to_string()));++ access.revoke(EntityType::Channel, PRIVATE_CHANNEL);+ let revoked = grants.revalidate(&access).await;++ assert_eq!(revoked.len(), 1);+ assert_eq!(revoked[0].entity_type, EntityType::Channel);+ assert_eq!(revoked[0].entity_id, PRIVATE_CHANNEL);+ assert_eq!(+ grants.granted,+ HashSet::from([+ (EntityType::Document, SHARED_DOCUMENT.to_string()),+ (EntityType::Chat, BROKEN_CHAT.to_string()),+ ])+ );++ access.grant(EntityType::Channel, PRIVATE_CHANNEL);+ assert!(grants.revalidate(&access).await.is_empty());+}++#[test]+fn access_errors_that_mean_no_access_read_as_not_viewable() {+ assert!(viewable(Ok::<_, AccessError>(())).unwrap());+ for denied in [+ AccessError::Unauthorized,+ AccessError::UnauthorizedWithMessage("no"),+ AccessError::NotFound("document"),+ AccessError::BadRequest("Unsupported entity type"),+ ] {+ assert!(!viewable(Err::<(), _>(denied)).unwrap());+ }+ assert!(viewable(Err::<(), _>(AccessError::internal("boom"))).is_err());+}++#[tokio::test]+async fn revalidation_stops_stream_listeners_whose_access_was_lost() {+ let access = FakeAccess::with(&[+ (EntityType::Document, SHARED_DOCUMENT),+ (EntityType::Channel, PRIVATE_CHANNEL),+ ]);+ let mut grants = grants();+ let kept = tokio::spawn(std::future::pending::<()>());+ let lost = tokio::spawn(std::future::pending::<()>());+ grants.hold_stream_listener(+ &entity(EntityType::Document, SHARED_DOCUMENT),+ kept.abort_handle(),+ );+ grants.hold_stream_listener(+ &entity(EntityType::Channel, PRIVATE_CHANNEL),+ lost.abort_handle(),+ );++ access.revoke(EntityType::Channel, PRIVATE_CHANNEL);+ let revoked = grants.revalidate(&access).await;++ assert!(revoked.is_empty(), "stream events are not tracked rows");+ assert!(lost.await.unwrap_err().is_cancelled());+ assert!(!kept.is_finished());++ drop(grants);+ assert!(+ kept.await.unwrap_err().is_cancelled(),+ "a closed connection stops its listeners"+ );+}++#[tokio::test]+async fn a_repeated_stream_events_frame_replaces_the_listener() {+ let mut grants = grants();+ let document = entity(EntityType::Document, SHARED_DOCUMENT);+ let first = tokio::spawn(std::future::pending::<()>());+ let second = tokio::spawn(std::future::pending::<()>());++ grants.hold_stream_listener(&document, first.abort_handle());+ grants.hold_stream_listener(&document, second.abort_handle());++ assert!(first.await.unwrap_err().is_cancelled());+ assert!(!second.is_finished());+}diff --git a/services/connection_gateway/src/service/mod.rs b/services/connection_gateway/src/service/mod.rsindex 02eb553f0c..50bbfb83a7 100644--- a/services/connection_gateway/src/service/mod.rs+++ b/services/connection_gateway/src/service/mod.rs@@ -1,5 +1,6 @@pub mod connection;pub mod dynamodb;+pub mod entity_grants;pub mod fanout;pub mod redis;pub mod sender;diff --git a/services/connection_gateway/src/service/stream_event.rs b/services/connection_gateway/src/service/stream_event.rsindex 803e2692d0..9acbe8b417 100644--- a/services/connection_gateway/src/service/stream_event.rs+++ b/services/connection_gateway/src/service/stream_event.rs@@ -4,7 +4,7 @@//! while StreamItems represent stream data itselfuse stream::domain::{StreamEvent, StreamId};-use tokio::sync::mpsc::Sender;+use tokio::{sync::mpsc::Sender, task::AbortHandle};use crate::model::{connection::ConnectionContext, message::OutgoingMessage, websocket::StreamEvents,@@ -15,10 +15,10 @@ pub fn handle_stream_events(context: ConnectionContext<'_>,sender: Sender<OutgoingMessage>,incoming_message: StreamEvents,-) {+) -> AbortHandle {let repo = context.api_context.stream_manager.repo();- tokio::spawn(async move {+ let listener = tokio::spawn(async move {// if any prexisting streams exist emit open entityif let Ok(streams) = repo.active_streams(&incoming_message.entity.entity_id)@@ -69,4 +69,5 @@ pub fn handle_stream_events(}}});+ listener.abort_handle()}