Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(connection-gateway): fix connectivity bug (#9282)

teo25e31075da1b
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.json
index e455121bfb..4625f57ed9 100644
--- a/.github/workspace-dep-closures.json
+++ b/.github/workspace-dep-closures.json
@@ -4255,8 +4255,10 @@
"crates/connection_gateway_client",
"crates/connection_gateway_models",
"crates/cowlike",
+ "crates/database_env_vars",
"crates/document_sub_type",
"crates/ensure_exists",
+ "crates/entity_access",
"crates/filter_ast",
"crates/frecency",
"crates/genai_telemetry",
@@ -4282,6 +4284,7 @@
"crates/model_owner",
"crates/model_user",
"crates/models_bulk_upload",
+ "crates/models_entity_access_management",
"crates/models_pagination",
"crates/models_permissions",
"crates/non_empty",
diff --git a/Cargo.lock b/Cargo.lock
index e21ec02430..8855038919 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -4902,6 +4902,7 @@ dependencies = [
"dashmap",
"doppler-config",
"ensure_exists",
+ "entity_access",
"frecency",
"futures",
"http-body-util",
diff --git a/services/connection_gateway/AGENTS.md b/services/connection_gateway/AGENTS.md
index 7ff9dc868e..adbe845def 100644
--- a/services/connection_gateway/AGENTS.md
+++ b/services/connection_gateway/AGENTS.md
@@ -15,19 +15,23 @@ the socket themselves.
backed by DynamoDB (tracks which entity is on which instance).
- `src/service/sender.rs` + `src/service/redis.rs`: cross-instance message
relay (Redis pub/sub) so a send reaches a connection on another instance.
+- `src/service/entity_grants.rs`: which entities a connection may subscribe
+ to (`track_entity`, `stream_events`), checked through `entity_access`.
- `scripts/stale_connections.rs` (`stale_connections` bin): scans DynamoDB for
connections past the 60s timeout; `README.md` has usage.
## Scope
- Owns: WebSocket connection tracking and message delivery to a connected
- entity, across instances.
+ entity, across instances, and authorizing which entities a connection may
+ subscribe to.
- Not here: the message/event payloads themselves, or who is authorized to
send to whom — callers decide that before calling in.
## Boundaries
-- Depends on: `connection_gateway_models`, `macro_authorization`.
+- Depends on: `connection_gateway_models`, `macro_authorization`,
+ `entity_access`.
- Used by: nearly every other service, through `connection_gateway_client`
(document_storage_service, document_cognition_service, email_service,
calendar_service, agent_harness_service, mcp_service, scheduled_action,
@@ -50,6 +54,11 @@ the socket themselves.
- A send can target a connection on a different instance — it always goes
through the Redis relay in `src/service/redis.rs`, never a direct in-process
lookup; don't bypass it when adding a new send path.
+- Every subscription frame is authorized: a `user` entity only for that
+ user, any other type for any permission `EntityAccessService` grants on it.
+ Types it has no check for (`static_file`, `skill`, `channel_message`) are
+ refused, so a new entity type clients track needs an access check first.
+ Grants are cached per connection and rechecked every 5 minutes.
- Presence (`user_tracking_change`) counts an entity row as active for 60s
after its last `open` or `ping`, compared across instances' clocks. A `ping`
for a row that is gone re-creates it, so a lost `open` heals on the next
diff --git a/services/connection_gateway/Cargo.toml b/services/connection_gateway/Cargo.toml
index f4e6d6486e..a1e8d4a4f2 100644
--- a/services/connection_gateway/Cargo.toml
+++ b/services/connection_gateway/Cargo.toml
@@ -40,6 +40,7 @@ connection_gateway_models = { path = "../../crates/connection_gateway_models" }
cowlike = { path = "../../crates/cowlike" }
dashmap = { workspace = true }
ensure_exists = { path = "../../crates/ensure_exists", features = ["dynamodb"] }
+entity_access = { path = "../../crates/entity_access", default-features = false, features = ["outbound"] }
frecency = { path = "../../crates/frecency", features = [
"inbound",
"outbound",
diff --git a/services/connection_gateway/src/api/connection.rs b/services/connection_gateway/src/api/connection.rs
index 8b180879a4..43b72cfdae 100644
--- a/services/connection_gateway/src/api/connection.rs
+++ b/services/connection_gateway/src/api/connection.rs
@@ -7,6 +7,7 @@ use crate::{
message::{OutgoingMessage, record_span_error},
tracking::EntityConnectionExt,
},
+ service::entity_grants::EntityGrants,
};
use anyhow::Result;
use axum::{
@@ -105,7 +106,12 @@ async fn handle_websocket_connection(
crate::service::fanout::connected(&connection_context, &macro_user_id).await;
- let receiver_task = handle_websocket_stream(connection_context, stream, sender.clone()).fuse();
+ let grants = EntityGrants::new(
+ macro_user_id.clone(),
+ user_context.organization_id.map(i64::from),
+ );
+ let receiver_task =
+ handle_websocket_stream(connection_context, stream, sender.clone(), grants).fuse();
tokio::select! {
res = sender_task => {
diff --git a/services/connection_gateway/src/api/connection/messages.rs b/services/connection_gateway/src/api/connection/messages.rs
index 506ab63d0e..bc90ac0847 100644
--- a/services/connection_gateway/src/api/connection/messages.rs
+++ b/services/connection_gateway/src/api/connection/messages.rs
@@ -1,43 +1,68 @@
use crate::{
+ constants::GRANT_REVALIDATION_INTERVAL,
model::{
connection::ConnectionContext,
message::OutgoingMessage,
- tracking::{EntityConnectionExt, TrackingData},
+ tracking::{EntityConnectionExt, TrackAction, TrackingData},
websocket::ToWebsocketMessage,
},
- service::{fanout, stream_event::handle_stream_events, tracker},
+ service::{
+ entity_grants::{Admission, EntityGrants},
+ fanout,
+ stream_event::handle_stream_events,
+ tracker,
+ },
};
use anyhow::{Context, Result};
use axum::extract::ws::{Message, WebSocket};
use cowlike::CowLike;
use futures::{StreamExt, stream::SplitStream};
use macro_user_id::user_id::MacroUserIdStr;
-use tokio::sync::mpsc::Sender;
+use model_entity::Entity;
+use tokio::{
+ sync::mpsc::Sender,
+ time::{Instant, MissedTickBehavior},
+};
pub async fn handle_websocket_stream(
connection_context: ConnectionContext<'_>,
mut stream: SplitStream<WebSocket>,
sender: Sender<OutgoingMessage>,
+ mut grants: EntityGrants,
) -> Result<()> {
- while let Some(msg) = stream.next().await {
- match msg {
- Ok(msg) => {
- if let Err(e) = handle_message(connection_context, msg, &sender).await {
- tracing::warn!(error=?e, "error handling message");
+ let mut revalidation = tokio::time::interval_at(
+ Instant::now() + GRANT_REVALIDATION_INTERVAL,
+ GRANT_REVALIDATION_INTERVAL,
+ );
+ revalidation.set_missed_tick_behavior(MissedTickBehavior::Delay);
+
+ loop {
+ tokio::select! {
+ msg = stream.next() => match msg {
+ Some(Ok(msg)) => {
+ if let Err(e) =
+ handle_message(connection_context, msg, &sender, &mut grants).await
+ {
+ tracing::warn!(error=?e, "error handling message");
+ }
}
+ // A read error means the client went away without a close frame
+ // (connection reset, broken pipe, no closing handshake).
+ Some(Err(err)) => {
+ tracing::info!(
+ error = ?err,
+ connection_id = %connection_context.connection_id,
+ user_id = %connection_context.user_context.user_id,
+ "web-socket closed with error",
+ );
+ break;
+ }
+ None => break,
+ },
+ _ = revalidation.tick() => {
+ revoke_lost_grants(connection_context, &sender, &mut grants).await;
}
- // A read error means the client went away without a close frame
- // (connection reset, broken pipe, no closing handshake).
- Err(err) => {
- tracing::info!(
- error = ?err,
- connection_id = %connection_context.connection_id,
- user_id = %connection_context.user_context.user_id,
- "web-socket closed with error",
- );
- break;
- }
- };
+ }
}
Ok(())
@@ -49,6 +74,7 @@ pub async fn handle_message(
connection_context: ConnectionContext<'_>,
message: Message,
sender: &Sender<OutgoingMessage>,
+ grants: &mut EntityGrants,
) -> Result<()> {
// Every client frame is republished to the fanout channel, unparsed,
// before any in-gateway handling. Backend services consume what they care
@@ -98,30 +124,95 @@ pub async fn handle_message(
let parsed_message = serde_json::from_str::<ToWebsocketMessage>(&text_message)
.context("unable to parse message")?;
+ let access = connection_context.api_context.entity_access.as_ref();
match parsed_message {
ToWebsocketMessage::TrackEntityMessage(message) => {
- let entity_id = message.extra.entity_id.to_string();
-
- tracker::track_entity(
- sender,
- connection_context,
- TrackingData {
- entity: message
- .extra
- .entity_type
- .with_entity_str(&entity_id)
- .with_connection_str(connection_context.connection_id)
- .with_user_str(&connection_context.user_context.user_id),
- action: message.action,
- },
- )
- .await
- .ok();
+ let entity = message.extra;
+ match grants.admit(access, &entity, message.action).await {
+ Ok(Admission::Admitted) => {
+ track(connection_context, sender, &entity, message.action).await;
+ }
+ Ok(Admission::Refused) => {
+ log_refused(connection_context, &entity, message.action.into());
+ }
+ Ok(Admission::Revoked) => {
+ log_refused(connection_context, &entity, message.action.into());
+ track(connection_context, sender, &entity, TrackAction::Close).await;
+ }
+ Err(error) => log_check_failed(connection_context, &entity, &error),
+ }
}
ToWebsocketMessage::StreamEvents(message) => {
- handle_stream_events(connection_context, sender.clone(), message)
+ match grants.permits(access, &message.entity).await {
+ Ok(true) => {
+ let entity = message.entity.clone();
+ let listener =
+ handle_stream_events(connection_context, sender.clone(), message);
+ grants.hold_stream_listener(&entity, listener);
+ }
+ Ok(false) => log_refused(connection_context, &message.entity, "stream_events"),
+ Err(error) => log_check_failed(connection_context, &message.entity, &error),
+ }
}
};
Ok(())
}
+
+async fn track(
+ connection_context: ConnectionContext<'_>,
+ sender: &Sender<OutgoingMessage>,
+ entity: &Entity<'_>,
+ action: TrackAction,
+) {
+ tracker::track_entity(
+ sender,
+ connection_context,
+ TrackingData {
+ entity: entity
+ .copied()
+ .with_connection_str(connection_context.connection_id)
+ .with_user_str(&connection_context.user_context.user_id),
+ action,
+ },
+ )
+ .await
+ .ok();
+}
+
+/// Tears down subscriptions to entities the user can no longer view.
+async fn revoke_lost_grants(
+ connection_context: ConnectionContext<'_>,
+ sender: &Sender<OutgoingMessage>,
+ grants: &mut EntityGrants,
+) {
+ let access = connection_context.api_context.entity_access.as_ref();
+ for entity in grants.revalidate(access).await {
+ log_refused(connection_context, &entity, "revalidation");
+ track(connection_context, sender, &entity, TrackAction::Close).await;
+ }
+}
+
+fn log_refused(connection_context: ConnectionContext<'_>, entity: &Entity<'_>, frame: &str) {
+ tracing::warn!(
+ connection_id = %connection_context.connection_id,
+ entity_type = %entity.entity_type,
+ entity_id = %entity.entity_id,
+ frame,
+ "refused an entity subscription the user cannot view"
+ );
+}
+
+fn log_check_failed(
+ connection_context: ConnectionContext<'_>,
+ entity: &Entity<'_>,
+ error: &entity_access::domain::models::AccessError,
+) {
+ tracing::error!(
+ error = ?error,
+ connection_id = %connection_context.connection_id,
+ entity_type = %entity.entity_type,
+ entity_id = %entity.entity_id,
+ "could not check entity access; dropped the subscription frame"
+ );
+}
diff --git a/services/connection_gateway/src/constants.rs b/services/connection_gateway/src/constants.rs
index dc4d1d9f97..be7b8111e5 100644
--- a/services/connection_gateway/src/constants.rs
+++ b/services/connection_gateway/src/constants.rs
@@ -32,3 +32,7 @@ pub const DEFAULT_TIMEOUT_THRESHOLD: u64 = 60_000;
/// Emit diagnostics while a websocket queue or write remains blocked this long.
pub(crate) const SLOW_WEBSOCKET_OPERATION_THRESHOLD: Duration = Duration::from_secs(1);
+
+/// How often a connection re-checks that its user can still view each entity
+/// it is subscribed to.
+pub(crate) const GRANT_REVALIDATION_INTERVAL: Duration = Duration::from_secs(300);
diff --git a/services/connection_gateway/src/context.rs b/services/connection_gateway/src/context.rs
index 2b48abdd89..7da17b86ec 100644
--- a/services/connection_gateway/src/context.rs
+++ b/services/connection_gateway/src/context.rs
@@ -1,5 +1,6 @@
use crate::config::Config;
use axum::extract::FromRef;
+use entity_access::{domain::service::EntityAccessServiceImpl, outbound::PgAccessRepository};
use frecency::{
domain::services::EventIngestorImpl,
inbound::polling_aggregator::FrecencyAggregatorWorkerHandle,
@@ -15,9 +16,13 @@ use stream::domain::StreamManager;
pub type AuthorizationService = MacroAuthorizationServiceImpl<MacroAuthJwtValidator>;
+pub type EntityAccess = EntityAccessServiceImpl<PgAccessRepository>;
+
#[derive(Clone, FromRef)]
pub struct ApiContext {
pub connection_manager: crate::service::connection::ConnectionManager,
+ /// Decides which entities a connection may subscribe to.
+ pub entity_access: Arc<EntityAccess>,
/// Boot-unique id of this gateway instance, used to tag fanout
/// messages and name this instance's outbound fanout channel.
pub fanout_gateway_id: Arc<str>,
diff --git a/services/connection_gateway/src/main.rs b/services/connection_gateway/src/main.rs
index 2ef4f22f89..3e691249ec 100644
--- a/services/connection_gateway/src/main.rs
+++ b/services/connection_gateway/src/main.rs
@@ -18,6 +18,7 @@ use axum::http::{
};
use config::Config;
use constants::ORIGINS;
+use entity_access::{domain::service::EntityAccessServiceImpl, outbound::PgAccessRepository};
use frecency::{
domain::services::{EventIngestorImpl, PullAggregatorImpl},
inbound::polling_aggregator::FrecencyAggregatorWorkerHandle,
@@ -117,6 +118,9 @@ async fn main() -> Result<()> {
let fanout_gateway_id: Arc<str> = Arc::from(uuid::Uuid::new_v4().to_string());
let context = context::ApiContext {
connection_manager,
+ entity_access: Arc::new(EntityAccessServiceImpl::new(PgAccessRepository::new(
+ pgpool.clone(),
+ ))),
fanout_gateway_id: Arc::clone(&fanout_gateway_id),
redis_client: Arc::clone(&redis_client),
redis_connection,
diff --git a/services/connection_gateway/src/service/entity_grants.rs b/services/connection_gateway/src/service/entity_grants.rs
new file mode 100644
index 0000000000..3f7d25b095
--- /dev/null
+++ b/services/connection_gateway/src/service/entity_grants.rs
@@ -0,0 +1,238 @@
+//! Which entities a websocket connection may receive realtime events for.
+//!
+//! A connection subscribes to an entity by tracking it (`track_entity`) or by
+//! asking for its stream events (`stream_events`). Services that send to an
+//! entity, or read its trackers to pick recipients, deliver to every
+//! subscriber, so a connection may only subscribe to entities its user can
+//! view, and loses the subscription when that access goes away.
+
+use std::collections::{HashMap, HashSet};
+
+use entity_access::domain::{
+ models::{AccessError, AnyEntityPermission},
+ ports::{AccessRepository, EntityAccessService},
+ service::EntityAccessServiceImpl,
+};
+use macro_user_id::user_id::MacroUserIdStr;
+use model_entity::{Entity, EntityType};
+use tokio::task::AbortHandle;
+
+use crate::model::tracking::TrackAction;
+
+#[cfg(test)]
+mod test;
+
+/// Answers whether a user can view an entity.
+pub trait EntityViewAccess: Send + Sync {
+ /// `Ok(false)` when the user cannot view the entity, including when it
+ /// does not exist or its type has no access check; `Err` only when access
+ /// could not be determined.
+ fn can_view(
+ &self,
+ user_id: &MacroUserIdStr<'_>,
+ organization_id: Option<i64>,
+ entity: &Entity<'_>,
+ ) -> impl Future<Output = Result<bool, AccessError>> + Send;
+}
+
+impl<R: AccessRepository> EntityViewAccess for EntityAccessServiceImpl<R> {
+ // Any permission at all: the lowest access level is view, and channels and
+ // teams answer with a role rather than an access level.
+ async fn can_view(
+ &self,
+ user_id: &MacroUserIdStr<'_>,
+ organization_id: Option<i64>,
+ entity: &Entity<'_>,
+ ) -> Result<bool, AccessError> {
+ viewable(
+ self.generate_entity_access_receipt::<AnyEntityPermission>(
+ &user_id.0,
+ organization_id,
+ &entity.entity_id,
+ entity.entity_type,
+ )
+ .await,
+ )
+ }
+}
+
+/// Reads an access check's result as a yes/no, keeping only the errors that
+/// mean the answer is unknown.
+fn viewable<T>(result: Result<T, AccessError>) -> Result<bool, AccessError> {
+ match result {
+ Ok(_) => Ok(true),
+ Err(
+ AccessError::Unauthorized
+ | AccessError::UnauthorizedWithMessage(_)
+ | AccessError::NotFound(_)
+ | AccessError::BadRequest(_),
+ ) => Ok(false),
+ Err(error @ (AccessError::Unavailable(_) | AccessError::Internal(_))) => Err(error),
+ }
+}
+
+/// What to do with a `track_entity` frame.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Admission {
+ /// Handle the frame as sent.
+ Admitted,
+ /// Drop the frame.
+ Refused,
+ /// Drop the frame and tear down the connection's existing subscription:
+ /// it was granted earlier, and the user can no longer view the entity.
+ Revoked,
+}
+
+type GrantKey = (EntityType, String);
+
+fn key(entity: &Entity<'_>) -> GrantKey {
+ (entity.entity_type, entity.entity_id.to_string())
+}
+
+/// The entities one connection is subscribed to, for the connection's user.
+pub struct EntityGrants {
+ user_id: MacroUserIdStr<'static>,
+ organization_id: Option<i64>,
+ /// Entities tracked with `track_entity`.
+ granted: HashSet<GrantKey>,
+ /// Listeners started by `stream_events`, stopped when access is lost.
+ stream_listeners: HashMap<GrantKey, AbortHandle>,
+}
+
+impl EntityGrants {
+ pub fn new(user_id: MacroUserIdStr<'static>, organization_id: Option<i64>) -> Self {
+ Self {
+ user_id,
+ organization_id,
+ granted: HashSet::new(),
+ stream_listeners: HashMap::new(),
+ }
+ }
+
+ /// Decides a `track_entity` frame and records the grant it creates or
+ /// ends. An `Err` means access could not be checked; drop the frame.
+ pub async fn admit(
+ &mut self,
+ access: &impl EntityViewAccess,
+ entity: &Entity<'_>,
+ action: TrackAction,
+ ) -> Result<Admission, AccessError> {
+ let key = key(entity);
+ match action {
+ // A close removes only this connection's row, but it also records
+ // frecency and announces presence, so it needs a prior grant.
+ TrackAction::Close => Ok(if self.granted.remove(&key) {
+ Admission::Admitted
+ } else {
+ Admission::Refused
+ }),
+ // An open already checked this entity; its pings stay cheap.
+ TrackAction::Ping if self.granted.contains(&key) => Ok(Admission::Admitted),
+ // A ping with no grant re-creates the row, the same as an open.
+ TrackAction::Open | TrackAction::Ping => {
+ if self.can_view(access, entity).await? {
+ self.granted.insert(key);
+ Ok(Admission::Admitted)
+ } else if self.granted.remove(&key) {
+ Ok(Admission::Revoked)
+ } else {
+ Ok(Admission::Refused)
+ }
+ }
+ }
+ }
+
+ /// Whether the user may receive `entity`'s stream events.
+ pub async fn permits(
+ &self,
+ access: &impl EntityViewAccess,
+ entity: &Entity<'_>,
+ ) -> Result<bool, AccessError> {
+ self.can_view(access, entity).await
+ }
+
+ /// Keeps the listener a permitted `stream_events` frame started, so it
+ /// can be stopped when access is lost, replacing any earlier one.
+ pub fn hold_stream_listener(&mut self, entity: &Entity<'_>, listener: AbortHandle) {
+ if let Some(replaced) = self.stream_listeners.insert(key(entity), listener) {
+ replaced.abort();
+ }
+ }
+
+ /// Re-checks every grant and drops the ones the user can no longer view:
+ /// stops their stream listeners, and returns the tracked ones so their
+ /// subscriptions can be torn down. A grant whose check fails is kept until
+ /// the next pass. Checks run one at a time to keep a connection's share of
+ /// the database pool small.
+ pub async fn revalidate(&mut self, access: &impl EntityViewAccess) -> Vec<Entity<'static>> {
+ let held: HashSet<GrantKey> = self
+ .granted
+ .iter()
+ .chain(self.stream_listeners.keys())
+ .cloned()
+ .collect();
+ let mut revoked = Vec::new();
+ for (entity_type, entity_id) in held {
+ let entity = entity_type.with_entity_string(entity_id);
+ let result = self.can_view(access, &entity).await;
+ match result {
+ Ok(true) => {}
+ Ok(false) => {
+ let key = key(&entity);
+ if let Some(listener) = self.stream_listeners.remove(&key) {
+ listener.abort();
+ }
+ if self.granted.remove(&key) {
+ revoked.push(entity);
+ }
+ }
+ Err(error) => tracing::warn!(
+ error = ?error,
+ entity_type = %entity.entity_type,
+ entity_id = %entity.entity_id,
+ "could not recheck an entity subscription; keeping it until the next pass"
+ ),
+ }
+ }
+ revoked
+ }
+
+ #[tracing::instrument(
+ name = "connection_gateway.authorize_subscription",
+ skip(self, access),
+ fields(
+ entity_type = %entity.entity_type,
+ entity_id = %entity.entity_id,
+ allowed = tracing::field::Empty,
+ ),
+ err
+ )]
+ async fn can_view(
+ &self,
+ access: &impl EntityViewAccess,
+ entity: &Entity<'_>,
+ ) -> Result<bool, AccessError> {
+ let allowed = match entity.entity_type {
+ // A user entity carries everything sent to that user, so only the
+ // user themselves may subscribe to it.
+ EntityType::User => MacroUserIdStr::parse_from_str(&entity.entity_id)
+ .is_ok_and(|requested| requested.0.as_ref() == self.user_id.0.as_ref()),
+ _ => {
+ access
+ .can_view(&self.user_id, self.organization_id, entity)
+ .await?
+ }
+ };
+ tracing::Span::current().record("allowed", allowed);
+ Ok(allowed)
+ }
+}
+
+impl Drop for EntityGrants {
+ /// The connection is gone; its stream listeners have no one to serve.
+ fn drop(&mut self) {
+ for listener in self.stream_listeners.values() {
+ listener.abort();
+ }
+ }
+}
diff --git a/services/connection_gateway/src/service/entity_grants/test.rs b/services/connection_gateway/src/service/entity_grants/test.rs
new file mode 100644
index 0000000000..98433b185c
--- /dev/null
+++ b/services/connection_gateway/src/service/entity_grants/test.rs
@@ -0,0 +1,428 @@
+use std::{
+ collections::HashSet,
+ sync::{
+ Mutex,
+ atomic::{AtomicUsize, Ordering},
+ },
+};
+
+use entity_access::domain::models::AccessError;
+use macro_user_id::user_id::MacroUserIdStr;
+use model_entity::{Entity, EntityType};
+
+use super::{Admission, EntityGrants, EntityViewAccess, viewable};
+use crate::model::tracking::TrackAction;
+
+const VIEWER: &str = "macro|[email protected]";
+const OTHER_USER: &str = "macro|[email protected]";
+const SHARED_DOCUMENT: &str = "0198f7a2-0000-7000-8000-000000000001";
+const PRIVATE_CHANNEL: &str = "0198f7a2-0000-7000-8000-000000000002";
+const BROKEN_CHAT: &str = "0198f7a2-0000-7000-8000-000000000003";
+
+/// Grants view access to a fixed set of entities and fails for `BROKEN_CHAT`.
+#[derive(Default)]
+struct FakeAccess {
+ viewable: Mutex<HashSet<(EntityType, String)>>,
+ lookups: AtomicUsize,
+}
+
+impl FakeAccess {
+ fn with(entities: &[(EntityType, &str)]) -> Self {
+ let access = Self::default();
+ for (entity_type, entity_id) in entities {
+ access.grant(*entity_type, entity_id);
+ }
+ access
+ }
+
+ fn grant(&self, entity_type: EntityType, entity_id: &str) {
+ self.viewable
+ .lock()
+ .unwrap()
+ .insert((entity_type, entity_id.to_string()));
+ }
+
+ fn revoke(&self, entity_type: EntityType, entity_id: &str) {
+ self.viewable
+ .lock()
+ .unwrap()
+ .remove(&(entity_type, entity_id.to_string()));
+ }
+
+ fn lookups(&self) -> usize {
+ self.lookups.load(Ordering::SeqCst)
+ }
+}
+
+impl EntityViewAccess for FakeAccess {
+ async fn can_view(
+ &self,
+ user_id: &MacroUserIdStr<'_>,
+ _organization_id: Option<i64>,
+ entity: &Entity<'_>,
+ ) -> Result<bool, AccessError> {
+ self.lookups.fetch_add(1, Ordering::SeqCst);
+ assert_eq!(user_id.0.as_ref(), VIEWER);
+ if entity.entity_id == BROKEN_CHAT {
+ return Err(AccessError::internal("database unavailable"));
+ }
+ Ok(self
+ .viewable
+ .lock()
+ .unwrap()
+ .contains(&(entity.entity_type, entity.entity_id.to_string())))
+ }
+}
+
+fn grants() -> EntityGrants {
+ EntityGrants::new(MacroUserIdStr::parse_from_str(VIEWER).unwrap(), None)
+}
+
+fn entity(entity_type: EntityType, entity_id: &str) -> Entity<'_> {
+ entity_type.with_entity_str(entity_id)
+}
+
+#[tokio::test]
+async fn only_the_connected_user_may_track_their_user_entity() {
+ let access = FakeAccess::default();
+ let mut grants = grants();
+
+ assert_eq!(
+ grants
+ .admit(
+ &access,
+ &entity(EntityType::User, OTHER_USER),
+ TrackAction::Open
+ )
+ .await
+ .unwrap(),
+ Admission::Refused
+ );
+ assert_eq!(
+ grants
+ .admit(
+ &access,
+ &entity(EntityType::User, OTHER_USER),
+ TrackAction::Ping
+ )
+ .await
+ .unwrap(),
+ Admission::Refused
+ );
+ assert!(
+ !grants
+ .permits(&access, &entity(EntityType::User, OTHER_USER))
+ .await
+ .unwrap()
+ );
+ assert_eq!(
+ grants
+ .admit(
+ &access,
+ &entity(EntityType::User, "macro|[email protected]"),
+ TrackAction::Open
+ )
+ .await
+ .unwrap(),
+ Admission::Admitted
+ );
+ assert_eq!(
+ grants
+ .admit(
+ &access,
+ &entity(EntityType::User, "not a user id"),
+ TrackAction::Open
+ )
+ .await
+ .unwrap(),
+ Admission::Refused
+ );
+ assert_eq!(
+ access.lookups(),
+ 0,
+ "user entities never reach entity access"
+ );
+}
+
+#[tokio::test]
+async fn open_is_admitted_only_for_viewable_entities() {
+ let access = FakeAccess::with(&[(EntityType::Document, SHARED_DOCUMENT)]);
+ let mut grants = grants();
+
+ assert_eq!(
+ grants
+ .admit(
+ &access,
+ &entity(EntityType::Document, SHARED_DOCUMENT),
+ TrackAction::Open
+ )
+ .await
+ .unwrap(),
+ Admission::Admitted
+ );
+ assert_eq!(
+ grants
+ .admit(
+ &access,
+ &entity(EntityType::Channel, PRIVATE_CHANNEL),
+ TrackAction::Open
+ )
+ .await
+ .unwrap(),
+ Admission::Refused
+ );
+ // The id alone is not enough: the type is part of what was checked.
+ assert_eq!(
+ grants
+ .admit(
+ &access,
+ &entity(EntityType::Chat, SHARED_DOCUMENT),
+ TrackAction::Open
+ )
+ .await
+ .unwrap(),
+ Admission::Refused
+ );
+}
+
+#[tokio::test]
+async fn ping_without_a_grant_is_checked_like_an_open() {
+ let access = FakeAccess::with(&[(EntityType::Document, SHARED_DOCUMENT)]);
+ let mut grants = grants();
+
+ assert_eq!(
+ grants
+ .admit(
+ &access,
+ &entity(EntityType::Channel, PRIVATE_CHANNEL),
+ TrackAction::Ping
+ )
+ .await
+ .unwrap(),
+ Admission::Refused
+ );
+ assert_eq!(
+ grants
+ .admit(
+ &access,
+ &entity(EntityType::Document, SHARED_DOCUMENT),
+ TrackAction::Ping
+ )
+ .await
+ .unwrap(),
+ Admission::Admitted
+ );
+ assert_eq!(access.lookups(), 2);
+
+ // Granted now, so later pings skip the lookup.
+ for _ in 0..3 {
+ assert_eq!(
+ grants
+ .admit(
+ &access,
+ &entity(EntityType::Document, SHARED_DOCUMENT),
+ TrackAction::Ping
+ )
+ .await
+ .unwrap(),
+ Admission::Admitted
+ );
+ }
+ assert_eq!(access.lookups(), 2);
+}
+
+#[tokio::test]
+async fn close_needs_a_grant_and_ends_it() {
+ let access = FakeAccess::with(&[(EntityType::Document, SHARED_DOCUMENT)]);
+ let mut grants = grants();
+ let document = entity(EntityType::Document, SHARED_DOCUMENT);
+
+ assert_eq!(
+ grants
+ .admit(
+ &access,
+ &entity(EntityType::Channel, PRIVATE_CHANNEL),
+ TrackAction::Close
+ )
+ .await
+ .unwrap(),
+ Admission::Refused
+ );
+ assert_eq!(
+ grants
+ .admit(&access, &document, TrackAction::Close)
+ .await
+ .unwrap(),
+ Admission::Refused,
+ "never opened on this connection"
+ );
+
+ grants
+ .admit(&access, &document, TrackAction::Open)
+ .await
+ .unwrap();
+ assert_eq!(
+ grants
+ .admit(&access, &document, TrackAction::Close)
+ .await
+ .unwrap(),
+ Admission::Admitted
+ );
+
+ let lookups = access.lookups();
+ grants
+ .admit(&access, &document, TrackAction::Ping)
+ .await
+ .unwrap();
+ assert_eq!(access.lookups(), lookups + 1, "the close dropped the grant");
+}
+
+#[tokio::test]
+async fn reopening_an_entity_whose_access_was_removed_revokes_it() {
+ let access = FakeAccess::with(&[(EntityType::Document, SHARED_DOCUMENT)]);
+ let mut grants = grants();
+ let document = entity(EntityType::Document, SHARED_DOCUMENT);
+
+ grants
+ .admit(&access, &document, TrackAction::Open)
+ .await
+ .unwrap();
+ access.revoke(EntityType::Document, SHARED_DOCUMENT);
+
+ assert_eq!(
+ grants
+ .admit(&access, &document, TrackAction::Open)
+ .await
+ .unwrap(),
+ Admission::Revoked
+ );
+ assert_eq!(
+ grants
+ .admit(&access, &document, TrackAction::Close)
+ .await
+ .unwrap(),
+ Admission::Refused
+ );
+}
+
+#[tokio::test]
+async fn a_failed_check_is_an_error_and_grants_nothing() {
+ let access = FakeAccess::default();
+ let mut grants = grants();
+ let chat = entity(EntityType::Chat, BROKEN_CHAT);
+
+ assert!(
+ grants
+ .admit(&access, &chat, TrackAction::Open)
+ .await
+ .is_err()
+ );
+ assert!(grants.permits(&access, &chat).await.is_err());
+ assert_eq!(
+ grants
+ .admit(&access, &chat, TrackAction::Close)
+ .await
+ .unwrap(),
+ Admission::Refused
+ );
+}
+
+#[tokio::test]
+async fn revalidation_revokes_lost_access_and_keeps_the_rest() {
+ let access = FakeAccess::with(&[
+ (EntityType::Document, SHARED_DOCUMENT),
+ (EntityType::Channel, PRIVATE_CHANNEL),
+ (EntityType::Chat, BROKEN_CHAT),
+ ]);
+ let mut grants = grants();
+ for granted in [
+ entity(EntityType::Document, SHARED_DOCUMENT),
+ entity(EntityType::Channel, PRIVATE_CHANNEL),
+ ] {
+ grants
+ .admit(&access, &granted, TrackAction::Open)
+ .await
+ .unwrap();
+ }
+ // A grant whose recheck will fail.
+ grants
+ .granted
+ .insert((EntityType::Chat, BROKEN_CHAT.to_string()));
+
+ access.revoke(EntityType::Channel, PRIVATE_CHANNEL);
+ let revoked = grants.revalidate(&access).await;
+
+ assert_eq!(revoked.len(), 1);
+ assert_eq!(revoked[0].entity_type, EntityType::Channel);
+ assert_eq!(revoked[0].entity_id, PRIVATE_CHANNEL);
+ assert_eq!(
+ grants.granted,
+ HashSet::from([
+ (EntityType::Document, SHARED_DOCUMENT.to_string()),
+ (EntityType::Chat, BROKEN_CHAT.to_string()),
+ ])
+ );
+
+ access.grant(EntityType::Channel, PRIVATE_CHANNEL);
+ assert!(grants.revalidate(&access).await.is_empty());
+}
+
+#[test]
+fn access_errors_that_mean_no_access_read_as_not_viewable() {
+ assert!(viewable(Ok::<_, AccessError>(())).unwrap());
+ for denied in [
+ AccessError::Unauthorized,
+ AccessError::UnauthorizedWithMessage("no"),
+ AccessError::NotFound("document"),
+ AccessError::BadRequest("Unsupported entity type"),
+ ] {
+ assert!(!viewable(Err::<(), _>(denied)).unwrap());
+ }
+ assert!(viewable(Err::<(), _>(AccessError::internal("boom"))).is_err());
+}
+
+#[tokio::test]
+async fn revalidation_stops_stream_listeners_whose_access_was_lost() {
+ let access = FakeAccess::with(&[
+ (EntityType::Document, SHARED_DOCUMENT),
+ (EntityType::Channel, PRIVATE_CHANNEL),
+ ]);
+ let mut grants = grants();
+ let kept = tokio::spawn(std::future::pending::<()>());
+ let lost = tokio::spawn(std::future::pending::<()>());
+ grants.hold_stream_listener(
+ &entity(EntityType::Document, SHARED_DOCUMENT),
+ kept.abort_handle(),
+ );
+ grants.hold_stream_listener(
+ &entity(EntityType::Channel, PRIVATE_CHANNEL),
+ lost.abort_handle(),
+ );
+
+ access.revoke(EntityType::Channel, PRIVATE_CHANNEL);
+ let revoked = grants.revalidate(&access).await;
+
+ assert!(revoked.is_empty(), "stream events are not tracked rows");
+ assert!(lost.await.unwrap_err().is_cancelled());
+ assert!(!kept.is_finished());
+
+ drop(grants);
+ assert!(
+ kept.await.unwrap_err().is_cancelled(),
+ "a closed connection stops its listeners"
+ );
+}
+
+#[tokio::test]
+async fn a_repeated_stream_events_frame_replaces_the_listener() {
+ let mut grants = grants();
+ let document = entity(EntityType::Document, SHARED_DOCUMENT);
+ let first = tokio::spawn(std::future::pending::<()>());
+ let second = tokio::spawn(std::future::pending::<()>());
+
+ grants.hold_stream_listener(&document, first.abort_handle());
+ grants.hold_stream_listener(&document, second.abort_handle());
+
+ assert!(first.await.unwrap_err().is_cancelled());
+ assert!(!second.is_finished());
+}
diff --git a/services/connection_gateway/src/service/mod.rs b/services/connection_gateway/src/service/mod.rs
index 02eb553f0c..50bbfb83a7 100644
--- a/services/connection_gateway/src/service/mod.rs
+++ b/services/connection_gateway/src/service/mod.rs
@@ -1,5 +1,6 @@
pub mod connection;
pub mod dynamodb;
+pub mod entity_grants;
pub mod fanout;
pub mod redis;
pub mod sender;
diff --git a/services/connection_gateway/src/service/stream_event.rs b/services/connection_gateway/src/service/stream_event.rs
index 803e2692d0..9acbe8b417 100644
--- a/services/connection_gateway/src/service/stream_event.rs
+++ b/services/connection_gateway/src/service/stream_event.rs
@@ -4,7 +4,7 @@
//! while StreamItems represent stream data itself
use stream::domain::{StreamEvent, StreamId};
-use tokio::sync::mpsc::Sender;
+use tokio::{sync::mpsc::Sender, task::AbortHandle};
use crate::model::{
connection::ConnectionContext, message::OutgoingMessage, websocket::StreamEvents,
@@ -15,10 +15,10 @@ pub fn handle_stream_events(
context: ConnectionContext<'_>,
sender: Sender<OutgoingMessage>,
incoming_message: StreamEvents,
-) {
+) -> AbortHandle {
let repo = context.api_context.stream_manager.repo();
- tokio::spawn(async move {
+ let listener = tokio::spawn(async move {
// if any prexisting streams exist emit open entity
if let Ok(streams) = repo
.active_streams(&incoming_message.entity.entity_id)
@@ -69,4 +69,5 @@ pub fn handle_stream_events(
}
}
});
+ listener.abort_handle()
}