Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

11.7 KB
Raw Blame
use crate::model::{
    connection::{Connection, StoredConnectionEntity},
    message::{Message, OutgoingMessage},
    tracking::{EntityConnection, UserEntityConnection},
};
use anyhow::Result;
use async_trait::async_trait;
use dashmap::DashMap;
use model_entity::Entity;
use std::{
    sync::{Arc, atomic::AtomicUsize},
    time::{SystemTime, UNIX_EPOCH},
};
use tokio::{sync::mpsc::Sender, task::AbortHandle};

#[cfg(test)]
mod test;

#[async_trait]
pub trait ConnectionRepo: Send + Sync {
    /// Inserts a new [StoredConnectionEntity] into storage
    async fn insert_connection_entry(
        &self,
        connection: UserEntityConnection<'_>,
    ) -> anyhow::Result<StoredConnectionEntity>;

    /// Get a list of [StoredConnectionEntity] by an entity type and id
    /// Multiple connections can be associated with an entity
    async fn get_entries_by_entity(
        &self,
        entity: &Entity<'_>,
    ) -> anyhow::Result<Vec<StoredConnectionEntity>>;

    /// Gets all entries for a given `connection_id`
    async fn get_entries_by_connection_id(
        &self,
        connection_id: &str,
    ) -> anyhow::Result<Vec<StoredConnectionEntity>>;

    /// Gets a connection by a connection_id
    /// This returns the top level connection item for a user
    /// so it would return a #user#<user_id> with connection_id
    async fn get_connection(&self, connection_id: &str) -> anyhow::Result<StoredConnectionEntity>;

    async fn get_entry_for_connection_entity(
        &self,
        entity: EntityConnection<'_>,
    ) -> anyhow::Result<Option<StoredConnectionEntity>>;

    /// Deletes all [StoredConnectionEntity] entries for a given `connection_id`
    async fn remove_all_entries_for_by_connection_id(
        &self,
        connection_id: &str,
    ) -> anyhow::Result<()>;

    /// Deletes a [StoredConnectionEntity] from the database
    async fn remove_entity(&self, entity: &EntityConnection<'_>) -> anyhow::Result<()>;

    /// Updates the last_ping timestamp for a given connection entity, or
    /// returns `None` when there is no entry for it
    async fn update_last_entity_ping(
        &self,
        entity: &EntityConnection<'_>,
        timestamp: u64,
    ) -> anyhow::Result<Option<StoredConnectionEntity>>;

    /// Updates the last_ping timestamp for a given top level connection
    async fn update_user_connection_last_ping(
        &self,
        connection_id: &str,
        user: &str,
        timestamp: u64,
    ) -> anyhow::Result<()>;
}

/// What became of a message offered to a connection's outbound queue.
#[derive(Debug, PartialEq, Eq)]
pub enum SendOutcome {
    /// The message is queued for the connection's forwarder to write.
    Queued,
    /// The connection would not take the message, so it was dropped.
    Dropped,
}

#[derive(Clone)]
pub struct ConnectionManager {
    pub connections: Arc<DashMap<String, Connection>>,
    pub connection_count: Arc<AtomicUsize>,
    pub persistence: Arc<dyn ConnectionRepo>,
}

impl ConnectionManager {
    pub fn new(persistence: impl ConnectionRepo + 'static) -> Self {
        tracing::info!("creating new connection manager");
        Self {
            connections: Arc::new(DashMap::new()),
            connection_count: Arc::new(AtomicUsize::new(0)),
            persistence: Arc::new(persistence),
        }
    }

    /// Checks wether or not a given connection_id exists in the connection manager
    /// A connection_id might still exists in the database, but it is not handled by this instance
    /// of the connection manager.
    pub fn has_connection(&self, id: &str) -> bool {
        self.connections.contains_key(id)
    }

    /// Initiate a new connection for a user
    /// A connection represents a users activity in a single tab/session
    /// A connection_id is unique to a user, but it can be shared across multiple
    /// connection_entities that are associated with the same user.
    pub async fn add_connection(
        &self,
        connection: UserEntityConnection<'_>,
        sender: Sender<OutgoingMessage>,
        abort_handle: AbortHandle,
    ) -> Result<()> {
        if self
            .connections
            .contains_key(connection.extra.connection_id.as_ref())
        {
            tracing::warn!("connection already exists for connection id");
            return Ok(());
        }
        self.connections.insert(
            connection.extra.connection_id.to_string(),
            Connection {
                sender,
                abort_handle,
            },
        );
        self.connection_count
            .fetch_add(1, std::sync::atomic::Ordering::SeqCst);

        self.persistence.insert_connection_entry(connection).await?;

        Ok(())
    }

    /// Adds a new connection entity associated with a user connection
    /// A connection entity can represetn a users activity on a document, chat, channel etc...
    #[tracing::instrument(err, skip(self))]
    pub async fn add_connection_entity(
        &self,
        connection_entity: UserEntityConnection<'_>,
    ) -> Result<()> {
        self.persistence
            .insert_connection_entry(connection_entity)
            .await?;

        Ok(())
    }

    /// Removes all associated connection entities for the given connection id
    /// IMPORTANT: this removes all entities associated with the connection_id
    /// and not just a specific entity like document, chat, channel activity.
    pub async fn remove_connection(&self, connection_id: &str) -> Result<()> {
        // Removing twice is the normal case, not an edge one: a refused send
        // removes the connection and aborts its forwarder, and the socket
        // handler removes it again when it sees that task end. Taking the
        // entry out is the one step that cannot happen twice, so it is what
        // the count and the abort hang off - `fetch_sub` on a count already
        // at zero wraps to `usize::MAX`.
        if let Some((_, connection)) = self.connections.remove(connection_id) {
            self.connection_count
                .fetch_sub(1, std::sync::atomic::Ordering::SeqCst);
            // Killing the task that forwards messages to the websocket.
            if !connection.abort_handle.is_finished() {
                connection.abort_handle.abort();
            }
        }

        // Unconditional: a connection this instance never held still has rows
        // to clear, and the delete is idempotent.
        self.persistence
            .remove_all_entries_for_by_connection_id(connection_id)
            .await?;

        Ok(())
    }

    /// Removes an individual entity associated with a connection_id and entity
    /// The actual connection_id associated with the user is not removed
    /// Just the single instance of the entity associated with a document, chat, channel etc...
    #[tracing::instrument(err, skip(self))]
    pub async fn remove_connection_entity(&self, entity: &EntityConnection<'_>) -> Result<()> {
        self.persistence.remove_entity(entity).await?;

        Ok(())
    }

    /// Updates the last_ping timestamp for a given connection entity
    /// This also propogates to the last_ping timestamp for the top level user connection
    ///
    /// A ping for an entity with no entry re-creates it: the client is still
    /// watching, so a lost `open` (a failed write, a row cleared under a live
    /// connection) must not hide the user until they reopen the entity.
    #[tracing::instrument(err, skip(self))]
    pub async fn refresh_connection_entity(
        &self,
        connection_entity: UserEntityConnection<'_>,
    ) -> Result<()> {
        let updated_timestamp = SystemTime::now()
            .duration_since(UNIX_EPOCH)
            .unwrap()
            .as_millis() as u64;
        let connection_id = connection_entity.extra.connection_id.to_string();
        let user_id = connection_entity.user_id.to_string();

        let refreshed = self
            .persistence
            .update_last_entity_ping(&connection_entity.extra, updated_timestamp)
            .await?;
        if refreshed.is_none() {
            tracing::warn!(
                connection_id = %connection_id,
                entity_type = %connection_entity.extra.extra.entity_type,
                entity_id = %connection_entity.extra.extra.entity_id,
                "ping for an untracked entity; re-creating its entry"
            );
            self.persistence
                .insert_connection_entry(connection_entity)
                .await?;
        }

        self.persistence
            .update_user_connection_last_ping(&connection_id, &user_id, updated_timestamp)
            .await?;

        Ok(())
    }

    /// Offers a message to a connection's outbound queue, and drops the
    /// connection if it will not take it.
    ///
    /// Never waits. The queue is this connection's alone but the callers are
    /// everyone, so a consumer that has stopped draining must cost them
    /// nothing: waiting for capacity here is indistinguishable from being
    /// down to whoever is publishing, and a publisher holding its own
    /// deadline fails the work behind the send rather than the one delivery.
    ///
    /// A full queue is not a slow consumer, it is a consumer already a whole
    /// queue behind, so it is treated the same as a closed one: the
    /// connection goes, which also aborts the forwarder that stopped
    /// draining, and the client reconnects. That is the designed way to shed
    /// a client that stopped reading, not a failed send, so it is reported as
    /// [SendOutcome::Dropped] rather than an error.
    #[tracing::instrument(
        name = "connection_gateway.queue_send",
        err,
        skip(self, id, message),
        fields(
            connection.id = id,
            connection.queue.observed_depth = tracing::field::Empty,
            connection.queue.max_capacity = tracing::field::Empty,
            connection.dropped = tracing::field::Empty,
        )
    )]
    pub async fn send_message(&self, id: &str, message: Message) -> Result<SendOutcome> {
        let sender = match self.connections.get(id) {
            Some(connection) => connection.sender.clone(),
            None => anyhow::bail!("connection not found"),
        };

        let remaining_capacity = sender.capacity();
        let max_capacity = sender.max_capacity();
        let depth = max_capacity.saturating_sub(remaining_capacity);
        let span = tracing::Span::current();
        span.record("connection.queue.observed_depth", depth as u64);
        span.record("connection.queue.max_capacity", max_capacity as u64);

        if let Err(err) = sender.try_send(OutgoingMessage::Message(message)) {
            span.record("connection.dropped", true);
            tracing::warn!(
                error = %err,
                connection.id = id,
                connection.queue.observed_depth = depth,
                connection.queue.max_capacity = max_capacity,
                "websocket outbound queue would not take a message; dropping the connection"
            );
            self.remove_connection(id).await?;
            return Ok(SendOutcome::Dropped);
        }

        Ok(SendOutcome::Queued)
    }

    /// Send a raw binary frame to a connection held by this instance.
    pub async fn send_binary(&self, id: &str, bytes: Vec<u8>) -> Result<()> {
        let sender = match self.connections.get(id) {
            Some(connection) => connection.sender.clone(),
            None => anyhow::bail!("connection not found"),
        };

        if let Err(err) = sender.try_send(OutgoingMessage::Binary(bytes)) {
            self.remove_connection(id).await?;
            anyhow::bail!("failed to send binary message: {}", err);
        }

        Ok(())
    }

    pub fn get_entries_by_entity<'a>(
        &'a self,
        entity: &'a Entity<'a>,
    ) -> impl Future<Output = Result<Vec<StoredConnectionEntity>>> + 'a {
        self.persistence.get_entries_by_entity(entity)
    }
}