Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(connection-gateway): skip and retire malformed connection rows on read (#9726)

Wolf Mermelsteine76c6e667b95
diff --git a/services/connection_gateway/AGENTS.md b/services/connection_gateway/AGENTS.md
index 9fed660144..1089fabf3b 100644
--- a/services/connection_gateway/AGENTS.md
+++ b/services/connection_gateway/AGENTS.md
@@ -63,6 +63,13 @@ the socket themselves.
after its last `open` or `ping`, compared across instances' clocks. A `ping`
for a row that is gone re-creates it, so a lost `open` heals on the next
heartbeat.
+- Reads skip rows that do not parse as a connection entry, so one bad row
+ never fails delivery to the rest of an entity or a batch. A row missing
+ entry attributes (left by pings before they checked their row exists) is
+ deleted on read, conditioned on those attributes still being absent; a
+ whole row this instance cannot read (e.g. an entity type from a newer
+ deploy) is kept. Keep every write of a new attribute conditional on the
+ row existing.
- Broadcasts use their own Redis channel (`BROADCAST_CHANNEL`): each instance
forwards them to every connection it holds, with no DynamoDB lookup and no
receipts. Keep them rare (e.g. the platform status banner); a broadcast is
diff --git a/services/connection_gateway/src/service/dynamodb.rs b/services/connection_gateway/src/service/dynamodb.rs
index a7151a8842..b21d615bc0 100644
--- a/services/connection_gateway/src/service/dynamodb.rs
+++ b/services/connection_gateway/src/service/dynamodb.rs
@@ -92,6 +92,128 @@ struct DynamoDbConnectionGatewayPersistence {
const CONNECTION_ID_GSI: &str = "ConnectionPkIndex";
+/// The attributes every connection entry has, besides its keys.
+const ENTRY_ATTRIBUTES: [&str; 5] = [
+ "entity_type",
+ "entity_id",
+ "connection_id",
+ "created_at",
+ "user_id",
+];
+
+/// A row in the table that does not read as a connection entry.
+#[derive(Debug, PartialEq, Eq)]
+struct MalformedRow {
+ pk: String,
+ sk: String,
+ /// The [ENTRY_ATTRIBUTES] the row lacks. A row lacking any can never be
+ /// a live connection: pings wrote rows like that for entries already
+ /// gone, before they checked that their row exists. A row lacking none
+ /// is whole but unreadable here, such as one a newer instance wrote with
+ /// an entity type this one does not know.
+ missing: Vec<&'static str>,
+}
+
+/// Rows read from the table, split into connection entries and the rows that
+/// are not one.
+#[derive(Debug, Default)]
+struct ReadRows {
+ entries: Vec<StoredConnectionEntity>,
+ malformed: Vec<MalformedRow>,
+}
+
+fn read_rows(items: Vec<HashMap<String, AttributeValue>>) -> Result<ReadRows> {
+ let mut rows = ReadRows::default();
+ for item in items {
+ let key = |name: &str| {
+ item.get(name)
+ .and_then(|value| value.as_s().ok())
+ .cloned()
+ .with_context(|| format!("connection row without a {name} key"))
+ };
+ let (pk, sk) = (key("PK")?, key("SK")?);
+ let missing = ENTRY_ATTRIBUTES
+ .into_iter()
+ .filter(|attribute| !item.contains_key(*attribute))
+ .collect();
+ match serde_dynamo::from_item(item) {
+ Ok(entry) => rows.entries.push(entry),
+ Err(_) => rows.malformed.push(MalformedRow { pk, sk, missing }),
+ }
+ }
+ Ok(rows)
+}
+
+impl DynamoDbConnectionGatewayPersistence {
+ /// The connection entries among `items`, deleting the rows that can never
+ /// be one so they stop costing every read of their entity.
+ ///
+ /// A malformed row is left out rather than failing the read: it is one
+ /// connection, and failing would cut off every other connection to the
+ /// entity, and every other entity in a batch send beside it.
+ async fn connection_entries(
+ &self,
+ items: Vec<HashMap<String, AttributeValue>>,
+ ) -> Result<Vec<StoredConnectionEntity>> {
+ let ReadRows { entries, malformed } = read_rows(items)?;
+ for row in malformed {
+ if row.missing.is_empty() {
+ tracing::warn!(
+ connection_id = %row.sk,
+ "skipping a connection row this instance cannot read"
+ );
+ continue;
+ }
+ let retired = self.retire(&row).await;
+ tracing::warn!(
+ connection_id = %row.sk,
+ missing = ?row.missing,
+ retired = retired.is_ok(),
+ error = ?retired.err(),
+ "skipping a connection row missing entry attributes"
+ );
+ }
+ Ok(entries)
+ }
+
+ /// Deletes a row missing entry attributes, unless a whole entry has
+ /// replaced it since it was read.
+ async fn retire(&self, row: &MalformedRow) -> Result<()> {
+ let names: HashMap<String, String> = row
+ .missing
+ .iter()
+ .map(|attribute| (format!("#{attribute}"), (*attribute).to_owned()))
+ .collect();
+ let condition = row
+ .missing
+ .iter()
+ .map(|attribute| format!("attribute_not_exists(#{attribute})"))
+ .collect::<Vec<_>>()
+ .join(" AND ");
+
+ match self
+ .client
+ .delete_item()
+ .table_name(self.table_exists.as_ref())
+ .key("PK", AttributeValue::S(row.pk.clone()))
+ .key("SK", AttributeValue::S(row.sk.clone()))
+ .condition_expression(condition)
+ .set_expression_attribute_names(Some(names))
+ .send()
+ .await
+ {
+ Ok(_) => Ok(()),
+ Err(e)
+ if e.as_service_error().and_then(|se| se.code())
+ == Some("ConditionalCheckFailedException") =>
+ {
+ Ok(())
+ }
+ Err(e) => Err(aws_sdk_dynamodb::Error::from(e)).context("failed to delete row"),
+ }
+ }
+}
+
#[async_trait]
impl ConnectionRepo for DynamoDbConnectionGatewayPersistence {
async fn insert_connection_entry(
@@ -133,10 +255,7 @@ impl ConnectionRepo for DynamoDbConnectionGatewayPersistence {
.items
.context("no items found")?;
- let items: Vec<StoredConnectionEntity> =
- serde_dynamo::from_items(result).context("failed to deserialize items")?;
-
- Ok(items)
+ self.connection_entries(result).await
}
async fn get_entries_by_connection_id(
@@ -158,10 +277,7 @@ impl ConnectionRepo for DynamoDbConnectionGatewayPersistence {
.items
.context("no items found")?;
- let items: Vec<StoredConnectionEntity> =
- serde_dynamo::from_items(result).context("failed to deserialize items")?;
-
- Ok(items)
+ self.connection_entries(result).await
}
async fn get_connection(&self, connection_id: &str) -> Result<StoredConnectionEntity> {
@@ -183,12 +299,11 @@ impl ConnectionRepo for DynamoDbConnectionGatewayPersistence {
let items = result.items.context("no items found")?;
- let first = items.first().context("no items found")?;
-
- let item: StoredConnectionEntity =
- serde_dynamo::from_item(first.clone()).context("failed to deserialize item")?;
-
- Ok(item)
+ self.connection_entries(items)
+ .await?
+ .into_iter()
+ .next()
+ .context("no items found")
}
async fn get_entry_for_connection_entity(
@@ -217,8 +332,7 @@ impl ConnectionRepo for DynamoDbConnectionGatewayPersistence {
.items
.context("no items found")?;
- let items: Vec<StoredConnectionEntity> =
- serde_dynamo::from_items(result).context("failed to deserialize items")?;
+ let items = self.connection_entries(result).await?;
if items.len() > 1 {
tracing::warn!(
@@ -384,3 +498,6 @@ pub async fn create_dynamo_db_connection_manager(
Ok(ConnectionManager::new(persistence))
}
+
+#[cfg(test)]
+mod tests;
diff --git a/services/connection_gateway/src/service/dynamodb/tests.rs b/services/connection_gateway/src/service/dynamodb/tests.rs
new file mode 100644
index 0000000000..c53cccbfdf
--- /dev/null
+++ b/services/connection_gateway/src/service/dynamodb/tests.rs
@@ -0,0 +1,103 @@
+use super::*;
+
+#[test]
+fn rows_missing_entry_attributes_are_left_out_beside_whole_entries() {
+ let whole = HashMap::from([
+ (
+ "PK".to_owned(),
+ AttributeValue::S("#user#macro|[email protected]".to_owned()),
+ ),
+ ("SK".to_owned(), AttributeValue::S("live".to_owned())),
+ (
+ "entity_type".to_owned(),
+ AttributeValue::S("user".to_owned()),
+ ),
+ (
+ "entity_id".to_owned(),
+ AttributeValue::S("macro|[email protected]".to_owned()),
+ ),
+ (
+ "connection_id".to_owned(),
+ AttributeValue::S("live".to_owned()),
+ ),
+ (
+ "created_at".to_owned(),
+ AttributeValue::N("1000".to_owned()),
+ ),
+ (
+ "user_id".to_owned(),
+ AttributeValue::S("macro|[email protected]".to_owned()),
+ ),
+ ("last_ping".to_owned(), AttributeValue::N("2000".to_owned())),
+ ]);
+ // What a ping for an entry already gone used to leave behind.
+ let pinged_after_close = HashMap::from([
+ (
+ "PK".to_owned(),
+ AttributeValue::S("#user#macro|[email protected]".to_owned()),
+ ),
+ ("SK".to_owned(), AttributeValue::S("closed".to_owned())),
+ ("last_ping".to_owned(), AttributeValue::N("3000".to_owned())),
+ ]);
+
+ let rows = read_rows(vec![pinged_after_close, whole]).unwrap();
+
+ assert_eq!(
+ rows.entries
+ .iter()
+ .map(|entry| entry.connection_id.as_str())
+ .collect::<Vec<_>>(),
+ vec!["live"]
+ );
+ assert_eq!(
+ rows.malformed,
+ vec![MalformedRow {
+ pk: "#user#macro|[email protected]".to_owned(),
+ sk: "closed".to_owned(),
+ missing: vec![
+ "entity_type",
+ "entity_id",
+ "connection_id",
+ "created_at",
+ "user_id"
+ ],
+ }]
+ );
+}
+
+#[test]
+fn a_whole_row_of_an_unknown_entity_type_is_malformed_but_missing_nothing() {
+ let newer = HashMap::from([
+ ("PK".to_owned(), AttributeValue::S("#hologram#h".to_owned())),
+ ("SK".to_owned(), AttributeValue::S("connection".to_owned())),
+ (
+ "entity_type".to_owned(),
+ AttributeValue::S("hologram".to_owned()),
+ ),
+ ("entity_id".to_owned(), AttributeValue::S("h".to_owned())),
+ (
+ "connection_id".to_owned(),
+ AttributeValue::S("connection".to_owned()),
+ ),
+ (
+ "created_at".to_owned(),
+ AttributeValue::N("1000".to_owned()),
+ ),
+ (
+ "user_id".to_owned(),
+ AttributeValue::S("macro|[email protected]".to_owned()),
+ ),
+ ]);
+
+ let rows = read_rows(vec![newer]).unwrap();
+
+ assert!(rows.entries.is_empty());
+ assert_eq!(
+ rows.malformed,
+ vec![MalformedRow {
+ pk: "#hologram#h".to_owned(),
+ sk: "connection".to_owned(),
+ missing: vec![],
+ }]
+ );
+}