Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
perf(frecency): claim event batches instead of holding the read open (#6332)
Will Hutchinson7db736e48b70
diff --git a/.sqlx/query-89004da6bcdd66bccd56ed64402d82dc4fb489cccaf5ca647c5a9403fa760ce9.json b/.sqlx/query-0115c52b6c77a377e6585308ba0df3daaaf7d30a19a37b28abcae7efbe9b4ca7.jsonsimilarity index 54%rename from .sqlx/query-89004da6bcdd66bccd56ed64402d82dc4fb489cccaf5ca647c5a9403fa760ce9.jsonrename to .sqlx/query-0115c52b6c77a377e6585308ba0df3daaaf7d30a19a37b28abcae7efbe9b4ca7.jsonindex a989abc850..6e5e69ddc6 100644--- a/.sqlx/query-89004da6bcdd66bccd56ed64402d82dc4fb489cccaf5ca647c5a9403fa760ce9.json+++ b/.sqlx/query-0115c52b6c77a377e6585308ba0df3daaaf7d30a19a37b28abcae7efbe9b4ca7.json@@ -1,11 +1,11 @@{"db_name": "PostgreSQL",- "query": "\n SELECT pg_try_advisory_xact_lock($1)\n ",+ "query": "SELECT pg_advisory_unlock($1)","describe": {"columns": [{"ordinal": 0,- "name": "pg_try_advisory_xact_lock",+ "name": "pg_advisory_unlock","type_info": "Bool"}],@@ -18,5 +18,5 @@null]},- "hash": "89004da6bcdd66bccd56ed64402d82dc4fb489cccaf5ca647c5a9403fa760ce9"+ "hash": "0115c52b6c77a377e6585308ba0df3daaaf7d30a19a37b28abcae7efbe9b4ca7"}diff --git a/.sqlx/query-d42e3270642ca0b9eff6b5807f600c24fd26d50d612a66edf0738c62d15680ba.json b/.sqlx/query-38b755d094bcfc511934bfc9b1e77fff2b01630b80dba6749f70884ff18f6afd.jsonsimilarity index 60%rename from .sqlx/query-d42e3270642ca0b9eff6b5807f600c24fd26d50d612a66edf0738c62d15680ba.jsonrename to .sqlx/query-38b755d094bcfc511934bfc9b1e77fff2b01630b80dba6749f70884ff18f6afd.jsonindex 563aad9bf4..935e0bbddd 100644--- a/.sqlx/query-d42e3270642ca0b9eff6b5807f600c24fd26d50d612a66edf0738c62d15680ba.json+++ b/.sqlx/query-38b755d094bcfc511934bfc9b1e77fff2b01630b80dba6749f70884ff18f6afd.json@@ -1,6 +1,6 @@{"db_name": "PostgreSQL",- "query": "\n SELECT *\n FROM\n frecency_events\n WHERE was_processed = false\n LIMIT $1\n ",+ "query": "\n UPDATE frecency_events\n SET was_processed = true\n WHERE id IN (\n SELECT id\n FROM frecency_events\n WHERE was_processed = false\n ORDER BY id\n LIMIT $1\n FOR UPDATE SKIP LOCKED\n )\n RETURNING\n id,\n user_id,\n entity_type,\n event_type,\n timestamp,\n connection_id,\n entity_id,\n was_processed\n ","describe": {"columns": [{@@ -60,5 +60,5 @@false]},- "hash": "d42e3270642ca0b9eff6b5807f600c24fd26d50d612a66edf0738c62d15680ba"+ "hash": "38b755d094bcfc511934bfc9b1e77fff2b01630b80dba6749f70884ff18f6afd"}diff --git a/.sqlx/query-5a98249c254d39fc8b136704a228f00783be5a8435a8aa4c3c30f08b4826d237.json b/.sqlx/query-5a98249c254d39fc8b136704a228f00783be5a8435a8aa4c3c30f08b4826d237.jsonnew file mode 100644index 0000000000..55106dace5--- /dev/null+++ b/.sqlx/query-5a98249c254d39fc8b136704a228f00783be5a8435a8aa4c3c30f08b4826d237.json@@ -0,0 +1,20 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT pg_advisory_unlock_all()",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "pg_advisory_unlock_all",+ "type_info": "Void"+ }+ ],+ "parameters": {+ "Left": []+ },+ "nullable": [+ null+ ]+ },+ "hash": "5a98249c254d39fc8b136704a228f00783be5a8435a8aa4c3c30f08b4826d237"+}diff --git a/.sqlx/query-96724ea1050e71438f7b892254514774f829b37d69f87286bd192af9cf702ac4.json b/.sqlx/query-96724ea1050e71438f7b892254514774f829b37d69f87286bd192af9cf702ac4.jsonnew file mode 100644index 0000000000..7f99c8770a--- /dev/null+++ b/.sqlx/query-96724ea1050e71438f7b892254514774f829b37d69f87286bd192af9cf702ac4.json@@ -0,0 +1,22 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT pg_try_advisory_lock($1)",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "pg_try_advisory_lock",+ "type_info": "Bool"+ }+ ],+ "parameters": {+ "Left": [+ "Int8"+ ]+ },+ "nullable": [+ null+ ]+ },+ "hash": "96724ea1050e71438f7b892254514774f829b37d69f87286bd192af9cf702ac4"+}diff --git a/crates/frecency/src/domain/services.rs b/crates/frecency/src/domain/services.rsindex 69d2becc3a..cd73ba40ac 100644--- a/crates/frecency/src/domain/services.rs+++ b/crates/frecency/src/domain/services.rs@@ -178,6 +178,18 @@ whereasync fn append_events_to_aggregate(&self) -> Result<EventAggregationStats, anyhow::Error> {let events: Vec<_> = self.event_storage.get_unprocessed_events().await?;+ // An idle queue is the steady state, not an edge case, and it has to+ // end the pass rather than fall through: `get_aggregates_for_users_entities`+ // builds its filter with `QueryBuilder::push_tuples`, which emits a bare+ // `IN ()` for an empty batch and fails as a syntax error. That error+ // propagates out of here before `mark_processed` runs, so the pass would+ // leave the aggregation lock held until the next tick cleaned it up --+ // once per poll, on every replica, whenever there is nothing to do.+ if events.is_empty() {+ self.event_storage.mark_processed(events).await?;+ return Ok(EventAggregationStats::default());+ }+let event_count = events.len();let ids: Result<Vec<_>, _> = events.iter().map(AggregateId::from_event_record).collect();@@ -262,3 +274,108 @@ whereOk(FrecencyPageResponse::new(res))}}++#[cfg(test)]+mod pull_aggregator_tests {+ use super::*;+ use crate::domain::ports::UnprocessedEventsRepo;+ use std::sync::atomic::{AtomicUsize, Ordering};++ #[derive(Default)]+ struct Calls {+ get_unprocessed: AtomicUsize,+ get_aggregates: AtomicUsize,+ set_aggregates: AtomicUsize,+ mark_processed: AtomicUsize,+ }++ /// Repo that always reports an idle queue, and records which methods the+ /// service reached.+ #[derive(Default)]+ struct IdleRepo {+ calls: Arc<Calls>,+ }++ impl UnprocessedEventsRepo for IdleRepo {+ type Err = anyhow::Error;+ type EventId = i64;++ async fn get_unprocessed_events(+ &self,+ ) -> Result<Vec<EventRecordWithId<'static, i64>>, Self::Err> {+ self.calls.get_unprocessed.fetch_add(1, Ordering::SeqCst);+ Ok(Vec::new())+ }++ async fn mark_processed<'a>(+ &self,+ _event: Vec<EventRecordWithId<'a, i64>>,+ ) -> Result<(), Self::Err> {+ self.calls.mark_processed.fetch_add(1, Ordering::SeqCst);+ Ok(())+ }++ async fn get_aggregates_for_users_entities(+ &self,+ _aggregates: Vec<AggregateId<'_>>,+ ) -> Result<Vec<AggregateFrecency>, Self::Err> {+ self.calls.get_aggregates.fetch_add(1, Ordering::SeqCst);+ // The real repo would render `IN ()` here and fail; this stands in+ // for that so the test fails loudly if the short-circuit is lost.+ Err(anyhow::anyhow!(+ "get_aggregates_for_users_entities must not be reached for an empty batch"+ ))+ }++ async fn set_aggregates(+ &self,+ _aggregates: Vec<AggregateFrecency>,+ ) -> Result<(), Self::Err> {+ self.calls.set_aggregates.fetch_add(1, Ordering::SeqCst);+ Ok(())+ }+ }++ struct FixedTime;+ impl TimeGetter for FixedTime {+ fn now(&self) -> chrono::DateTime<chrono::Utc> {+ chrono::Utc::now()+ }+ }++ /// An idle queue must end the pass cleanly: no aggregate lookup (which+ /// would render `IN ()` and fail), `mark_processed` still called so the+ /// aggregation lock is released, and zeroed stats returned.+ #[tokio::test]+ async fn an_empty_batch_short_circuits_and_still_releases_the_lock() {+ let calls = Arc::new(Calls::default());+ let service = PullAggregatorImpl::new(+ IdleRepo {+ calls: Arc::clone(&calls),+ },+ FixedTime,+ );++ let stats = service+ .append_events_to_aggregate()+ .await+ .expect("an idle queue is not an error");++ assert_eq!(stats.event_count, 0);+ assert_eq!(stats.existing_aggregate_count, 0);+ assert_eq!(stats.new_aggregate_count, 0);++ assert_eq!(calls.get_unprocessed.load(Ordering::SeqCst), 1);+ assert_eq!(+ calls.get_aggregates.load(Ordering::SeqCst),+ 0,+ "empty batch must not reach the aggregate lookup"+ );+ assert_eq!(calls.set_aggregates.load(Ordering::SeqCst), 0);+ assert_eq!(+ calls.mark_processed.load(Ordering::SeqCst),+ 1,+ "the pass must still be closed out so the lock is released"+ );+ }+}diff --git a/crates/frecency/src/outbound/postgres.rs b/crates/frecency/src/outbound/postgres.rsindex 706391bbf7..27c871fd41 100644--- a/crates/frecency/src/outbound/postgres.rs+++ b/crates/frecency/src/outbound/postgres.rs@@ -10,7 +10,7 @@ use chrono::{DateTime, Utc};use item_filters::ast::EntityFilterAst;use macro_user_id::{cowlike::CowLike, error::ParseErr, user_id::MacroUserIdStr};use model_entity::{Entity, EntityType};-use sqlx::{PgPool, Postgres, QueryBuilder, Row, Transaction, prelude::FromRow};+use sqlx::{PgPool, Postgres, QueryBuilder, Row, pool::PoolConnection, prelude::FromRow};use std::{borrow::Cow, collections::VecDeque, str::FromStr};use thiserror::Error;@@ -387,13 +387,20 @@ impl AggregateFrecencyStorage for FrecencyPgStorage {}/// concrete struct which implements [UnprocessedEventsRepo]-/// This uses transactions to ensure that the act of processing events remains atomic+///+/// Batches are claimed and committed up front, so no transaction stays open+/// across the aggregation work. Pollers are still serialised, by a+/// session-scoped advisory lock held on `aggregate_lock` for the length of one+/// pass — see [`UnprocessedEventsRepo::get_unprocessed_events`] for why that is+/// load-bearing.pub struct FrecencyPgProcessor {pool: PgPool,- tx: tokio::sync::Mutex<Option<Transaction<'static, Postgres>>>,+ /// Connection holding the session-level aggregation lock. Occupied only+ /// between a successful claim and its matching `mark_processed`.+ aggregate_lock: tokio::sync::Mutex<Option<PoolConnection<Postgres>>>,}-// Define a unique lock ID for frecency polling+// Define a unique lock ID for frecency aggregationconst FRECENCY_POLLER_LOCK_ID: i64 = 999_999_001;impl FrecencyPgProcessor {@@ -401,7 +408,7 @@ impl FrecencyPgProcessor {pub fn new(pool: PgPool) -> Self {FrecencyPgProcessor {pool,- tx: tokio::sync::Mutex::new(None),+ aggregate_lock: tokio::sync::Mutex::new(None),}}}@@ -411,13 +418,10 @@ type ExistingEventRow = EventRow<'static, i64>;/// the types of errors that can occur with the poller#[derive(Debug, Error)]pub enum PollerErr {- /// there was an issue accessing the db tx- #[error("Expected to be inside a transaction, but no transaction was present")]- TxErr,- /// failed to acquire advisory lock on db- #[error("another poller currently holds the frecency lock")]+ /// another poller currently holds the aggregation lock+ #[error("another poller currently holds the frecency aggregation lock")]DbLockErr,- /// failed to acquire mutex lock+ /// failed to acquire the in-process mutex guarding the lock connection#[error(transparent)]MutexErr(#[from] tokio::sync::TryLockError),/// sqlx database error@@ -440,90 +444,133 @@ impl UnprocessedEventsRepo for FrecencyPgProcessor {type Err = PollerErr;type EventId = i64;+ /// Take the aggregation lock, then claim a batch of unprocessed events,+ /// marking them processed and committing before returning.+ ///+ /// This previously opened a transaction, took `pg_advisory_xact_lock`, ran+ /// the read, and then stashed the *live* transaction so `mark_processed`+ /// could commit it after aggregation. That left the connection+ /// `idle in transaction` for the whole aggregation pass (20-60s in prod),+ /// which pinned `xmin`, blocked vacuum on `frecency_events`, and held a+ /// pool connection the entire time.+ ///+ /// Claiming up front makes the *transaction* one statement long. The lock+ /// still has to outlive the claim, though, and dropping it would be a+ /// correctness bug rather than a throughput trade: aggregation is a+ /// read-modify-write of `frecency_aggregates`, and disjoint event batches+ /// routinely map onto the same `(user_id, entity_type, entity_id)` key.+ /// Two pollers would each read the same base row, merge their own events+ /// onto it in memory, and then blind-overwrite it in `set_aggregates`+ /// (`SET event_count = EXCLUDED.event_count, ...`) — last writer wins and+ /// the other batch's contribution is silently dropped, permanently, since+ /// its events are already marked processed. `connection_gateway` runs+ /// three replicas in prod, so this is a steady-state race, not a rare one.+ ///+ /// So the lock is now session-scoped instead of transaction-scoped: held+ /// on a dedicated pooled connection across the whole pass and released by+ /// `mark_processed`. Same mutual exclusion as before, without an open+ /// transaction. `FOR UPDATE SKIP LOCKED` on the claim is belt and braces —+ /// it keeps the claim itself correct if this lock is ever narrowed.+ ///+ /// Delivery is at-most-once: a crash between the claim and+ /// `set_aggregates` drops that batch's contribution. Frecency is a ranking+ /// heuristic rather than a ledger, so a lost batch degrades ordering+ /// slightly rather than corrupting state. If that stops being acceptable,+ /// the fix is a `claimed_at` column reset by a sweeper, not a return to+ /// holding the transaction open.+ ///+ /// `ORDER BY id` gives `idx_frecency_events_unprocessed_id` something to+ /// walk; the columns are listed explicitly so adding a column to the table+ /// cannot silently widen this read.async fn get_unprocessed_events(&self,) -> Result<Vec<EventRecordWithId<'static, i64>>, Self::Err> {- // Clean up any stale transaction from a previous failed processing run.- // Without this, a failed run leaves its tx (and advisory lock) in the mutex,- // causing every subsequent poll to fail on pg_try_advisory_xact_lock.- {- let mut guard = self.tx.try_lock()?;- if let Some(stale_tx) = guard.take() {- tracing::warn!(- "rolling back stale frecency transaction from a previous failed run"- );- let _ = stale_tx.rollback().await.inspect_err(|e| {- tracing::error!(error = ?e, "failed to rollback stale frecency transaction");+ let mut guard = self.aggregate_lock.try_lock()?;++ // A pass that failed between the claim and mark_processed leaves its+ // connection here, still holding the lock. Release it explicitly+ // rather than trusting pool reset to do it.+ if let Some(mut stale) = guard.take() {+ tracing::warn!("releasing frecency aggregation lock left by a failed pass");+ let _ = sqlx::query!(r#"SELECT pg_advisory_unlock_all()"#)+ .execute(&mut *stale)+ .await+ .inspect_err(|e| {+ tracing::error!(error = ?e, "failed to release stale frecency lock");});- }}- let mut tx = self.pool.begin().await?;- let true = sqlx::query_scalar!(- r#"- SELECT pg_try_advisory_xact_lock($1)- "#,+ let mut conn = self.pool.acquire().await?;+ let locked = sqlx::query_scalar!(+ r#"SELECT pg_try_advisory_lock($1)"#,FRECENCY_POLLER_LOCK_ID)- .fetch_one(&mut *tx)+ .fetch_one(&mut *conn).await?- .unwrap_or(false) else {+ .unwrap_or(false);++ if !locked {+ // Nothing was acquired, so returning the connection is enough.return Err(PollerErr::DbLockErr);- };+ }- let res: Vec<ExistingEventRow> = sqlx::query_as!(+ // Stash before running anything that can fail, so a mid-pass error+ // leaves the lock reachable for the cleanup above.+ *guard = Some(conn);+ let conn = guard.as_mut().expect("connection was just stashed");++ let claimed: Vec<ExistingEventRow> = sqlx::query_as!(ExistingEventRow,r#"- SELECT *- FROM- frecency_events- WHERE was_processed = false- LIMIT $1+ UPDATE frecency_events+ SET was_processed = true+ WHERE id IN (+ SELECT id+ FROM frecency_events+ WHERE was_processed = false+ ORDER BY id+ LIMIT $1+ FOR UPDATE SKIP LOCKED+ )+ RETURNING+ id,+ user_id,+ entity_type,+ event_type,+ timestamp,+ connection_id,+ entity_id,+ was_processed"#,FETCH_LIMIT as i64)- .fetch_all(&mut *tx)+ .fetch_all(&mut **conn).await?;- let res: Result<Vec<_>, _> = res.into_iter().map(|r| r.into_event_record()).collect();-- let mut guard = self.tx.try_lock()?;- *guard = Some(tx);- res.map_err(PollerErr::from)+ claimed+ .into_iter()+ .map(|r| r.into_event_record())+ .collect::<Result<Vec<_>, _>>()+ .map_err(PollerErr::from)}+ /// Releases the aggregation lock taken by+ /// [`UnprocessedEventsRepo::get_unprocessed_events`].+ ///+ /// The events themselves were already marked processed by the claim; what+ /// still has to happen at the end of a pass is letting the next poller in,+ /// which must not happen until this pass's `set_aggregates` has landed.async fn mark_processed<'a>(&self,- event: Vec<EventRecordWithId<'a, i64>>,+ _event: Vec<EventRecordWithId<'a, i64>>,) -> Result<(), Self::Err> {- let mut guard = self.tx.try_lock()?;-- let mut tx = guard.take().ok_or(PollerErr::TxErr)?;-- if event.is_empty() {- tx.commit().await?;+ let mut guard = self.aggregate_lock.try_lock()?;+ let Some(mut conn) = guard.take() else {return Ok(());- }-- let mut query_builder = QueryBuilder::<Postgres>::new(- r#"- UPDATE frecency_events- SET was_processed = true- WHERE id IN (- "#,- );-- let mut separated = query_builder.separated(", ");- for e in event {- separated.push_bind(e.id);- }- separated.push_unseparated(")");-- let query = query_builder.build();-- query.execute(&mut *tx).await?;-- tx.commit().await?;+ };+ sqlx::query_scalar!(r#"SELECT pg_advisory_unlock($1)"#, FRECENCY_POLLER_LOCK_ID)+ .fetch_one(&mut *conn)+ .await?;Ok(())}@@ -531,8 +578,12 @@ impl UnprocessedEventsRepo for FrecencyPgProcessor {&self,aggregates: Vec<AggregateId<'_>>,) -> Result<Vec<AggregateFrecency>, Self::Err> {- let mut guard = self.tx.try_lock()?;- let tx = guard.as_deref_mut().ok_or(PollerErr::TxErr)?;+ // `push_tuples` renders an empty iterator as `IN ()`, which Postgres+ // rejects as a syntax error. Callers should not reach here with an+ // empty batch, but nothing left to look up has an obvious answer.+ if aggregates.is_empty() {+ return Ok(Vec::new());+ }let mut query_builder = QueryBuilder::<Postgres>::new(r#"@@ -559,7 +610,7 @@ impl UnprocessedEventsRepo for FrecencyPgProcessor {let query = query_builder.build_query_as::<AggregateRow>();- let output = query.fetch_all(tx).await?;+ let output = query.fetch_all(&self.pool).await?;let out: Result<Vec<_>, _> = output.into_iter()@@ -570,9 +621,6 @@ impl UnprocessedEventsRepo for FrecencyPgProcessor {}async fn set_aggregates(&self, aggregates: Vec<AggregateFrecency>) -> Result<(), Self::Err> {- let mut guard = self.tx.try_lock()?;- let tx = guard.as_deref_mut().ok_or(PollerErr::TxErr)?;-if aggregates.is_empty() {return Ok(());}@@ -617,7 +665,7 @@ impl UnprocessedEventsRepo for FrecencyPgProcessor {);let query = query_builder.build();- query.execute(tx).await?;+ query.execute(&self.pool).await?;Ok(())}diff --git a/crates/frecency/src/outbound/postgres/tests.rs b/crates/frecency/src/outbound/postgres/tests.rsindex a229d580dc..2e21493121 100644--- a/crates/frecency/src/outbound/postgres/tests.rs+++ b/crates/frecency/src/outbound/postgres/tests.rs@@ -523,7 +523,7 @@ async fn test_processor_mark_processed(pool: PgPool) {event_ids.push(id);}- // Get unprocessed events (starts a transaction)+ // Claims the batch and commits; the rows are already marked processed.let unprocessed = processor.get_unprocessed_events().await.unwrap();// Filter to just our test events@@ -534,7 +534,7 @@ async fn test_processor_mark_processed(pool: PgPool) {assert_eq!(to_mark.len(), 2);- // Mark them as processed (commits the transaction)+ // Releases the aggregation lock; the rows were marked by the claim.processor.mark_processed(to_mark).await.unwrap();// Verify they are marked as processed in the database@@ -794,7 +794,7 @@ async fn test_processor_set_aggregates_empty_list(pool: PgPool) {}#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]-async fn test_processor_transaction_lifecycle(pool: PgPool) {+async fn test_processor_claim_aggregate_lifecycle(pool: PgPool) {let processor = FrecencyPgProcessor::new(pool.clone());let test_user_id = MacroUserIdStr::parse_from_str("macro|[email protected]").unwrap();@@ -819,7 +819,7 @@ async fn test_processor_transaction_lifecycle(pool: PgPool) {.await.unwrap();- // Get unprocessed events (starts transaction)+ // Claims the batch and commits.let events = processor.get_unprocessed_events().await.unwrap();let test_event = events.into_iter().find(|e| e.id == event_id);assert!(test_event.is_some());@@ -838,10 +838,10 @@ async fn test_processor_transaction_lifecycle(pool: PgPool) {},};- // Set the aggregate (uses same transaction)+ // Written on its own statement, after the claim committed.processor.set_aggregates(vec![aggregate]).await.unwrap();- // Mark as processed (commits the same transaction)+ // Releases the aggregation lock taken by the claim.processor.mark_processed(vec![test_event.unwrap()]).await@@ -875,62 +875,62 @@ async fn test_processor_transaction_lifecycle(pool: PgPool) {assert!(was_processed);}+/// An idle queue must not strand the aggregation lock.+///+/// The pass still takes the lock before discovering there is nothing to claim,+/// so it has to be closed out by mark_processed like any other. Otherwise the+/// lock would sit held until the next tick cleaned it up -- once per poll, on+/// every replica, for as long as the queue stays empty.#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]-async fn it_cannot_be_read_concurrently(pool: PgPool) {- let test_user_id = "test_processor_tx_lifecycle";- // Insert an unprocessed event- let event_id = sqlx::query_scalar!(- r#"- INSERT INTO frecency_events (- user_id, entity_type, event_type, timestamp,- connection_id, entity_id, was_processed- )- VALUES ($1, $2, $3, $4, $5, $6, false)- RETURNING id- "#,- test_user_id,- "document",- "open",- Utc::now(),- "conn_tx",- "doc_tx"- )- .fetch_one(&pool)- .await- .unwrap();-+async fn an_empty_batch_still_releases_the_aggregation_lock(pool: PgPool) {let first = FrecencyPgProcessor::new(pool.clone());let second = FrecencyPgProcessor::new(pool);+ // No events inserted: the claim finds nothing.let events = first.get_unprocessed_events().await.unwrap();- assert_eq!(events.len(), 1);- assert_eq!(events.first().unwrap().id, event_id);+ assert!(events.is_empty());- // try to read from a second processor- // this fails because the first processor still holds the lock+ // The lock was taken regardless, so the second poller is still shut out.let err = second.get_unprocessed_events().await.unwrap_err();- assert!(matches!(err, PollerErr::DbLockErr));+ assert!(matches!(err, PollerErr::DbLockErr), "got {err:?}");- // finish the tx+ // Closing out the empty pass releases it.first.mark_processed(Vec::new()).await.unwrap();-- // now second can read because the transaction has finishedlet res = second.get_unprocessed_events().await.unwrap();- assert_eq!(res.len(), 1);- assert_eq!(res.first().unwrap().id, event_id);+ assert!(res.is_empty());}+/// The aggregate lookup renders its filter with `push_tuples`, which emits a+/// bare `IN ()` for an empty batch. Callers should short-circuit before here,+/// but the method must not turn "nothing to look up" into a syntax error.#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]-async fn test_stale_transaction_is_cleaned_up(pool: PgPool) {- let test_user_id = "test_stale_tx_cleanup";+async fn empty_aggregate_lookup_returns_empty_rather_than_failing(pool: PgPool) {+ let processor = FrecencyPgProcessor::new(pool);+ let res = processor+ .get_aggregates_for_users_entities(Vec::new())+ .await+ .expect("an empty lookup must not be a syntax error");+ assert!(res.is_empty());+}- sqlx::query!(+/// Only one poller may aggregate at a time.+///+/// Disjoint event batches can still merge into the same aggregate key, and+/// set_aggregates blind-overwrites that row, so two concurrent passes would+/// lose one of their contributions. The second poller must be turned away+/// while the first holds the lock, and admitted once it releases.+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn a_second_poller_is_locked_out_until_the_first_finishes(pool: PgPool) {+ let test_user_id = "test_processor_claim_is_exclusive";+ // Insert an unprocessed event+ let event_id = sqlx::query_scalar!(r#"INSERT INTO frecency_events (user_id, entity_type, event_type, timestamp,connection_id, entity_id, was_processed)VALUES ($1, $2, $3, $4, $5, $6, false)+ RETURNING id"#,test_user_id,"document",@@ -939,24 +939,41 @@ async fn test_stale_transaction_is_cleaned_up(pool: PgPool) {"conn_tx","doc_tx")- .execute(&pool)+ .fetch_one(&pool).await.unwrap();- let processor = FrecencyPgProcessor::new(pool);+ let pool_handle = pool.clone();+ let first = FrecencyPgProcessor::new(pool.clone());+ let second = FrecencyPgProcessor::new(pool);- // First call succeeds, starts a transaction and stores it in the mutex- let events = processor.get_unprocessed_events().await.unwrap();+ let events = first.get_unprocessed_events().await.unwrap();assert_eq!(events.len(), 1);+ assert_eq!(events.first().unwrap().id, event_id);- // Simulate a processing failure by NOT calling mark_processed.- // The next call to get_unprocessed_events should clean up the stale tx- // and successfully start a new processing cycle.- let events = processor.get_unprocessed_events().await.unwrap();- assert_eq!(events.len(), 1);+ // The first poller holds the aggregation lock across its whole pass, so+ // the second must not be allowed to read-modify-write concurrently.+ let err = second.get_unprocessed_events().await.unwrap_err();+ assert!(+ matches!(err, PollerErr::DbLockErr),+ "second poller should be locked out, got {err:?}"+ );- // Verify we can still complete the full lifecycle- processor.mark_processed(Vec::new()).await.unwrap();+ // The claim itself already marked the row, independent of the lock.+ let was_processed = sqlx::query_scalar!(+ r#"SELECT was_processed AS "was_processed!" FROM frecency_events WHERE id = $1"#,+ event_id+ )+ .fetch_one(&pool_handle)+ .await+ .unwrap();+ assert!(was_processed);++ // Ending the first pass releases the lock and lets the second in; there+ // is simply nothing left for it to claim.+ first.mark_processed(Vec::new()).await.unwrap();+ let res = second.get_unprocessed_events().await.unwrap();+ assert!(res.is_empty());}/// Tests that mixing supported (chat_id) and unsupported (role) filtersdiff --git a/crates/macro_db_client/migrations/20260910144342_frecency_events_unprocessed_id_index.sql b/crates/macro_db_client/migrations/20260910144342_frecency_events_unprocessed_id_index.sqlnew file mode 100644index 0000000000..24f8d6eeea--- /dev/null+++ b/crates/macro_db_client/migrations/20260910144342_frecency_events_unprocessed_id_index.sql@@ -0,0 +1,12 @@+-- no-transaction+-- Replaces idx_frecency_events_unprocessed, which keys on was_processed under+-- the predicate `was_processed = false`. Every entry in that index therefore+-- holds the same key value, so it offers neither ordering nor selectivity and+-- the planner never chose it -- Datadog reported it as unused.+-- The poller claims batches with `ORDER BY id ... FOR UPDATE SKIP LOCKED`, so+-- keying on id under the same predicate gives that scan something to walk.+-- Added before the old index is dropped (20260910144343) so no deploy window+-- runs without one.+-- Single statement: see 20260910144339 for the CONCURRENTLY constraint.+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_frecency_events_unprocessed_id+ ON frecency_events (id) WHERE was_processed = false;diff --git a/crates/macro_db_client/migrations/20260910144343_drop_frecency_events_unprocessed_index.sql b/crates/macro_db_client/migrations/20260910144343_drop_frecency_events_unprocessed_index.sqlnew file mode 100644index 0000000000..f5c5140d45--- /dev/null+++ b/crates/macro_db_client/migrations/20260910144343_drop_frecency_events_unprocessed_index.sql@@ -0,0 +1,5 @@+-- no-transaction+-- Superseded by idx_frecency_events_unprocessed_id (20260910144342), which+-- indexes the same rows on a column that can actually drive a scan.+-- Single statement: see 20260910144339 for the CONCURRENTLY constraint.+DROP INDEX CONCURRENTLY IF EXISTS idx_frecency_events_unprocessed;