Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

perf(frecency): claim event batches instead of holding the read open (#6332)

Will Hutchinson7db736e48b70
diff --git a/.sqlx/query-89004da6bcdd66bccd56ed64402d82dc4fb489cccaf5ca647c5a9403fa760ce9.json b/.sqlx/query-0115c52b6c77a377e6585308ba0df3daaaf7d30a19a37b28abcae7efbe9b4ca7.json
similarity index 54%
rename from .sqlx/query-89004da6bcdd66bccd56ed64402d82dc4fb489cccaf5ca647c5a9403fa760ce9.json
rename to .sqlx/query-0115c52b6c77a377e6585308ba0df3daaaf7d30a19a37b28abcae7efbe9b4ca7.json
index a989abc850..6e5e69ddc6 100644
--- a/.sqlx/query-89004da6bcdd66bccd56ed64402d82dc4fb489cccaf5ca647c5a9403fa760ce9.json
+++ b/.sqlx/query-0115c52b6c77a377e6585308ba0df3daaaf7d30a19a37b28abcae7efbe9b4ca7.json
@@ -1,11 +1,11 @@
{
"db_name": "PostgreSQL",
- "query": "\n SELECT pg_try_advisory_xact_lock($1)\n ",
+ "query": "SELECT pg_advisory_unlock($1)",
"describe": {
"columns": [
{
"ordinal": 0,
- "name": "pg_try_advisory_xact_lock",
+ "name": "pg_advisory_unlock",
"type_info": "Bool"
}
],
@@ -18,5 +18,5 @@
null
]
},
- "hash": "89004da6bcdd66bccd56ed64402d82dc4fb489cccaf5ca647c5a9403fa760ce9"
+ "hash": "0115c52b6c77a377e6585308ba0df3daaaf7d30a19a37b28abcae7efbe9b4ca7"
}
diff --git a/.sqlx/query-d42e3270642ca0b9eff6b5807f600c24fd26d50d612a66edf0738c62d15680ba.json b/.sqlx/query-38b755d094bcfc511934bfc9b1e77fff2b01630b80dba6749f70884ff18f6afd.json
similarity index 60%
rename from .sqlx/query-d42e3270642ca0b9eff6b5807f600c24fd26d50d612a66edf0738c62d15680ba.json
rename to .sqlx/query-38b755d094bcfc511934bfc9b1e77fff2b01630b80dba6749f70884ff18f6afd.json
index 563aad9bf4..935e0bbddd 100644
--- a/.sqlx/query-d42e3270642ca0b9eff6b5807f600c24fd26d50d612a66edf0738c62d15680ba.json
+++ b/.sqlx/query-38b755d094bcfc511934bfc9b1e77fff2b01630b80dba6749f70884ff18f6afd.json
@@ -1,6 +1,6 @@
{
"db_name": "PostgreSQL",
- "query": "\n SELECT *\n FROM\n frecency_events\n WHERE was_processed = false\n LIMIT $1\n ",
+ "query": "\n UPDATE frecency_events\n SET was_processed = true\n WHERE id IN (\n SELECT id\n FROM frecency_events\n WHERE was_processed = false\n ORDER BY id\n LIMIT $1\n FOR UPDATE SKIP LOCKED\n )\n RETURNING\n id,\n user_id,\n entity_type,\n event_type,\n timestamp,\n connection_id,\n entity_id,\n was_processed\n ",
"describe": {
"columns": [
{
@@ -60,5 +60,5 @@
false
]
},
- "hash": "d42e3270642ca0b9eff6b5807f600c24fd26d50d612a66edf0738c62d15680ba"
+ "hash": "38b755d094bcfc511934bfc9b1e77fff2b01630b80dba6749f70884ff18f6afd"
}
diff --git a/.sqlx/query-5a98249c254d39fc8b136704a228f00783be5a8435a8aa4c3c30f08b4826d237.json b/.sqlx/query-5a98249c254d39fc8b136704a228f00783be5a8435a8aa4c3c30f08b4826d237.json
new file mode 100644
index 0000000000..55106dace5
--- /dev/null
+++ b/.sqlx/query-5a98249c254d39fc8b136704a228f00783be5a8435a8aa4c3c30f08b4826d237.json
@@ -0,0 +1,20 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT pg_advisory_unlock_all()",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "pg_advisory_unlock_all",
+ "type_info": "Void"
+ }
+ ],
+ "parameters": {
+ "Left": []
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "5a98249c254d39fc8b136704a228f00783be5a8435a8aa4c3c30f08b4826d237"
+}
diff --git a/.sqlx/query-96724ea1050e71438f7b892254514774f829b37d69f87286bd192af9cf702ac4.json b/.sqlx/query-96724ea1050e71438f7b892254514774f829b37d69f87286bd192af9cf702ac4.json
new file mode 100644
index 0000000000..7f99c8770a
--- /dev/null
+++ b/.sqlx/query-96724ea1050e71438f7b892254514774f829b37d69f87286bd192af9cf702ac4.json
@@ -0,0 +1,22 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "SELECT pg_try_advisory_lock($1)",
+ "describe": {
+ "columns": [
+ {
+ "ordinal": 0,
+ "name": "pg_try_advisory_lock",
+ "type_info": "Bool"
+ }
+ ],
+ "parameters": {
+ "Left": [
+ "Int8"
+ ]
+ },
+ "nullable": [
+ null
+ ]
+ },
+ "hash": "96724ea1050e71438f7b892254514774f829b37d69f87286bd192af9cf702ac4"
+}
diff --git a/crates/frecency/src/domain/services.rs b/crates/frecency/src/domain/services.rs
index 69d2becc3a..cd73ba40ac 100644
--- a/crates/frecency/src/domain/services.rs
+++ b/crates/frecency/src/domain/services.rs
@@ -178,6 +178,18 @@ where
async fn append_events_to_aggregate(&self) -> Result<EventAggregationStats, anyhow::Error> {
let events: Vec<_> = self.event_storage.get_unprocessed_events().await?;
+ // An idle queue is the steady state, not an edge case, and it has to
+ // end the pass rather than fall through: `get_aggregates_for_users_entities`
+ // builds its filter with `QueryBuilder::push_tuples`, which emits a bare
+ // `IN ()` for an empty batch and fails as a syntax error. That error
+ // propagates out of here before `mark_processed` runs, so the pass would
+ // leave the aggregation lock held until the next tick cleaned it up --
+ // once per poll, on every replica, whenever there is nothing to do.
+ if events.is_empty() {
+ self.event_storage.mark_processed(events).await?;
+ return Ok(EventAggregationStats::default());
+ }
+
let event_count = events.len();
let ids: Result<Vec<_>, _> = events.iter().map(AggregateId::from_event_record).collect();
@@ -262,3 +274,108 @@ where
Ok(FrecencyPageResponse::new(res))
}
}
+
+#[cfg(test)]
+mod pull_aggregator_tests {
+ use super::*;
+ use crate::domain::ports::UnprocessedEventsRepo;
+ use std::sync::atomic::{AtomicUsize, Ordering};
+
+ #[derive(Default)]
+ struct Calls {
+ get_unprocessed: AtomicUsize,
+ get_aggregates: AtomicUsize,
+ set_aggregates: AtomicUsize,
+ mark_processed: AtomicUsize,
+ }
+
+ /// Repo that always reports an idle queue, and records which methods the
+ /// service reached.
+ #[derive(Default)]
+ struct IdleRepo {
+ calls: Arc<Calls>,
+ }
+
+ impl UnprocessedEventsRepo for IdleRepo {
+ type Err = anyhow::Error;
+ type EventId = i64;
+
+ async fn get_unprocessed_events(
+ &self,
+ ) -> Result<Vec<EventRecordWithId<'static, i64>>, Self::Err> {
+ self.calls.get_unprocessed.fetch_add(1, Ordering::SeqCst);
+ Ok(Vec::new())
+ }
+
+ async fn mark_processed<'a>(
+ &self,
+ _event: Vec<EventRecordWithId<'a, i64>>,
+ ) -> Result<(), Self::Err> {
+ self.calls.mark_processed.fetch_add(1, Ordering::SeqCst);
+ Ok(())
+ }
+
+ async fn get_aggregates_for_users_entities(
+ &self,
+ _aggregates: Vec<AggregateId<'_>>,
+ ) -> Result<Vec<AggregateFrecency>, Self::Err> {
+ self.calls.get_aggregates.fetch_add(1, Ordering::SeqCst);
+ // The real repo would render `IN ()` here and fail; this stands in
+ // for that so the test fails loudly if the short-circuit is lost.
+ Err(anyhow::anyhow!(
+ "get_aggregates_for_users_entities must not be reached for an empty batch"
+ ))
+ }
+
+ async fn set_aggregates(
+ &self,
+ _aggregates: Vec<AggregateFrecency>,
+ ) -> Result<(), Self::Err> {
+ self.calls.set_aggregates.fetch_add(1, Ordering::SeqCst);
+ Ok(())
+ }
+ }
+
+ struct FixedTime;
+ impl TimeGetter for FixedTime {
+ fn now(&self) -> chrono::DateTime<chrono::Utc> {
+ chrono::Utc::now()
+ }
+ }
+
+ /// An idle queue must end the pass cleanly: no aggregate lookup (which
+ /// would render `IN ()` and fail), `mark_processed` still called so the
+ /// aggregation lock is released, and zeroed stats returned.
+ #[tokio::test]
+ async fn an_empty_batch_short_circuits_and_still_releases_the_lock() {
+ let calls = Arc::new(Calls::default());
+ let service = PullAggregatorImpl::new(
+ IdleRepo {
+ calls: Arc::clone(&calls),
+ },
+ FixedTime,
+ );
+
+ let stats = service
+ .append_events_to_aggregate()
+ .await
+ .expect("an idle queue is not an error");
+
+ assert_eq!(stats.event_count, 0);
+ assert_eq!(stats.existing_aggregate_count, 0);
+ assert_eq!(stats.new_aggregate_count, 0);
+
+ assert_eq!(calls.get_unprocessed.load(Ordering::SeqCst), 1);
+ assert_eq!(
+ calls.get_aggregates.load(Ordering::SeqCst),
+ 0,
+ "empty batch must not reach the aggregate lookup"
+ );
+ assert_eq!(calls.set_aggregates.load(Ordering::SeqCst), 0);
+ assert_eq!(
+ calls.mark_processed.load(Ordering::SeqCst),
+ 1,
+ "the pass must still be closed out so the lock is released"
+ );
+ }
+}
diff --git a/crates/frecency/src/outbound/postgres.rs b/crates/frecency/src/outbound/postgres.rs
index 706391bbf7..27c871fd41 100644
--- a/crates/frecency/src/outbound/postgres.rs
+++ b/crates/frecency/src/outbound/postgres.rs
@@ -10,7 +10,7 @@ use chrono::{DateTime, Utc};
use item_filters::ast::EntityFilterAst;
use macro_user_id::{cowlike::CowLike, error::ParseErr, user_id::MacroUserIdStr};
use model_entity::{Entity, EntityType};
-use sqlx::{PgPool, Postgres, QueryBuilder, Row, Transaction, prelude::FromRow};
+use sqlx::{PgPool, Postgres, QueryBuilder, Row, pool::PoolConnection, prelude::FromRow};
use std::{borrow::Cow, collections::VecDeque, str::FromStr};
use thiserror::Error;
@@ -387,13 +387,20 @@ impl AggregateFrecencyStorage for FrecencyPgStorage {
}
/// concrete struct which implements [UnprocessedEventsRepo]
-/// This uses transactions to ensure that the act of processing events remains atomic
+///
+/// Batches are claimed and committed up front, so no transaction stays open
+/// across the aggregation work. Pollers are still serialised, by a
+/// session-scoped advisory lock held on `aggregate_lock` for the length of one
+/// pass — see [`UnprocessedEventsRepo::get_unprocessed_events`] for why that is
+/// load-bearing.
pub struct FrecencyPgProcessor {
pool: PgPool,
- tx: tokio::sync::Mutex<Option<Transaction<'static, Postgres>>>,
+ /// Connection holding the session-level aggregation lock. Occupied only
+ /// between a successful claim and its matching `mark_processed`.
+ aggregate_lock: tokio::sync::Mutex<Option<PoolConnection<Postgres>>>,
}
-// Define a unique lock ID for frecency polling
+// Define a unique lock ID for frecency aggregation
const FRECENCY_POLLER_LOCK_ID: i64 = 999_999_001;
impl FrecencyPgProcessor {
@@ -401,7 +408,7 @@ impl FrecencyPgProcessor {
pub fn new(pool: PgPool) -> Self {
FrecencyPgProcessor {
pool,
- tx: tokio::sync::Mutex::new(None),
+ aggregate_lock: tokio::sync::Mutex::new(None),
}
}
}
@@ -411,13 +418,10 @@ type ExistingEventRow = EventRow<'static, i64>;
/// the types of errors that can occur with the poller
#[derive(Debug, Error)]
pub enum PollerErr {
- /// there was an issue accessing the db tx
- #[error("Expected to be inside a transaction, but no transaction was present")]
- TxErr,
- /// failed to acquire advisory lock on db
- #[error("another poller currently holds the frecency lock")]
+ /// another poller currently holds the aggregation lock
+ #[error("another poller currently holds the frecency aggregation lock")]
DbLockErr,
- /// failed to acquire mutex lock
+ /// failed to acquire the in-process mutex guarding the lock connection
#[error(transparent)]
MutexErr(#[from] tokio::sync::TryLockError),
/// sqlx database error
@@ -440,90 +444,133 @@ impl UnprocessedEventsRepo for FrecencyPgProcessor {
type Err = PollerErr;
type EventId = i64;
+ /// Take the aggregation lock, then claim a batch of unprocessed events,
+ /// marking them processed and committing before returning.
+ ///
+ /// This previously opened a transaction, took `pg_advisory_xact_lock`, ran
+ /// the read, and then stashed the *live* transaction so `mark_processed`
+ /// could commit it after aggregation. That left the connection
+ /// `idle in transaction` for the whole aggregation pass (20-60s in prod),
+ /// which pinned `xmin`, blocked vacuum on `frecency_events`, and held a
+ /// pool connection the entire time.
+ ///
+ /// Claiming up front makes the *transaction* one statement long. The lock
+ /// still has to outlive the claim, though, and dropping it would be a
+ /// correctness bug rather than a throughput trade: aggregation is a
+ /// read-modify-write of `frecency_aggregates`, and disjoint event batches
+ /// routinely map onto the same `(user_id, entity_type, entity_id)` key.
+ /// Two pollers would each read the same base row, merge their own events
+ /// onto it in memory, and then blind-overwrite it in `set_aggregates`
+ /// (`SET event_count = EXCLUDED.event_count, ...`) — last writer wins and
+ /// the other batch's contribution is silently dropped, permanently, since
+ /// its events are already marked processed. `connection_gateway` runs
+ /// three replicas in prod, so this is a steady-state race, not a rare one.
+ ///
+ /// So the lock is now session-scoped instead of transaction-scoped: held
+ /// on a dedicated pooled connection across the whole pass and released by
+ /// `mark_processed`. Same mutual exclusion as before, without an open
+ /// transaction. `FOR UPDATE SKIP LOCKED` on the claim is belt and braces —
+ /// it keeps the claim itself correct if this lock is ever narrowed.
+ ///
+ /// Delivery is at-most-once: a crash between the claim and
+ /// `set_aggregates` drops that batch's contribution. Frecency is a ranking
+ /// heuristic rather than a ledger, so a lost batch degrades ordering
+ /// slightly rather than corrupting state. If that stops being acceptable,
+ /// the fix is a `claimed_at` column reset by a sweeper, not a return to
+ /// holding the transaction open.
+ ///
+ /// `ORDER BY id` gives `idx_frecency_events_unprocessed_id` something to
+ /// walk; the columns are listed explicitly so adding a column to the table
+ /// cannot silently widen this read.
async fn get_unprocessed_events(
&self,
) -> Result<Vec<EventRecordWithId<'static, i64>>, Self::Err> {
- // Clean up any stale transaction from a previous failed processing run.
- // Without this, a failed run leaves its tx (and advisory lock) in the mutex,
- // causing every subsequent poll to fail on pg_try_advisory_xact_lock.
- {
- let mut guard = self.tx.try_lock()?;
- if let Some(stale_tx) = guard.take() {
- tracing::warn!(
- "rolling back stale frecency transaction from a previous failed run"
- );
- let _ = stale_tx.rollback().await.inspect_err(|e| {
- tracing::error!(error = ?e, "failed to rollback stale frecency transaction");
+ let mut guard = self.aggregate_lock.try_lock()?;
+
+ // A pass that failed between the claim and mark_processed leaves its
+ // connection here, still holding the lock. Release it explicitly
+ // rather than trusting pool reset to do it.
+ if let Some(mut stale) = guard.take() {
+ tracing::warn!("releasing frecency aggregation lock left by a failed pass");
+ let _ = sqlx::query!(r#"SELECT pg_advisory_unlock_all()"#)
+ .execute(&mut *stale)
+ .await
+ .inspect_err(|e| {
+ tracing::error!(error = ?e, "failed to release stale frecency lock");
});
- }
}
- let mut tx = self.pool.begin().await?;
- let true = sqlx::query_scalar!(
- r#"
- SELECT pg_try_advisory_xact_lock($1)
- "#,
+ let mut conn = self.pool.acquire().await?;
+ let locked = sqlx::query_scalar!(
+ r#"SELECT pg_try_advisory_lock($1)"#,
FRECENCY_POLLER_LOCK_ID
)
- .fetch_one(&mut *tx)
+ .fetch_one(&mut *conn)
.await?
- .unwrap_or(false) else {
+ .unwrap_or(false);
+
+ if !locked {
+ // Nothing was acquired, so returning the connection is enough.
return Err(PollerErr::DbLockErr);
- };
+ }
- let res: Vec<ExistingEventRow> = sqlx::query_as!(
+ // Stash before running anything that can fail, so a mid-pass error
+ // leaves the lock reachable for the cleanup above.
+ *guard = Some(conn);
+ let conn = guard.as_mut().expect("connection was just stashed");
+
+ let claimed: Vec<ExistingEventRow> = sqlx::query_as!(
ExistingEventRow,
r#"
- SELECT *
- FROM
- frecency_events
- WHERE was_processed = false
- LIMIT $1
+ UPDATE frecency_events
+ SET was_processed = true
+ WHERE id IN (
+ SELECT id
+ FROM frecency_events
+ WHERE was_processed = false
+ ORDER BY id
+ LIMIT $1
+ FOR UPDATE SKIP LOCKED
+ )
+ RETURNING
+ id,
+ user_id,
+ entity_type,
+ event_type,
+ timestamp,
+ connection_id,
+ entity_id,
+ was_processed
"#,
FETCH_LIMIT as i64
)
- .fetch_all(&mut *tx)
+ .fetch_all(&mut **conn)
.await?;
- let res: Result<Vec<_>, _> = res.into_iter().map(|r| r.into_event_record()).collect();
-
- let mut guard = self.tx.try_lock()?;
- *guard = Some(tx);
- res.map_err(PollerErr::from)
+ claimed
+ .into_iter()
+ .map(|r| r.into_event_record())
+ .collect::<Result<Vec<_>, _>>()
+ .map_err(PollerErr::from)
}
+ /// Releases the aggregation lock taken by
+ /// [`UnprocessedEventsRepo::get_unprocessed_events`].
+ ///
+ /// The events themselves were already marked processed by the claim; what
+ /// still has to happen at the end of a pass is letting the next poller in,
+ /// which must not happen until this pass's `set_aggregates` has landed.
async fn mark_processed<'a>(
&self,
- event: Vec<EventRecordWithId<'a, i64>>,
+ _event: Vec<EventRecordWithId<'a, i64>>,
) -> Result<(), Self::Err> {
- let mut guard = self.tx.try_lock()?;
-
- let mut tx = guard.take().ok_or(PollerErr::TxErr)?;
-
- if event.is_empty() {
- tx.commit().await?;
+ let mut guard = self.aggregate_lock.try_lock()?;
+ let Some(mut conn) = guard.take() else {
return Ok(());
- }
-
- let mut query_builder = QueryBuilder::<Postgres>::new(
- r#"
- UPDATE frecency_events
- SET was_processed = true
- WHERE id IN (
- "#,
- );
-
- let mut separated = query_builder.separated(", ");
- for e in event {
- separated.push_bind(e.id);
- }
- separated.push_unseparated(")");
-
- let query = query_builder.build();
-
- query.execute(&mut *tx).await?;
-
- tx.commit().await?;
+ };
+ sqlx::query_scalar!(r#"SELECT pg_advisory_unlock($1)"#, FRECENCY_POLLER_LOCK_ID)
+ .fetch_one(&mut *conn)
+ .await?;
Ok(())
}
@@ -531,8 +578,12 @@ impl UnprocessedEventsRepo for FrecencyPgProcessor {
&self,
aggregates: Vec<AggregateId<'_>>,
) -> Result<Vec<AggregateFrecency>, Self::Err> {
- let mut guard = self.tx.try_lock()?;
- let tx = guard.as_deref_mut().ok_or(PollerErr::TxErr)?;
+ // `push_tuples` renders an empty iterator as `IN ()`, which Postgres
+ // rejects as a syntax error. Callers should not reach here with an
+ // empty batch, but nothing left to look up has an obvious answer.
+ if aggregates.is_empty() {
+ return Ok(Vec::new());
+ }
let mut query_builder = QueryBuilder::<Postgres>::new(
r#"
@@ -559,7 +610,7 @@ impl UnprocessedEventsRepo for FrecencyPgProcessor {
let query = query_builder.build_query_as::<AggregateRow>();
- let output = query.fetch_all(tx).await?;
+ let output = query.fetch_all(&self.pool).await?;
let out: Result<Vec<_>, _> = output
.into_iter()
@@ -570,9 +621,6 @@ impl UnprocessedEventsRepo for FrecencyPgProcessor {
}
async fn set_aggregates(&self, aggregates: Vec<AggregateFrecency>) -> Result<(), Self::Err> {
- let mut guard = self.tx.try_lock()?;
- let tx = guard.as_deref_mut().ok_or(PollerErr::TxErr)?;
-
if aggregates.is_empty() {
return Ok(());
}
@@ -617,7 +665,7 @@ impl UnprocessedEventsRepo for FrecencyPgProcessor {
);
let query = query_builder.build();
- query.execute(tx).await?;
+ query.execute(&self.pool).await?;
Ok(())
}
diff --git a/crates/frecency/src/outbound/postgres/tests.rs b/crates/frecency/src/outbound/postgres/tests.rs
index a229d580dc..2e21493121 100644
--- a/crates/frecency/src/outbound/postgres/tests.rs
+++ b/crates/frecency/src/outbound/postgres/tests.rs
@@ -523,7 +523,7 @@ async fn test_processor_mark_processed(pool: PgPool) {
event_ids.push(id);
}
- // Get unprocessed events (starts a transaction)
+ // Claims the batch and commits; the rows are already marked processed.
let unprocessed = processor.get_unprocessed_events().await.unwrap();
// Filter to just our test events
@@ -534,7 +534,7 @@ async fn test_processor_mark_processed(pool: PgPool) {
assert_eq!(to_mark.len(), 2);
- // Mark them as processed (commits the transaction)
+ // Releases the aggregation lock; the rows were marked by the claim.
processor.mark_processed(to_mark).await.unwrap();
// Verify they are marked as processed in the database
@@ -794,7 +794,7 @@ async fn test_processor_set_aggregates_empty_list(pool: PgPool) {
}
#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
-async fn test_processor_transaction_lifecycle(pool: PgPool) {
+async fn test_processor_claim_aggregate_lifecycle(pool: PgPool) {
let processor = FrecencyPgProcessor::new(pool.clone());
let test_user_id = MacroUserIdStr::parse_from_str("macro|[email protected]").unwrap();
@@ -819,7 +819,7 @@ async fn test_processor_transaction_lifecycle(pool: PgPool) {
.await
.unwrap();
- // Get unprocessed events (starts transaction)
+ // Claims the batch and commits.
let events = processor.get_unprocessed_events().await.unwrap();
let test_event = events.into_iter().find(|e| e.id == event_id);
assert!(test_event.is_some());
@@ -838,10 +838,10 @@ async fn test_processor_transaction_lifecycle(pool: PgPool) {
},
};
- // Set the aggregate (uses same transaction)
+ // Written on its own statement, after the claim committed.
processor.set_aggregates(vec![aggregate]).await.unwrap();
- // Mark as processed (commits the same transaction)
+ // Releases the aggregation lock taken by the claim.
processor
.mark_processed(vec![test_event.unwrap()])
.await
@@ -875,62 +875,62 @@ async fn test_processor_transaction_lifecycle(pool: PgPool) {
assert!(was_processed);
}
+/// An idle queue must not strand the aggregation lock.
+///
+/// The pass still takes the lock before discovering there is nothing to claim,
+/// so it has to be closed out by mark_processed like any other. Otherwise the
+/// lock would sit held until the next tick cleaned it up -- once per poll, on
+/// every replica, for as long as the queue stays empty.
#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
-async fn it_cannot_be_read_concurrently(pool: PgPool) {
- let test_user_id = "test_processor_tx_lifecycle";
- // Insert an unprocessed event
- let event_id = sqlx::query_scalar!(
- r#"
- INSERT INTO frecency_events (
- user_id, entity_type, event_type, timestamp,
- connection_id, entity_id, was_processed
- )
- VALUES ($1, $2, $3, $4, $5, $6, false)
- RETURNING id
- "#,
- test_user_id,
- "document",
- "open",
- Utc::now(),
- "conn_tx",
- "doc_tx"
- )
- .fetch_one(&pool)
- .await
- .unwrap();
-
+async fn an_empty_batch_still_releases_the_aggregation_lock(pool: PgPool) {
let first = FrecencyPgProcessor::new(pool.clone());
let second = FrecencyPgProcessor::new(pool);
+ // No events inserted: the claim finds nothing.
let events = first.get_unprocessed_events().await.unwrap();
- assert_eq!(events.len(), 1);
- assert_eq!(events.first().unwrap().id, event_id);
+ assert!(events.is_empty());
- // try to read from a second processor
- // this fails because the first processor still holds the lock
+ // The lock was taken regardless, so the second poller is still shut out.
let err = second.get_unprocessed_events().await.unwrap_err();
- assert!(matches!(err, PollerErr::DbLockErr));
+ assert!(matches!(err, PollerErr::DbLockErr), "got {err:?}");
- // finish the tx
+ // Closing out the empty pass releases it.
first.mark_processed(Vec::new()).await.unwrap();
-
- // now second can read because the transaction has finished
let res = second.get_unprocessed_events().await.unwrap();
- assert_eq!(res.len(), 1);
- assert_eq!(res.first().unwrap().id, event_id);
+ assert!(res.is_empty());
}
+/// The aggregate lookup renders its filter with `push_tuples`, which emits a
+/// bare `IN ()` for an empty batch. Callers should short-circuit before here,
+/// but the method must not turn "nothing to look up" into a syntax error.
#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
-async fn test_stale_transaction_is_cleaned_up(pool: PgPool) {
- let test_user_id = "test_stale_tx_cleanup";
+async fn empty_aggregate_lookup_returns_empty_rather_than_failing(pool: PgPool) {
+ let processor = FrecencyPgProcessor::new(pool);
+ let res = processor
+ .get_aggregates_for_users_entities(Vec::new())
+ .await
+ .expect("an empty lookup must not be a syntax error");
+ assert!(res.is_empty());
+}
- sqlx::query!(
+/// Only one poller may aggregate at a time.
+///
+/// Disjoint event batches can still merge into the same aggregate key, and
+/// set_aggregates blind-overwrites that row, so two concurrent passes would
+/// lose one of their contributions. The second poller must be turned away
+/// while the first holds the lock, and admitted once it releases.
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn a_second_poller_is_locked_out_until_the_first_finishes(pool: PgPool) {
+ let test_user_id = "test_processor_claim_is_exclusive";
+ // Insert an unprocessed event
+ let event_id = sqlx::query_scalar!(
r#"
INSERT INTO frecency_events (
user_id, entity_type, event_type, timestamp,
connection_id, entity_id, was_processed
)
VALUES ($1, $2, $3, $4, $5, $6, false)
+ RETURNING id
"#,
test_user_id,
"document",
@@ -939,24 +939,41 @@ async fn test_stale_transaction_is_cleaned_up(pool: PgPool) {
"conn_tx",
"doc_tx"
)
- .execute(&pool)
+ .fetch_one(&pool)
.await
.unwrap();
- let processor = FrecencyPgProcessor::new(pool);
+ let pool_handle = pool.clone();
+ let first = FrecencyPgProcessor::new(pool.clone());
+ let second = FrecencyPgProcessor::new(pool);
- // First call succeeds, starts a transaction and stores it in the mutex
- let events = processor.get_unprocessed_events().await.unwrap();
+ let events = first.get_unprocessed_events().await.unwrap();
assert_eq!(events.len(), 1);
+ assert_eq!(events.first().unwrap().id, event_id);
- // Simulate a processing failure by NOT calling mark_processed.
- // The next call to get_unprocessed_events should clean up the stale tx
- // and successfully start a new processing cycle.
- let events = processor.get_unprocessed_events().await.unwrap();
- assert_eq!(events.len(), 1);
+ // The first poller holds the aggregation lock across its whole pass, so
+ // the second must not be allowed to read-modify-write concurrently.
+ let err = second.get_unprocessed_events().await.unwrap_err();
+ assert!(
+ matches!(err, PollerErr::DbLockErr),
+ "second poller should be locked out, got {err:?}"
+ );
- // Verify we can still complete the full lifecycle
- processor.mark_processed(Vec::new()).await.unwrap();
+ // The claim itself already marked the row, independent of the lock.
+ let was_processed = sqlx::query_scalar!(
+ r#"SELECT was_processed AS "was_processed!" FROM frecency_events WHERE id = $1"#,
+ event_id
+ )
+ .fetch_one(&pool_handle)
+ .await
+ .unwrap();
+ assert!(was_processed);
+
+ // Ending the first pass releases the lock and lets the second in; there
+ // is simply nothing left for it to claim.
+ first.mark_processed(Vec::new()).await.unwrap();
+ let res = second.get_unprocessed_events().await.unwrap();
+ assert!(res.is_empty());
}
/// Tests that mixing supported (chat_id) and unsupported (role) filters
diff --git a/crates/macro_db_client/migrations/20260910144342_frecency_events_unprocessed_id_index.sql b/crates/macro_db_client/migrations/20260910144342_frecency_events_unprocessed_id_index.sql
new file mode 100644
index 0000000000..24f8d6eeea
--- /dev/null
+++ b/crates/macro_db_client/migrations/20260910144342_frecency_events_unprocessed_id_index.sql
@@ -0,0 +1,12 @@
+-- no-transaction
+-- Replaces idx_frecency_events_unprocessed, which keys on was_processed under
+-- the predicate `was_processed = false`. Every entry in that index therefore
+-- holds the same key value, so it offers neither ordering nor selectivity and
+-- the planner never chose it -- Datadog reported it as unused.
+-- The poller claims batches with `ORDER BY id ... FOR UPDATE SKIP LOCKED`, so
+-- keying on id under the same predicate gives that scan something to walk.
+-- Added before the old index is dropped (20260910144343) so no deploy window
+-- runs without one.
+-- Single statement: see 20260910144339 for the CONCURRENTLY constraint.
+CREATE INDEX CONCURRENTLY IF NOT EXISTS idx_frecency_events_unprocessed_id
+ ON frecency_events (id) WHERE was_processed = false;
diff --git a/crates/macro_db_client/migrations/20260910144343_drop_frecency_events_unprocessed_index.sql b/crates/macro_db_client/migrations/20260910144343_drop_frecency_events_unprocessed_index.sql
new file mode 100644
index 0000000000..f5c5140d45
--- /dev/null
+++ b/crates/macro_db_client/migrations/20260910144343_drop_frecency_events_unprocessed_index.sql
@@ -0,0 +1,5 @@
+-- no-transaction
+-- Superseded by idx_frecency_events_unprocessed_id (20260910144342), which
+-- indexes the same rows on a column that can actually drive a scan.
+-- Single statement: see 20260910144339 for the CONCURRENTLY constraint.
+DROP INDEX CONCURRENTLY IF EXISTS idx_frecency_events_unprocessed;