Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(ai-projections): recover recommendations stuck loading (#9300)
Wolf Mermelsteinbe5df119cfc6
diff --git a/.sqlx/query-2139f35ae8c3363bad3251ee3274fa19fa64c46d0fcaa98f5ae6ae83d186c556.json b/.sqlx/query-2139f35ae8c3363bad3251ee3274fa19fa64c46d0fcaa98f5ae6ae83d186c556.jsonnew file mode 100644index 0000000000..b361c6b94c--- /dev/null+++ b/.sqlx/query-2139f35ae8c3363bad3251ee3274fa19fa64c46d0fcaa98f5ae6ae83d186c556.json@@ -0,0 +1,14 @@+{+ "db_name": "PostgreSQL",+ "query": "\n DELETE FROM processing_ai_projections\n WHERE created_at < NOW() - make_interval(secs => $1)\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Float8"+ ]+ },+ "nullable": []+ },+ "hash": "2139f35ae8c3363bad3251ee3274fa19fa64c46d0fcaa98f5ae6ae83d186c556"+}diff --git a/.sqlx/query-3aea5be9ab63b6b45e9dffc5862c0981979a0a681228c15d0ae3d733f4713b3f.json b/.sqlx/query-3aea5be9ab63b6b45e9dffc5862c0981979a0a681228c15d0ae3d733f4713b3f.jsondeleted file mode 100644index c6270364f2..0000000000--- a/.sqlx/query-3aea5be9ab63b6b45e9dffc5862c0981979a0a681228c15d0ae3d733f4713b3f.json+++ /dev/null@@ -1,12 +0,0 @@-{- "db_name": "PostgreSQL",- "query": "\n DELETE FROM processing_ai_projections\n WHERE created_at < NOW() - INTERVAL '15 minutes'\n ",- "describe": {- "columns": [],- "parameters": {- "Left": []- },- "nullable": []- },- "hash": "3aea5be9ab63b6b45e9dffc5862c0981979a0a681228c15d0ae3d733f4713b3f"-}diff --git a/.sqlx/query-c64ed6396b2cfe14c398da9f3ebb6479d3383948033af0992a85e494ccf5ca6a.json b/.sqlx/query-c64ed6396b2cfe14c398da9f3ebb6479d3383948033af0992a85e494ccf5ca6a.jsonnew file mode 100644index 0000000000..0cfcea2fff--- /dev/null+++ b/.sqlx/query-c64ed6396b2cfe14c398da9f3ebb6479d3383948033af0992a85e494ccf5ca6a.json@@ -0,0 +1,16 @@+{+ "db_name": "PostgreSQL",+ "query": "\n UPDATE user_ai_projection instance\n SET status = 'cold'\n WHERE instance.ai_projection_id = $1 AND instance.target_id = $2\n AND instance.status = 'loading'\n AND NOT EXISTS (\n SELECT 1 FROM processing_ai_projections processing\n WHERE processing.ai_projection_id = instance.ai_projection_id\n AND processing.target_id = instance.target_id\n AND processing.created_at >= NOW() - make_interval(secs => $3)\n )\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Text",+ "Text",+ "Float8"+ ]+ },+ "nullable": []+ },+ "hash": "c64ed6396b2cfe14c398da9f3ebb6479d3383948033af0992a85e494ccf5ca6a"+}diff --git a/crates/ai_projections/AGENTS.md b/crates/ai_projections/AGENTS.mdindex 427f3bac9c..fd92f05565 100644--- a/crates/ai_projections/AGENTS.md+++ b/crates/ai_projections/AGENTS.md@@ -11,6 +11,12 @@ regenerated on demand or when cold.- `src/lib.rs` doc comment: the full lifecycle — upsert-and-read on`POST /ai-projections`, async vs. `await: true` materialization,`prompt_hash` versioning, push via connection gateway.+- Reads recover abandoned loading instances as cold when their+ processing claim is missing or expired; active claims and retained results+ stay intact; queued refreshing instances need no processing claim. The+ persistence adapter shares one processing lease for recovery and claim+ replacement. Recovery locks the instance before checking its claim in a+ fresh statement snapshot.- `src/domain/ai_projection_service.rs`: the upsert/read/invalidate logic.- `src/domain/projection_generator.rs`: the generation port.- `src/outbound/agent_generator.rs`: the agent-backed generator (behind thediff --git a/crates/ai_projections/fixtures/projection_read_barrier.sql b/crates/ai_projections/fixtures/projection_read_barrier.sqlnew file mode 100644index 0000000000..3856544022--- /dev/null+++ b/crates/ai_projections/fixtures/projection_read_barrier.sql@@ -0,0 +1,16 @@+-- Release the reader's session lock only after its INSERT snapshot exists.+-- The controlling transaction can then commit a claim before releasing the+-- conflicting instance row. This models a worker starting during an upsert.+CREATE FUNCTION release_projection_reader_barrier() RETURNS trigger+LANGUAGE plpgsql AS $$+BEGIN+ IF current_setting('application_name') = 'projection-concurrent-reader' THEN+ PERFORM pg_advisory_unlock(710471789::bigint);+ END IF;+ RETURN NEW;+END;+$$;++CREATE TRIGGER projection_reader_barrier+BEFORE INSERT ON user_ai_projection+FOR EACH ROW EXECUTE FUNCTION release_projection_reader_barrier();diff --git a/crates/ai_projections/src/domain/ai_projection_repo.rs b/crates/ai_projections/src/domain/ai_projection_repo.rsindex e0e239ced8..2cf70b142d 100644--- a/crates/ai_projections/src/domain/ai_projection_repo.rs+++ b/crates/ai_projections/src/domain/ai_projection_repo.rs@@ -28,7 +28,11 @@ pub trait AiProjectionRepository: Clone + Send + Sync + 'static {/// Gets the target's instance of a projection, or creates a cold instance/// if one does not exist. An existing instance whose `prompt_hash` differs/// from the definition's is reset to cold (keeping its previous result- /// visible until regeneration overwrites it).+ /// visible until regeneration overwrites it). Loading instances without a+ /// live processing claim are also recovered as cold,+ /// so a later read can restart abandoned generation. Active claims retain+ /// their state and existing results remain available. Refreshing instances+ /// can still be queued without a claim and are left intact.fn get_or_create_target_projection(&self,ai_projection_id: &str,diff --git a/crates/ai_projections/src/lib.rs b/crates/ai_projections/src/lib.rsindex 82a471af2a..cf78ea0c04 100644--- a/crates/ai_projections/src/lib.rs+++ b/crates/ai_projections/src/lib.rs@@ -15,7 +15,9 @@//! `ai_projection_queue` by default, or inline when the request sets//! `await: true`. Completed (or failed) materializations are pushed to the//! target's connected clients through the connection gateway as-//! `ai_projection_updated` messages, so the frontend never polls.+//! `ai_projection_updated` messages, so the frontend never polls. Reads recover+//! loading instances with a missing or expired processing claim+//! as cold, allowing generation to restart without discarding retained data.//!//! The definition's `prompt_hash` versions the whole generation recipe//! (prompt, optional `model` override, optional `output_schema`): re-upsertingdiff --git a/crates/ai_projections/src/outbound/ai_projection_repo.rs b/crates/ai_projections/src/outbound/ai_projection_repo.rsindex 3d767386e0..9a606c46d9 100644--- a/crates/ai_projections/src/outbound/ai_projection_repo.rs+++ b/crates/ai_projections/src/outbound/ai_projection_repo.rs@@ -14,6 +14,9 @@ use crate::domain::{#[cfg(test)]mod test;+// The same lease bounds read-time recovery and processing-claim replacement.+const PROCESSING_LEASE_SECONDS: f64 = 15.0 * 60.0;+/// The AiProjectionRepositoryImpl is a wrapper around a sqlx::PgPool connected/// to macrodb.#[derive(Clone)]@@ -111,9 +114,9 @@ impl AiProjectionRepository for AiProjectionRepositoryImpl {// A request for a projection bumps `last_requested_at` so the background// refresh handler can tell active instances apart from abandoned ones// (which it eventually deletes once they fall outside their expiry- // window). An instance created for an older prompt version is reset to- // cold — its previous result stays visible until regeneration- // overwrites it — while a same-version instance keeps its status.+ // window). Revised or abandoned in-flight instances become cold; retained+ // results remain visible until a new generation overwrites them.+ let mut transaction = self.pool.begin().await.map_err(sqlx_err)?;sqlx::query!(r#"INSERT INTO user_ai_projection (ai_projection_id, target_id, prompt_hash, status)@@ -134,9 +137,35 @@ impl AiProjectionRepository for AiProjectionRepositoryImpl {prompt_hash,ProjectionStatus::Cold.to_string(),)- .execute(&self.pool)+ .execute(&mut *transaction)+ .await+ .map_err(sqlx_err)?;++ // The upsert holds the instance row lock. Inspect its claim in a fresh+ // statement snapshot: ON CONFLICT can observe a row committed after+ // its original snapshot, so checking the claim inside it could miss+ // the worker that just marked that instance loading.+ sqlx::query!(+ r#"+ UPDATE user_ai_projection instance+ SET status = 'cold'+ WHERE instance.ai_projection_id = $1 AND instance.target_id = $2+ AND instance.status = 'loading'+ AND NOT EXISTS (+ SELECT 1 FROM processing_ai_projections processing+ WHERE processing.ai_projection_id = instance.ai_projection_id+ AND processing.target_id = instance.target_id+ AND processing.created_at >= NOW() - make_interval(secs => $3)+ )+ "#,+ ai_projection_id,+ target_id,+ PROCESSING_LEASE_SECONDS,+ )+ .execute(&mut *transaction).await.map_err(sqlx_err)?;+ transaction.commit().await.map_err(sqlx_err)?;self.get_target_projection(ai_projection_id, target_id).await@@ -209,13 +238,14 @@ impl AiProjectionRepository for AiProjectionRepositoryImpl {target_id: &str,) -> Result<bool, AiProjectionError> {// Reclaim claims left behind by crashed/stuck workers so they do not- // block reprocessing forever. The threshold is a SQL literal because- // `query!` binds parameters as typed values, not as an interval string.+ // block reprocessing forever. Reads use this same processing lease+ // when recovering an abandoned loading instance.sqlx::query!(r#"DELETE FROM processing_ai_projections- WHERE created_at < NOW() - INTERVAL '15 minutes'+ WHERE created_at < NOW() - make_interval(secs => $1)"#,+ PROCESSING_LEASE_SECONDS,).execute(&self.pool).awaitdiff --git a/crates/ai_projections/src/outbound/ai_projection_repo/test.rs b/crates/ai_projections/src/outbound/ai_projection_repo/test.rsindex 4f3c96a3ae..db71c377c1 100644--- a/crates/ai_projections/src/outbound/ai_projection_repo/test.rs+++ b/crates/ai_projections/src/outbound/ai_projection_repo/test.rs@@ -464,3 +464,186 @@ async fn user_has_permission_reflects_roles(pool: Pool<Postgres>) -> anyhow::ResOk(())}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("ai_projections"))+)]+async fn reading_abandoned_loading_recovers_generation(pool: Pool<Postgres>) -> anyhow::Result<()> {+ let repo = AiProjectionRepositoryImpl::new(pool);+ let target = "macro|[email protected]";+ upsert_definition(&repo, "inbox/recovery", "Summarize unread items", "hash_v1").await?;+ repo.get_or_create_target_projection("inbox/recovery", target, "hash_v1")+ .await?;+ assert!(repo.try_start_processing("inbox/recovery", target).await?);+ repo.set_projection_loading("inbox/recovery", target, "hash_v1")+ .await?;+ let active = repo+ .get_or_create_target_projection("inbox/recovery", target, "hash_v1")+ .await?;+ assert_eq!(active.status, ProjectionStatus::Loading);+ repo.finish_processing("inbox/recovery", target).await?;++ let recovered = repo+ .get_or_create_target_projection("inbox/recovery", target, "hash_v1")+ .await?;+ assert_eq!(recovered.status, ProjectionStatus::Cold);+ assert!(recovered.result.is_none());+ assert!(repo.try_start_processing("inbox/recovery", target).await?);+ Ok(())+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("ai_projections"))+)]+async fn reading_refresh_keeps_queued_work_and_recovers_abandoned_generation(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ let repo = AiProjectionRepositoryImpl::new(pool);+ let target = "macro|[email protected]";+ upsert_definition(&repo, "inbox/recovery", "Summarize unread items", "hash_v1").await?;+ repo.get_or_create_target_projection("inbox/recovery", target, "hash_v1")+ .await?;+ let generated_at = chrono::Utc::now();+ repo.set_projection_result(+ "inbox/recovery",+ target,+ "hash_v1",+ "retained result",+ generated_at,+ generated_at + Expiry::Day.to_duration(),+ )+ .await?;+ repo.set_projection_refreshing("inbox/recovery", target)+ .await?;+ let queued = repo+ .get_or_create_target_projection("inbox/recovery", target, "hash_v1")+ .await?;+ assert_eq!(queued.status, ProjectionStatus::Refreshing);+ assert_eq!(queued.result.as_deref(), Some("retained result"));+ assert!(repo.try_start_processing("inbox/recovery", target).await?);+ repo.set_projection_loading("inbox/recovery", target, "hash_v1")+ .await?;++ let active = repo+ .get_or_create_target_projection("inbox/recovery", target, "hash_v1")+ .await?;+ assert_eq!(active.status, ProjectionStatus::Loading);+ assert_eq!(active.result.as_deref(), Some("retained result"));+ assert!(!repo.try_start_processing("inbox/recovery", target).await?);++ repo.finish_processing("inbox/recovery", target).await?;+ let recovered = repo+ .get_or_create_target_projection("inbox/recovery", target, "hash_v1")+ .await?;+ assert_eq!(recovered.status, ProjectionStatus::Cold);+ assert_eq!(recovered.result.as_deref(), Some("retained result"));+ Ok(())+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(path = "../../../fixtures", scripts("ai_projections"))+)]+async fn reading_loading_reclaims_an_expired_processing_lease(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ let repo = AiProjectionRepositoryImpl::new(pool.clone());+ let target = "macro|[email protected]";+ upsert_definition(&repo, "inbox/recovery", "Summarize unread items", "hash_v1").await?;+ repo.get_or_create_target_projection("inbox/recovery", target, "hash_v1")+ .await?;+ assert!(repo.try_start_processing("inbox/recovery", target).await?);+ repo.set_projection_loading("inbox/recovery", target, "hash_v1")+ .await?;+ sqlx::query!(+ "UPDATE processing_ai_projections SET created_at = NOW() - INTERVAL '16 minutes' WHERE ai_projection_id = $1 AND target_id = $2",+ "inbox/recovery",+ target,+ )+ .execute(&pool)+ .await?;++ let recovered = repo+ .get_or_create_target_projection("inbox/recovery", target, "hash_v1")+ .await?;+ assert_eq!(recovered.status, ProjectionStatus::Cold);+ assert!(repo.try_start_processing("inbox/recovery", target).await?);+ assert!(!repo.try_start_processing("inbox/recovery", target).await?);+ Ok(())+}++#[sqlx::test(+ migrator = "MACRO_DB_MIGRATIONS",+ fixtures(+ path = "../../../fixtures",+ scripts("ai_projections", "projection_read_barrier")+ )+)]+async fn reading_during_worker_start_preserves_the_new_processing_claim(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ let repo = AiProjectionRepositoryImpl::new(pool.clone());+ let target = "macro|[email protected]";+ upsert_definition(&repo, "inbox/recovery", "Summarize unread items", "hash_v1").await?;+ repo.get_or_create_target_projection("inbox/recovery", target, "hash_v1")+ .await?;++ // One connection ensures the repository read uses the session holding the+ // barrier. Its BEFORE INSERT trigger releases that barrier after PostgreSQL+ // takes the statement snapshot, before ON CONFLICT waits for the row lock.+ let reader_pool = sqlx::postgres::PgPoolOptions::new()+ .max_connections(1)+ .connect_with(+ (*pool.connect_options())+ .clone()+ .application_name("projection-concurrent-reader"),+ )+ .await?;+ sqlx::query!(+ "SELECT pg_advisory_lock($1) IS NULL AS barrier",+ 710471789_i64+ )+ .fetch_one(&reader_pool)+ .await?;+ let mut worker = pool.begin().await?;+ sqlx::query!(+ "SELECT ai_projection_id FROM user_ai_projection WHERE ai_projection_id = $1 AND target_id = $2 FOR UPDATE",+ "inbox/recovery",+ target,+ )+ .fetch_one(&mut *worker)+ .await?;+ let reader = tokio::spawn(async move {+ AiProjectionRepositoryImpl::new(reader_pool)+ .get_or_create_target_projection("inbox/recovery", target, "hash_v1")+ .await+ });+ sqlx::query!(+ "SELECT pg_advisory_xact_lock($1) IS NULL AS barrier",+ 710471789_i64+ )+ .fetch_one(&mut *worker)+ .await?;+ sqlx::query!(+ "INSERT INTO processing_ai_projections (ai_projection_id, target_id) VALUES ($1, $2)",+ "inbox/recovery",+ target,+ )+ .execute(&mut *worker)+ .await?;+ sqlx::query!(+ "UPDATE user_ai_projection SET status = 'loading' WHERE ai_projection_id = $1 AND target_id = $2",+ "inbox/recovery",+ target,+ )+ .execute(&mut *worker)+ .await?;+ worker.commit().await?;++ let active = reader.await??;+ assert_eq!(active.status, ProjectionStatus::Loading);+ assert!(!repo.try_start_processing("inbox/recovery", target).await?);+ Ok(())+}