Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(ai-projections): recover recommendations stuck loading (#9300)

Wolf Mermelsteinbe5df119cfc6
diff --git a/.sqlx/query-2139f35ae8c3363bad3251ee3274fa19fa64c46d0fcaa98f5ae6ae83d186c556.json b/.sqlx/query-2139f35ae8c3363bad3251ee3274fa19fa64c46d0fcaa98f5ae6ae83d186c556.json
new file mode 100644
index 0000000000..b361c6b94c
--- /dev/null
+++ b/.sqlx/query-2139f35ae8c3363bad3251ee3274fa19fa64c46d0fcaa98f5ae6ae83d186c556.json
@@ -0,0 +1,14 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n DELETE FROM processing_ai_projections\n WHERE created_at < NOW() - make_interval(secs => $1)\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Float8"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "2139f35ae8c3363bad3251ee3274fa19fa64c46d0fcaa98f5ae6ae83d186c556"
+}
diff --git a/.sqlx/query-3aea5be9ab63b6b45e9dffc5862c0981979a0a681228c15d0ae3d733f4713b3f.json b/.sqlx/query-3aea5be9ab63b6b45e9dffc5862c0981979a0a681228c15d0ae3d733f4713b3f.json
deleted file mode 100644
index c6270364f2..0000000000
--- a/.sqlx/query-3aea5be9ab63b6b45e9dffc5862c0981979a0a681228c15d0ae3d733f4713b3f.json
+++ /dev/null
@@ -1,12 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "\n DELETE FROM processing_ai_projections\n WHERE created_at < NOW() - INTERVAL '15 minutes'\n ",
- "describe": {
- "columns": [],
- "parameters": {
- "Left": []
- },
- "nullable": []
- },
- "hash": "3aea5be9ab63b6b45e9dffc5862c0981979a0a681228c15d0ae3d733f4713b3f"
-}
diff --git a/.sqlx/query-c64ed6396b2cfe14c398da9f3ebb6479d3383948033af0992a85e494ccf5ca6a.json b/.sqlx/query-c64ed6396b2cfe14c398da9f3ebb6479d3383948033af0992a85e494ccf5ca6a.json
new file mode 100644
index 0000000000..0cfcea2fff
--- /dev/null
+++ b/.sqlx/query-c64ed6396b2cfe14c398da9f3ebb6479d3383948033af0992a85e494ccf5ca6a.json
@@ -0,0 +1,16 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n UPDATE user_ai_projection instance\n SET status = 'cold'\n WHERE instance.ai_projection_id = $1 AND instance.target_id = $2\n AND instance.status = 'loading'\n AND NOT EXISTS (\n SELECT 1 FROM processing_ai_projections processing\n WHERE processing.ai_projection_id = instance.ai_projection_id\n AND processing.target_id = instance.target_id\n AND processing.created_at >= NOW() - make_interval(secs => $3)\n )\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Text",
+ "Text",
+ "Float8"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "c64ed6396b2cfe14c398da9f3ebb6479d3383948033af0992a85e494ccf5ca6a"
+}
diff --git a/crates/ai_projections/AGENTS.md b/crates/ai_projections/AGENTS.md
index 427f3bac9c..fd92f05565 100644
--- a/crates/ai_projections/AGENTS.md
+++ b/crates/ai_projections/AGENTS.md
@@ -11,6 +11,12 @@ regenerated on demand or when cold.
- `src/lib.rs` doc comment: the full lifecycle — upsert-and-read on
`POST /ai-projections`, async vs. `await: true` materialization,
`prompt_hash` versioning, push via connection gateway.
+- Reads recover abandoned loading instances as cold when their
+ processing claim is missing or expired; active claims and retained results
+ stay intact; queued refreshing instances need no processing claim. The
+ persistence adapter shares one processing lease for recovery and claim
+ replacement. Recovery locks the instance before checking its claim in a
+ fresh statement snapshot.
- `src/domain/ai_projection_service.rs`: the upsert/read/invalidate logic.
- `src/domain/projection_generator.rs`: the generation port.
- `src/outbound/agent_generator.rs`: the agent-backed generator (behind the
diff --git a/crates/ai_projections/fixtures/projection_read_barrier.sql b/crates/ai_projections/fixtures/projection_read_barrier.sql
new file mode 100644
index 0000000000..3856544022
--- /dev/null
+++ b/crates/ai_projections/fixtures/projection_read_barrier.sql
@@ -0,0 +1,16 @@
+-- Release the reader's session lock only after its INSERT snapshot exists.
+-- The controlling transaction can then commit a claim before releasing the
+-- conflicting instance row. This models a worker starting during an upsert.
+CREATE FUNCTION release_projection_reader_barrier() RETURNS trigger
+LANGUAGE plpgsql AS $$
+BEGIN
+ IF current_setting('application_name') = 'projection-concurrent-reader' THEN
+ PERFORM pg_advisory_unlock(710471789::bigint);
+ END IF;
+ RETURN NEW;
+END;
+$$;
+
+CREATE TRIGGER projection_reader_barrier
+BEFORE INSERT ON user_ai_projection
+FOR EACH ROW EXECUTE FUNCTION release_projection_reader_barrier();
diff --git a/crates/ai_projections/src/domain/ai_projection_repo.rs b/crates/ai_projections/src/domain/ai_projection_repo.rs
index e0e239ced8..2cf70b142d 100644
--- a/crates/ai_projections/src/domain/ai_projection_repo.rs
+++ b/crates/ai_projections/src/domain/ai_projection_repo.rs
@@ -28,7 +28,11 @@ pub trait AiProjectionRepository: Clone + Send + Sync + 'static {
/// Gets the target's instance of a projection, or creates a cold instance
/// if one does not exist. An existing instance whose `prompt_hash` differs
/// from the definition's is reset to cold (keeping its previous result
- /// visible until regeneration overwrites it).
+ /// visible until regeneration overwrites it). Loading instances without a
+ /// live processing claim are also recovered as cold,
+ /// so a later read can restart abandoned generation. Active claims retain
+ /// their state and existing results remain available. Refreshing instances
+ /// can still be queued without a claim and are left intact.
fn get_or_create_target_projection(
&self,
ai_projection_id: &str,
diff --git a/crates/ai_projections/src/lib.rs b/crates/ai_projections/src/lib.rs
index 82a471af2a..cf78ea0c04 100644
--- a/crates/ai_projections/src/lib.rs
+++ b/crates/ai_projections/src/lib.rs
@@ -15,7 +15,9 @@
//! `ai_projection_queue` by default, or inline when the request sets
//! `await: true`. Completed (or failed) materializations are pushed to the
//! target's connected clients through the connection gateway as
-//! `ai_projection_updated` messages, so the frontend never polls.
+//! `ai_projection_updated` messages, so the frontend never polls. Reads recover
+//! loading instances with a missing or expired processing claim
+//! as cold, allowing generation to restart without discarding retained data.
//!
//! The definition's `prompt_hash` versions the whole generation recipe
//! (prompt, optional `model` override, optional `output_schema`): re-upserting
diff --git a/crates/ai_projections/src/outbound/ai_projection_repo.rs b/crates/ai_projections/src/outbound/ai_projection_repo.rs
index 3d767386e0..9a606c46d9 100644
--- a/crates/ai_projections/src/outbound/ai_projection_repo.rs
+++ b/crates/ai_projections/src/outbound/ai_projection_repo.rs
@@ -14,6 +14,9 @@ use crate::domain::{
#[cfg(test)]
mod test;
+// The same lease bounds read-time recovery and processing-claim replacement.
+const PROCESSING_LEASE_SECONDS: f64 = 15.0 * 60.0;
+
/// The AiProjectionRepositoryImpl is a wrapper around a sqlx::PgPool connected
/// to macrodb.
#[derive(Clone)]
@@ -111,9 +114,9 @@ impl AiProjectionRepository for AiProjectionRepositoryImpl {
// A request for a projection bumps `last_requested_at` so the background
// refresh handler can tell active instances apart from abandoned ones
// (which it eventually deletes once they fall outside their expiry
- // window). An instance created for an older prompt version is reset to
- // cold — its previous result stays visible until regeneration
- // overwrites it — while a same-version instance keeps its status.
+ // window). Revised or abandoned in-flight instances become cold; retained
+ // results remain visible until a new generation overwrites them.
+ let mut transaction = self.pool.begin().await.map_err(sqlx_err)?;
sqlx::query!(
r#"
INSERT INTO user_ai_projection (ai_projection_id, target_id, prompt_hash, status)
@@ -134,9 +137,35 @@ impl AiProjectionRepository for AiProjectionRepositoryImpl {
prompt_hash,
ProjectionStatus::Cold.to_string(),
)
- .execute(&self.pool)
+ .execute(&mut *transaction)
+ .await
+ .map_err(sqlx_err)?;
+
+ // The upsert holds the instance row lock. Inspect its claim in a fresh
+ // statement snapshot: ON CONFLICT can observe a row committed after
+ // its original snapshot, so checking the claim inside it could miss
+ // the worker that just marked that instance loading.
+ sqlx::query!(
+ r#"
+ UPDATE user_ai_projection instance
+ SET status = 'cold'
+ WHERE instance.ai_projection_id = $1 AND instance.target_id = $2
+ AND instance.status = 'loading'
+ AND NOT EXISTS (
+ SELECT 1 FROM processing_ai_projections processing
+ WHERE processing.ai_projection_id = instance.ai_projection_id
+ AND processing.target_id = instance.target_id
+ AND processing.created_at >= NOW() - make_interval(secs => $3)
+ )
+ "#,
+ ai_projection_id,
+ target_id,
+ PROCESSING_LEASE_SECONDS,
+ )
+ .execute(&mut *transaction)
.await
.map_err(sqlx_err)?;
+ transaction.commit().await.map_err(sqlx_err)?;
self.get_target_projection(ai_projection_id, target_id)
.await
@@ -209,13 +238,14 @@ impl AiProjectionRepository for AiProjectionRepositoryImpl {
target_id: &str,
) -> Result<bool, AiProjectionError> {
// Reclaim claims left behind by crashed/stuck workers so they do not
- // block reprocessing forever. The threshold is a SQL literal because
- // `query!` binds parameters as typed values, not as an interval string.
+ // block reprocessing forever. Reads use this same processing lease
+ // when recovering an abandoned loading instance.
sqlx::query!(
r#"
DELETE FROM processing_ai_projections
- WHERE created_at < NOW() - INTERVAL '15 minutes'
+ WHERE created_at < NOW() - make_interval(secs => $1)
"#,
+ PROCESSING_LEASE_SECONDS,
)
.execute(&self.pool)
.await
diff --git a/crates/ai_projections/src/outbound/ai_projection_repo/test.rs b/crates/ai_projections/src/outbound/ai_projection_repo/test.rs
index 4f3c96a3ae..db71c377c1 100644
--- a/crates/ai_projections/src/outbound/ai_projection_repo/test.rs
+++ b/crates/ai_projections/src/outbound/ai_projection_repo/test.rs
@@ -464,3 +464,186 @@ async fn user_has_permission_reflects_roles(pool: Pool<Postgres>) -> anyhow::Res
Ok(())
}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("ai_projections"))
+)]
+async fn reading_abandoned_loading_recovers_generation(pool: Pool<Postgres>) -> anyhow::Result<()> {
+ let repo = AiProjectionRepositoryImpl::new(pool);
+ let target = "macro|[email protected]";
+ upsert_definition(&repo, "inbox/recovery", "Summarize unread items", "hash_v1").await?;
+ repo.get_or_create_target_projection("inbox/recovery", target, "hash_v1")
+ .await?;
+ assert!(repo.try_start_processing("inbox/recovery", target).await?);
+ repo.set_projection_loading("inbox/recovery", target, "hash_v1")
+ .await?;
+ let active = repo
+ .get_or_create_target_projection("inbox/recovery", target, "hash_v1")
+ .await?;
+ assert_eq!(active.status, ProjectionStatus::Loading);
+ repo.finish_processing("inbox/recovery", target).await?;
+
+ let recovered = repo
+ .get_or_create_target_projection("inbox/recovery", target, "hash_v1")
+ .await?;
+ assert_eq!(recovered.status, ProjectionStatus::Cold);
+ assert!(recovered.result.is_none());
+ assert!(repo.try_start_processing("inbox/recovery", target).await?);
+ Ok(())
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("ai_projections"))
+)]
+async fn reading_refresh_keeps_queued_work_and_recovers_abandoned_generation(
+ pool: Pool<Postgres>,
+) -> anyhow::Result<()> {
+ let repo = AiProjectionRepositoryImpl::new(pool);
+ let target = "macro|[email protected]";
+ upsert_definition(&repo, "inbox/recovery", "Summarize unread items", "hash_v1").await?;
+ repo.get_or_create_target_projection("inbox/recovery", target, "hash_v1")
+ .await?;
+ let generated_at = chrono::Utc::now();
+ repo.set_projection_result(
+ "inbox/recovery",
+ target,
+ "hash_v1",
+ "retained result",
+ generated_at,
+ generated_at + Expiry::Day.to_duration(),
+ )
+ .await?;
+ repo.set_projection_refreshing("inbox/recovery", target)
+ .await?;
+ let queued = repo
+ .get_or_create_target_projection("inbox/recovery", target, "hash_v1")
+ .await?;
+ assert_eq!(queued.status, ProjectionStatus::Refreshing);
+ assert_eq!(queued.result.as_deref(), Some("retained result"));
+ assert!(repo.try_start_processing("inbox/recovery", target).await?);
+ repo.set_projection_loading("inbox/recovery", target, "hash_v1")
+ .await?;
+
+ let active = repo
+ .get_or_create_target_projection("inbox/recovery", target, "hash_v1")
+ .await?;
+ assert_eq!(active.status, ProjectionStatus::Loading);
+ assert_eq!(active.result.as_deref(), Some("retained result"));
+ assert!(!repo.try_start_processing("inbox/recovery", target).await?);
+
+ repo.finish_processing("inbox/recovery", target).await?;
+ let recovered = repo
+ .get_or_create_target_projection("inbox/recovery", target, "hash_v1")
+ .await?;
+ assert_eq!(recovered.status, ProjectionStatus::Cold);
+ assert_eq!(recovered.result.as_deref(), Some("retained result"));
+ Ok(())
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(path = "../../../fixtures", scripts("ai_projections"))
+)]
+async fn reading_loading_reclaims_an_expired_processing_lease(
+ pool: Pool<Postgres>,
+) -> anyhow::Result<()> {
+ let repo = AiProjectionRepositoryImpl::new(pool.clone());
+ let target = "macro|[email protected]";
+ upsert_definition(&repo, "inbox/recovery", "Summarize unread items", "hash_v1").await?;
+ repo.get_or_create_target_projection("inbox/recovery", target, "hash_v1")
+ .await?;
+ assert!(repo.try_start_processing("inbox/recovery", target).await?);
+ repo.set_projection_loading("inbox/recovery", target, "hash_v1")
+ .await?;
+ sqlx::query!(
+ "UPDATE processing_ai_projections SET created_at = NOW() - INTERVAL '16 minutes' WHERE ai_projection_id = $1 AND target_id = $2",
+ "inbox/recovery",
+ target,
+ )
+ .execute(&pool)
+ .await?;
+
+ let recovered = repo
+ .get_or_create_target_projection("inbox/recovery", target, "hash_v1")
+ .await?;
+ assert_eq!(recovered.status, ProjectionStatus::Cold);
+ assert!(repo.try_start_processing("inbox/recovery", target).await?);
+ assert!(!repo.try_start_processing("inbox/recovery", target).await?);
+ Ok(())
+}
+
+#[sqlx::test(
+ migrator = "MACRO_DB_MIGRATIONS",
+ fixtures(
+ path = "../../../fixtures",
+ scripts("ai_projections", "projection_read_barrier")
+ )
+)]
+async fn reading_during_worker_start_preserves_the_new_processing_claim(
+ pool: Pool<Postgres>,
+) -> anyhow::Result<()> {
+ let repo = AiProjectionRepositoryImpl::new(pool.clone());
+ let target = "macro|[email protected]";
+ upsert_definition(&repo, "inbox/recovery", "Summarize unread items", "hash_v1").await?;
+ repo.get_or_create_target_projection("inbox/recovery", target, "hash_v1")
+ .await?;
+
+ // One connection ensures the repository read uses the session holding the
+ // barrier. Its BEFORE INSERT trigger releases that barrier after PostgreSQL
+ // takes the statement snapshot, before ON CONFLICT waits for the row lock.
+ let reader_pool = sqlx::postgres::PgPoolOptions::new()
+ .max_connections(1)
+ .connect_with(
+ (*pool.connect_options())
+ .clone()
+ .application_name("projection-concurrent-reader"),
+ )
+ .await?;
+ sqlx::query!(
+ "SELECT pg_advisory_lock($1) IS NULL AS barrier",
+ 710471789_i64
+ )
+ .fetch_one(&reader_pool)
+ .await?;
+ let mut worker = pool.begin().await?;
+ sqlx::query!(
+ "SELECT ai_projection_id FROM user_ai_projection WHERE ai_projection_id = $1 AND target_id = $2 FOR UPDATE",
+ "inbox/recovery",
+ target,
+ )
+ .fetch_one(&mut *worker)
+ .await?;
+ let reader = tokio::spawn(async move {
+ AiProjectionRepositoryImpl::new(reader_pool)
+ .get_or_create_target_projection("inbox/recovery", target, "hash_v1")
+ .await
+ });
+ sqlx::query!(
+ "SELECT pg_advisory_xact_lock($1) IS NULL AS barrier",
+ 710471789_i64
+ )
+ .fetch_one(&mut *worker)
+ .await?;
+ sqlx::query!(
+ "INSERT INTO processing_ai_projections (ai_projection_id, target_id) VALUES ($1, $2)",
+ "inbox/recovery",
+ target,
+ )
+ .execute(&mut *worker)
+ .await?;
+ sqlx::query!(
+ "UPDATE user_ai_projection SET status = 'loading' WHERE ai_projection_id = $1 AND target_id = $2",
+ "inbox/recovery",
+ target,
+ )
+ .execute(&mut *worker)
+ .await?;
+ worker.commit().await?;
+
+ let active = reader.await??;
+ assert_eq!(active.status, ProjectionStatus::Loading);
+ assert!(!repo.try_start_processing("inbox/recovery", target).await?);
+ Ok(())
+}