Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(auth): stop queueing harness requests on their token's row lock (#8915)

teo0a0c43e70bc3
diff --git a/.sqlx/query-42f75f261b91698eb160964f5d5ba56c19602fb2c113bde6d00ecf62ee87812d.json b/.sqlx/query-42f75f261b91698eb160964f5d5ba56c19602fb2c113bde6d00ecf62ee87812d.json
deleted file mode 100644
index dbc72fa69a..0000000000
--- a/.sqlx/query-42f75f261b91698eb160964f5d5ba56c19602fb2c113bde6d00ecf62ee87812d.json
+++ /dev/null
@@ -1,14 +0,0 @@
-{
- "db_name": "PostgreSQL",
- "query": "\n UPDATE harness_tokens\n SET last_used_at = now()\n WHERE id = $1\n ",
- "describe": {
- "columns": [],
- "parameters": {
- "Left": [
- "Uuid"
- ]
- },
- "nullable": []
- },
- "hash": "42f75f261b91698eb160964f5d5ba56c19602fb2c113bde6d00ecf62ee87812d"
-}
diff --git a/.sqlx/query-66c2f489de73168edfd5f8d914d43721e763b29dd24efbd66562a30ed94bcc62.json b/.sqlx/query-66c2f489de73168edfd5f8d914d43721e763b29dd24efbd66562a30ed94bcc62.json
new file mode 100644
index 0000000000..f22dc7fbaf
--- /dev/null
+++ b/.sqlx/query-66c2f489de73168edfd5f8d914d43721e763b29dd24efbd66562a30ed94bcc62.json
@@ -0,0 +1,14 @@
+{
+ "db_name": "PostgreSQL",
+ "query": "\n UPDATE harness_tokens\n SET last_used_at = now()\n WHERE id = (\n SELECT id FROM harness_tokens\n WHERE id = $1\n AND (last_used_at IS NULL OR last_used_at < now() - interval '1 minute')\n FOR UPDATE SKIP LOCKED\n )\n ",
+ "describe": {
+ "columns": [],
+ "parameters": {
+ "Left": [
+ "Uuid"
+ ]
+ },
+ "nullable": []
+ },
+ "hash": "66c2f489de73168edfd5f8d914d43721e763b29dd24efbd66562a30ed94bcc62"
+}
diff --git a/crates/macro_authorization/AGENTS.md b/crates/macro_authorization/AGENTS.md
index a494c33f83..3c5c66500a 100644
--- a/crates/macro_authorization/AGENTS.md
+++ b/crates/macro_authorization/AGENTS.md
@@ -70,6 +70,11 @@ does not turn a user credential into bot authority or change endpoint policies.
- Cookies are ambient and only considered when no explicit credential is
present; an explicit credential always wins over a cookie.
- A valid principal not admitted by the chosen policy is a 403, not a 401.
+- `mark_harness_token_used` runs on every harness-authenticated request. It
+ writes `last_used_at` at most once a minute and skips a row another request
+ is already updating. Don't make it a plain per-request `UPDATE`: one busy
+ token would then queue its requests on the row lock, each holding a pool
+ connection.
## Related
diff --git a/crates/macro_authorization/src/outbound/pg_harness_authorization.rs b/crates/macro_authorization/src/outbound/pg_harness_authorization.rs
index f39910e41b..2f758b8be6 100644
--- a/crates/macro_authorization/src/outbound/pg_harness_authorization.rs
+++ b/crates/macro_authorization/src/outbound/pg_harness_authorization.rs
@@ -124,12 +124,24 @@ impl HarnessAuthorizationRepo for PgHarnessAuthorizationRepo {
row.map(HarnessTokenAuthorization::try_from).transpose()
}
+ /// Records a token's use, at most about once a minute per token.
+ ///
+ /// Every harness-authenticated request calls this, and one token can carry
+ /// tens of requests a second. Rewriting its row each time queues them all on
+ /// the row lock, each holding a connection, so one slow commit stalls every
+ /// request behind it. A recent timestamp is left alone, and a row another
+ /// request is already updating is skipped rather than waited on.
async fn mark_harness_token_used(&self, token_id: Uuid) -> Result<(), Self::Err> {
sqlx::query!(
r#"
UPDATE harness_tokens
SET last_used_at = now()
- WHERE id = $1
+ WHERE id = (
+ SELECT id FROM harness_tokens
+ WHERE id = $1
+ AND (last_used_at IS NULL OR last_used_at < now() - interval '1 minute')
+ FOR UPDATE SKIP LOCKED
+ )
"#,
token_id,
)
diff --git a/crates/macro_authorization/src/outbound/pg_harness_authorization/test.rs b/crates/macro_authorization/src/outbound/pg_harness_authorization/test.rs
index f3a5ad1dc5..53b6fa650d 100644
--- a/crates/macro_authorization/src/outbound/pg_harness_authorization/test.rs
+++ b/crates/macro_authorization/src/outbound/pg_harness_authorization/test.rs
@@ -107,6 +107,89 @@ async fn finds_valid_token_owner_and_marks_usage(pool: PgPool) {
assert!(last_used_at.is_some());
}
+/// The token's last use as Unix seconds, so tests compare stored values exactly.
+async fn last_used_at(pool: &PgPool, token_id: Uuid) -> Option<f64> {
+ sqlx::query_scalar!(
+ r#"SELECT EXTRACT(EPOCH FROM last_used_at)::float8 AS "last_used_at" FROM harness_tokens WHERE id = $1"#,
+ token_id,
+ )
+ .fetch_one(pool)
+ .await
+ .unwrap()
+}
+
+async fn set_last_used_ago(pool: &PgPool, token_id: Uuid, seconds: f64) {
+ sqlx::query!(
+ r#"
+ UPDATE harness_tokens
+ SET last_used_at = now() - make_interval(secs => $2)
+ WHERE id = $1
+ "#,
+ token_id,
+ seconds,
+ )
+ .execute(pool)
+ .await
+ .unwrap();
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn a_use_within_the_last_minute_is_not_rewritten(pool: PgPool) {
+ let (_, token_id) = insert_harness_token(&pool, Some(USER_ID), None, false).await;
+ set_last_used_ago(&pool, token_id, 10.0).await;
+ let before = last_used_at(&pool, token_id).await;
+ let repo = PgHarnessAuthorizationRepo::new(pool.clone());
+
+ repo.mark_harness_token_used(token_id).await.unwrap();
+
+ assert_eq!(last_used_at(&pool, token_id).await, before);
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn a_use_older_than_a_minute_is_refreshed(pool: PgPool) {
+ let (_, token_id) = insert_harness_token(&pool, Some(USER_ID), None, false).await;
+ set_last_used_ago(&pool, token_id, 120.0).await;
+ let before = last_used_at(&pool, token_id)
+ .await
+ .expect("seeded timestamp");
+ let repo = PgHarnessAuthorizationRepo::new(pool.clone());
+
+ repo.mark_harness_token_used(token_id).await.unwrap();
+
+ let after = last_used_at(&pool, token_id)
+ .await
+ .expect("refreshed timestamp");
+ assert!(after > before);
+}
+
+#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
+async fn a_token_row_another_request_is_updating_does_not_block_the_request(pool: PgPool) {
+ let (_, token_id) = insert_harness_token(&pool, Some(USER_ID), None, false).await;
+ // Another request's update holds the row lock until its transaction ends.
+ let mut holder = pool.begin().await.unwrap();
+ sqlx::query!(
+ r#"SELECT id FROM harness_tokens WHERE id = $1 FOR UPDATE"#,
+ token_id
+ )
+ .fetch_one(&mut *holder)
+ .await
+ .unwrap();
+ // A waiting update fails after a second instead of hanging the test.
+ let options = pool
+ .connect_options()
+ .as_ref()
+ .clone()
+ .options([("lock_timeout", "1s")]);
+ let impatient = PgPool::connect_with(options).await.unwrap();
+ let repo = PgHarnessAuthorizationRepo::new(impatient);
+
+ repo.mark_harness_token_used(token_id)
+ .await
+ .expect("marking a token used must not wait for another request's row lock");
+
+ holder.rollback().await.unwrap();
+}
+
#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]
async fn team_owned_harness_reports_its_team(pool: PgPool) {
insert_user(&pool).await;