Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
fix(storage): expire docx part reference counters (#9115)
Wolf Mermelsteinf8d2f9b5aebb
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.jsonindex bb438836fc..84d53cfbab 100644--- a/.github/workspace-dep-closures.json+++ b/.github/workspace-dep-closures.json@@ -12289,10 +12289,38 @@"crates/workspace-hack"],"sha_cleanup_worker": [+ "crates/agent",+ "crates/ai_toolset",+ "crates/ai_usage",+ "crates/attachment",+ "crates/bot_id",+ "crates/bot_token",+ "crates/cowlike",+ "crates/document_sub_type",+ "crates/genai_telemetry",+ "crates/harness_id",+ "crates/harness_token",+ "crates/macro_auth",+ "crates/macro_authorization","crates/macro_aws_config","crates/macro_entrypoint","crates/macro_env","crates/macro_env_var",+ "crates/macro_sha_count_client",+ "crates/macro_user_id",+ "crates/macro_uuid",+ "crates/model",+ "crates/model-entity",+ "crates/model-error-response",+ "crates/model_file_type",+ "crates/model_owner",+ "crates/model_user",+ "crates/models_bulk_upload",+ "crates/models_pagination",+ "crates/models_permissions",+ "crates/non_empty",+ "crates/prompt",+ "crates/remote_env_var","crates/workspace-hack","services/sha_cleanup_worker"],diff --git a/Cargo.lock b/Cargo.lockindex f2af8566c9..a5d8661ba1 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -12369,6 +12369,7 @@ name = "macro_sha_count_client"version = "0.1.0"dependencies = ["anyhow",+ "macro_env_var","model","redis","tokio",@@ -18204,6 +18205,7 @@ dependencies = ["macro_entrypoint","macro_env","macro_env_var",+ "macro_sha_count_client","mockall","redis","sqlx",diff --git a/crates/macro_sha_count_client/AGENTS.md b/crates/macro_sha_count_client/AGENTS.mdindex d72ea75c2f..57ac7af2cb 100644--- a/crates/macro_sha_count_client/AGENTS.md+++ b/crates/macro_sha_count_client/AGENTS.md@@ -16,7 +16,9 @@ reference is gone and it's safe to delete.## Scope- Owns: reference counting for file content SHAs and the `bucket:sha-delete`- tracking used to find SHAs ready for cleanup.+ tracking used to find SHAs ready for cleanup. Counters are a cache, not the+ source of truth: `"BomPart"` rows are, and `sha_cleanup_worker` recounts+ them before deleting anything.- Not here: actually deleting the underlying blob storage — this crate onlytracks the reference count.@@ -28,10 +30,17 @@ reference is gone and it's safe to delete.## Checks-- `cargo test -p macro_sha_count_client` (`SQLX_OFFLINE` unset).+- `cargo test -p macro_sha_count_client` (needs Redis at `REDIS_URI`, default+ `redis://127.0.0.1:6379`).## Gotchas+- Every `sha:` counter expires `SHA_REFERENCE_TTL` (30 days) after the last+ increment. Set the expiry in the same MULTI/EXEC as any write that can+ create a counter, or the key lives forever in the shared cache, where+ volatile-lru cannot evict it (the 2026-10-28 OOM). A missing counter only+ makes the next upload re-send that part.+- `release_shas` collapses a list with duplicate SHAs into per-SHA countsbefore decrementing (`reference_counts`) — releasing the same SHA twice inone call removes two references, not one, so callers must pass every BOMdiff --git a/crates/macro_sha_count_client/Cargo.toml b/crates/macro_sha_count_client/Cargo.tomlindex 85fd7e0e03..72dfa9cff8 100644--- a/crates/macro_sha_count_client/Cargo.toml+++ b/crates/macro_sha_count_client/Cargo.toml@@ -17,3 +17,7 @@ redis = { workspace = true, features = [tokio = { workspace = true }tracing = { workspace = true }workspace-hack = { version = "0.1", path = "../workspace-hack" }++[dev-dependencies]+macro_env_var = { path = "../macro_env_var" }+tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }diff --git a/crates/macro_sha_count_client/src/increment_count.rs b/crates/macro_sha_count_client/src/increment_count.rsindex 4d88f4eb88..a159cba1c1 100644--- a/crates/macro_sha_count_client/src/increment_count.rs+++ b/crates/macro_sha_count_client/src/increment_count.rs@@ -1,32 +1,38 @@-use super::{SHA_COUNT_KEY_PREFIX, SHA_DELETE_BUCKET};+use super::{SHA_COUNT_KEY_PREFIX, SHA_DELETE_BUCKET, SHA_REFERENCE_TTL};use anyhow::Context;-use redis::AsyncCommands;+/// Adds one reference to each SHA, restarts its expiry, and takes it out of+/// the delete bucket, all in one MULTI/EXEC so a counter never exists without+/// an expiry.+#[tracing::instrument(skip_all, fields(sha_count = shas.len()), err)]pub(crate) async fn increment_counts(client: &redis::Client,shas: Vec<String>,) -> anyhow::Result<()> {+ if shas.is_empty() {+ return Ok(());+ }let mut redis_connection = client.get_multiplexed_async_connection().await.context("unable to connect to redis")?;- for sha in shas {- let key = format!("{}{}", SHA_COUNT_KEY_PREFIX, sha);- // increments sha key by 1- redis_connection- .incr::<&str, i32, i32>(key.as_str(), 1)- .await- .map_err(anyhow::Error::from)?;-- // removes the key from the delete bucket in case it is in there- redis_connection- .srem::<&str, &str, bool>(SHA_DELETE_BUCKET, sha.as_str())- .await- .map_err(anyhow::Error::from)?;+ let mut pipeline = redis::pipe();+ pipeline.atomic();+ for sha in &shas {+ let key = format!("{SHA_COUNT_KEY_PREFIX}{sha}");+ pipeline+ .incr(&key, 1)+ .ignore()+ .expire(&key, SHA_REFERENCE_TTL.as_secs() as i64)+ .ignore()+ .srem(SHA_DELETE_BUCKET, sha)+ .ignore();}-- Ok(())+ pipeline+ .query_async::<()>(&mut redis_connection)+ .await+ .context("unable to increment sha counts")}#[cfg(test)]diff --git a/crates/macro_sha_count_client/src/lib.rs b/crates/macro_sha_count_client/src/lib.rsindex bc738b75d6..fe2f427862 100644--- a/crates/macro_sha_count_client/src/lib.rs+++ b/crates/macro_sha_count_client/src/lib.rs@@ -4,7 +4,7 @@ mod increment_count;#[cfg(test)]mod test;-use std::collections::BTreeMap;+use std::{collections::BTreeMap, time::Duration};use model::document::SaveBomPart;@@ -17,6 +17,13 @@ pub struct Redis {static SHA_COUNT_KEY_PREFIX: &str = "sha:";static SHA_DELETE_BUCKET: &str = "bucket:sha-delete";+/// How long a `sha:` counter lives after the last save that referenced it.+/// The counter only lets an upload skip parts that are already in S3, so an+/// expired one costs a re-upload; `sha_cleanup_worker` recounts `"BomPart"`+/// before deleting anything. The expiry also lets volatile-lru evict counters+/// when the shared cache is full.+pub const SHA_REFERENCE_TTL: Duration = Duration::from_secs(30 * 24 * 60 * 60);+impl Redis {pub fn new(inner: redis::Client) -> Self {Self { inner }diff --git a/crates/macro_sha_count_client/src/test.rs b/crates/macro_sha_count_client/src/test.rsindex bfb90068fa..1f97a3913d 100644--- a/crates/macro_sha_count_client/src/test.rs+++ b/crates/macro_sha_count_client/src/test.rs@@ -13,3 +13,32 @@ fn each_listing_of_a_sha_is_one_reference() {fn no_shas_release_nothing() {assert!(reference_counts(Vec::new()).is_empty());}++#[tokio::test]+async fn an_increment_restarts_the_counter_expiry() -> anyhow::Result<()> {+ use redis::AsyncCommands;++ let url = macro_env_var::optional_read_env_var("REDIS_URI")?+ .unwrap_or_else(|| "redis://127.0.0.1:6379".to_owned());+ let client = redis::Client::open(url)?;+ let mut connection = client.get_multiplexed_async_connection().await?;+ let sha = "an_increment_restarts_the_counter_expiry";+ let key = format!("sha:{sha}");+ connection.set_ex::<_, _, ()>(&key, 2, 60).await?;+ connection+ .sadd::<_, _, ()>("bucket:sha-delete", sha)+ .await?;++ super::Redis::new(client)+ .increment_counts(vec![sha.to_owned()])+ .await?;++ let count: i64 = connection.get(&key).await?;+ let ttl: i64 = connection.ttl(&key).await?;+ let queued: bool = connection.sismember("bucket:sha-delete", sha).await?;+ connection.del::<_, ()>(&key).await?;+ assert_eq!(count, 3);+ assert!(ttl > 60 && ttl <= super::SHA_REFERENCE_TTL.as_secs() as i64);+ assert!(!queued);+ Ok(())+}diff --git a/services/sha_cleanup_worker/AGENTS.md b/services/sha_cleanup_worker/AGENTS.mdindex 9d7f26a7af..f9ec558510 100644--- a/services/sha_cleanup_worker/AGENTS.md+++ b/services/sha_cleanup_worker/AGENTS.md@@ -4,12 +4,13 @@Shared infrastructure for document/attachment storage: a periodic workerthat deletes S3 objects whose SHA was placed in the `bucket:sha-delete`-Redis set, decrementing a shared reference count first.+Redis set once no `"BomPart"` row references them.## Start here-- `src/process.rs`: the cleanup loop — pulls SHAs from Redis, checks the- count, deletes from S3 when it hits zero.+- `src/process.rs`: the cleanup loop — pulls SHAs from Redis, recounts+ `"BomPart"`, deletes from S3 at zero, and otherwise writes the count back+ to the `sha:` counter.- `src/service/redis/{get_shas,increment,remove_set_member}.rs`: the Redisoperations this service makes.- `src/service/s3/delete.rs`, `src/service/db/get_sha_count.rs`: the S3 delete@@ -28,6 +29,9 @@ Redis set, decrementing a shared reference count first.## Gotchas+- Counters written back here take `macro_sha_count_client::SHA_REFERENCE_TTL`+ in the same MULTI/EXEC; never write a `sha:` key without an expiry.+- `cargo run --features local` bypasses all AWS calls and returns dummy data— use it only for wiring checks, not to verify real deletes.- Triggered on a schedule by `services/worker_trigger`'s ECS `RunTask`, not bydiff --git a/services/sha_cleanup_worker/Cargo.toml b/services/sha_cleanup_worker/Cargo.tomlindex 9ffafb1275..8e0f263c9b 100644--- a/services/sha_cleanup_worker/Cargo.toml+++ b/services/sha_cleanup_worker/Cargo.toml@@ -16,6 +16,7 @@ futures = { workspace = true }macro_entrypoint = { path = "../../crates/macro_entrypoint" }macro_env = { path = "../../crates/macro_env" }macro_env_var = { path = "../../crates/macro_env_var" }+macro_sha_count_client = { path = "../../crates/macro_sha_count_client" }mockall = { workspace = true }redis = { workspace = true, features = ["aio",diff --git a/services/sha_cleanup_worker/src/service/redis/increment.rs b/services/sha_cleanup_worker/src/service/redis/increment.rsindex f2ea3dd355..10c30c222d 100644--- a/services/sha_cleanup_worker/src/service/redis/increment.rs+++ b/services/sha_cleanup_worker/src/service/redis/increment.rs@@ -1,7 +1,8 @@use anyhow::Context;-use redis::AsyncCommands;+use macro_sha_count_client::SHA_REFERENCE_TTL;-/// Increments the value of a key by the given amount+/// Adds `value` to the counter and restarts its expiry in one MULTI/EXEC, so a+/// recounted SHA never leaves a counter without an expiry.#[tracing::instrument(skip(client))]pub(in crate::service::redis) async fn increment(client: &redis::Client,@@ -13,17 +14,13 @@ pub(in crate::service::redis) async fn increment(.await.context("unable to connect to redis")?;- // Set the key if it doesn't exist- redis_connection- .set_nx::<&str, i32, bool>(key, 0)+ redis::pipe()+ .atomic()+ .incr(key, value)+ .ignore()+ .expire(key, SHA_REFERENCE_TTL.as_secs() as i64)+ .ignore()+ .query_async::<()>(&mut redis_connection).await- .map_err(anyhow::Error::from)?;-- // increments sha key by 1- redis_connection- .incr::<&str, i64, i32>(key, value)- .await- .map_err(anyhow::Error::from)?;-- Ok(())+ .context("unable to set sha count")}