Skip to content

armgit / macro

Public Imported from GitHub
0

Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.

fix(storage): expire docx part reference counters (#9115)

Wolf Mermelsteinf8d2f9b5aebb
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.json
index bb438836fc..84d53cfbab 100644
--- a/.github/workspace-dep-closures.json
+++ b/.github/workspace-dep-closures.json
@@ -12289,10 +12289,38 @@
"crates/workspace-hack"
],
"sha_cleanup_worker": [
+ "crates/agent",
+ "crates/ai_toolset",
+ "crates/ai_usage",
+ "crates/attachment",
+ "crates/bot_id",
+ "crates/bot_token",
+ "crates/cowlike",
+ "crates/document_sub_type",
+ "crates/genai_telemetry",
+ "crates/harness_id",
+ "crates/harness_token",
+ "crates/macro_auth",
+ "crates/macro_authorization",
"crates/macro_aws_config",
"crates/macro_entrypoint",
"crates/macro_env",
"crates/macro_env_var",
+ "crates/macro_sha_count_client",
+ "crates/macro_user_id",
+ "crates/macro_uuid",
+ "crates/model",
+ "crates/model-entity",
+ "crates/model-error-response",
+ "crates/model_file_type",
+ "crates/model_owner",
+ "crates/model_user",
+ "crates/models_bulk_upload",
+ "crates/models_pagination",
+ "crates/models_permissions",
+ "crates/non_empty",
+ "crates/prompt",
+ "crates/remote_env_var",
"crates/workspace-hack",
"services/sha_cleanup_worker"
],
diff --git a/Cargo.lock b/Cargo.lock
index f2af8566c9..a5d8661ba1 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -12369,6 +12369,7 @@ name = "macro_sha_count_client"
version = "0.1.0"
dependencies = [
"anyhow",
+ "macro_env_var",
"model",
"redis",
"tokio",
@@ -18204,6 +18205,7 @@ dependencies = [
"macro_entrypoint",
"macro_env",
"macro_env_var",
+ "macro_sha_count_client",
"mockall",
"redis",
"sqlx",
diff --git a/crates/macro_sha_count_client/AGENTS.md b/crates/macro_sha_count_client/AGENTS.md
index d72ea75c2f..57ac7af2cb 100644
--- a/crates/macro_sha_count_client/AGENTS.md
+++ b/crates/macro_sha_count_client/AGENTS.md
@@ -16,7 +16,9 @@ reference is gone and it's safe to delete.
## Scope
- Owns: reference counting for file content SHAs and the `bucket:sha-delete`
- tracking used to find SHAs ready for cleanup.
+ tracking used to find SHAs ready for cleanup. Counters are a cache, not the
+ source of truth: `"BomPart"` rows are, and `sha_cleanup_worker` recounts
+ them before deleting anything.
- Not here: actually deleting the underlying blob storage — this crate only
tracks the reference count.
@@ -28,10 +30,17 @@ reference is gone and it's safe to delete.
## Checks
-- `cargo test -p macro_sha_count_client` (`SQLX_OFFLINE` unset).
+- `cargo test -p macro_sha_count_client` (needs Redis at `REDIS_URI`, default
+ `redis://127.0.0.1:6379`).
## Gotchas
+- Every `sha:` counter expires `SHA_REFERENCE_TTL` (30 days) after the last
+ increment. Set the expiry in the same MULTI/EXEC as any write that can
+ create a counter, or the key lives forever in the shared cache, where
+ volatile-lru cannot evict it (the 2026-10-28 OOM). A missing counter only
+ makes the next upload re-send that part.
+
- `release_shas` collapses a list with duplicate SHAs into per-SHA counts
before decrementing (`reference_counts`) — releasing the same SHA twice in
one call removes two references, not one, so callers must pass every BOM
diff --git a/crates/macro_sha_count_client/Cargo.toml b/crates/macro_sha_count_client/Cargo.toml
index 85fd7e0e03..72dfa9cff8 100644
--- a/crates/macro_sha_count_client/Cargo.toml
+++ b/crates/macro_sha_count_client/Cargo.toml
@@ -17,3 +17,7 @@ redis = { workspace = true, features = [
tokio = { workspace = true }
tracing = { workspace = true }
workspace-hack = { version = "0.1", path = "../workspace-hack" }
+
+[dev-dependencies]
+macro_env_var = { path = "../macro_env_var" }
+tokio = { workspace = true, features = ["macros", "rt-multi-thread"] }
diff --git a/crates/macro_sha_count_client/src/increment_count.rs b/crates/macro_sha_count_client/src/increment_count.rs
index 4d88f4eb88..a159cba1c1 100644
--- a/crates/macro_sha_count_client/src/increment_count.rs
+++ b/crates/macro_sha_count_client/src/increment_count.rs
@@ -1,32 +1,38 @@
-use super::{SHA_COUNT_KEY_PREFIX, SHA_DELETE_BUCKET};
+use super::{SHA_COUNT_KEY_PREFIX, SHA_DELETE_BUCKET, SHA_REFERENCE_TTL};
use anyhow::Context;
-use redis::AsyncCommands;
+/// Adds one reference to each SHA, restarts its expiry, and takes it out of
+/// the delete bucket, all in one MULTI/EXEC so a counter never exists without
+/// an expiry.
+#[tracing::instrument(skip_all, fields(sha_count = shas.len()), err)]
pub(crate) async fn increment_counts(
client: &redis::Client,
shas: Vec<String>,
) -> anyhow::Result<()> {
+ if shas.is_empty() {
+ return Ok(());
+ }
let mut redis_connection = client
.get_multiplexed_async_connection()
.await
.context("unable to connect to redis")?;
- for sha in shas {
- let key = format!("{}{}", SHA_COUNT_KEY_PREFIX, sha);
- // increments sha key by 1
- redis_connection
- .incr::<&str, i32, i32>(key.as_str(), 1)
- .await
- .map_err(anyhow::Error::from)?;
-
- // removes the key from the delete bucket in case it is in there
- redis_connection
- .srem::<&str, &str, bool>(SHA_DELETE_BUCKET, sha.as_str())
- .await
- .map_err(anyhow::Error::from)?;
+ let mut pipeline = redis::pipe();
+ pipeline.atomic();
+ for sha in &shas {
+ let key = format!("{SHA_COUNT_KEY_PREFIX}{sha}");
+ pipeline
+ .incr(&key, 1)
+ .ignore()
+ .expire(&key, SHA_REFERENCE_TTL.as_secs() as i64)
+ .ignore()
+ .srem(SHA_DELETE_BUCKET, sha)
+ .ignore();
}
-
- Ok(())
+ pipeline
+ .query_async::<()>(&mut redis_connection)
+ .await
+ .context("unable to increment sha counts")
}
#[cfg(test)]
diff --git a/crates/macro_sha_count_client/src/lib.rs b/crates/macro_sha_count_client/src/lib.rs
index bc738b75d6..fe2f427862 100644
--- a/crates/macro_sha_count_client/src/lib.rs
+++ b/crates/macro_sha_count_client/src/lib.rs
@@ -4,7 +4,7 @@ mod increment_count;
#[cfg(test)]
mod test;
-use std::collections::BTreeMap;
+use std::{collections::BTreeMap, time::Duration};
use model::document::SaveBomPart;
@@ -17,6 +17,13 @@ pub struct Redis {
static SHA_COUNT_KEY_PREFIX: &str = "sha:";
static SHA_DELETE_BUCKET: &str = "bucket:sha-delete";
+/// How long a `sha:` counter lives after the last save that referenced it.
+/// The counter only lets an upload skip parts that are already in S3, so an
+/// expired one costs a re-upload; `sha_cleanup_worker` recounts `"BomPart"`
+/// before deleting anything. The expiry also lets volatile-lru evict counters
+/// when the shared cache is full.
+pub const SHA_REFERENCE_TTL: Duration = Duration::from_secs(30 * 24 * 60 * 60);
+
impl Redis {
pub fn new(inner: redis::Client) -> Self {
Self { inner }
diff --git a/crates/macro_sha_count_client/src/test.rs b/crates/macro_sha_count_client/src/test.rs
index bfb90068fa..1f97a3913d 100644
--- a/crates/macro_sha_count_client/src/test.rs
+++ b/crates/macro_sha_count_client/src/test.rs
@@ -13,3 +13,32 @@ fn each_listing_of_a_sha_is_one_reference() {
fn no_shas_release_nothing() {
assert!(reference_counts(Vec::new()).is_empty());
}
+
+#[tokio::test]
+async fn an_increment_restarts_the_counter_expiry() -> anyhow::Result<()> {
+ use redis::AsyncCommands;
+
+ let url = macro_env_var::optional_read_env_var("REDIS_URI")?
+ .unwrap_or_else(|| "redis://127.0.0.1:6379".to_owned());
+ let client = redis::Client::open(url)?;
+ let mut connection = client.get_multiplexed_async_connection().await?;
+ let sha = "an_increment_restarts_the_counter_expiry";
+ let key = format!("sha:{sha}");
+ connection.set_ex::<_, _, ()>(&key, 2, 60).await?;
+ connection
+ .sadd::<_, _, ()>("bucket:sha-delete", sha)
+ .await?;
+
+ super::Redis::new(client)
+ .increment_counts(vec![sha.to_owned()])
+ .await?;
+
+ let count: i64 = connection.get(&key).await?;
+ let ttl: i64 = connection.ttl(&key).await?;
+ let queued: bool = connection.sismember("bucket:sha-delete", sha).await?;
+ connection.del::<_, ()>(&key).await?;
+ assert_eq!(count, 3);
+ assert!(ttl > 60 && ttl <= super::SHA_REFERENCE_TTL.as_secs() as i64);
+ assert!(!queued);
+ Ok(())
+}
diff --git a/services/sha_cleanup_worker/AGENTS.md b/services/sha_cleanup_worker/AGENTS.md
index 9d7f26a7af..f9ec558510 100644
--- a/services/sha_cleanup_worker/AGENTS.md
+++ b/services/sha_cleanup_worker/AGENTS.md
@@ -4,12 +4,13 @@
Shared infrastructure for document/attachment storage: a periodic worker
that deletes S3 objects whose SHA was placed in the `bucket:sha-delete`
-Redis set, decrementing a shared reference count first.
+Redis set once no `"BomPart"` row references them.
## Start here
-- `src/process.rs`: the cleanup loop — pulls SHAs from Redis, checks the
- count, deletes from S3 when it hits zero.
+- `src/process.rs`: the cleanup loop — pulls SHAs from Redis, recounts
+ `"BomPart"`, deletes from S3 at zero, and otherwise writes the count back
+ to the `sha:` counter.
- `src/service/redis/{get_shas,increment,remove_set_member}.rs`: the Redis
operations this service makes.
- `src/service/s3/delete.rs`, `src/service/db/get_sha_count.rs`: the S3 delete
@@ -28,6 +29,9 @@ Redis set, decrementing a shared reference count first.
## Gotchas
+- Counters written back here take `macro_sha_count_client::SHA_REFERENCE_TTL`
+ in the same MULTI/EXEC; never write a `sha:` key without an expiry.
+
- `cargo run --features local` bypasses all AWS calls and returns dummy data
— use it only for wiring checks, not to verify real deletes.
- Triggered on a schedule by `services/worker_trigger`'s ECS `RunTask`, not by
diff --git a/services/sha_cleanup_worker/Cargo.toml b/services/sha_cleanup_worker/Cargo.toml
index 9ffafb1275..8e0f263c9b 100644
--- a/services/sha_cleanup_worker/Cargo.toml
+++ b/services/sha_cleanup_worker/Cargo.toml
@@ -16,6 +16,7 @@ futures = { workspace = true }
macro_entrypoint = { path = "../../crates/macro_entrypoint" }
macro_env = { path = "../../crates/macro_env" }
macro_env_var = { path = "../../crates/macro_env_var" }
+macro_sha_count_client = { path = "../../crates/macro_sha_count_client" }
mockall = { workspace = true }
redis = { workspace = true, features = [
"aio",
diff --git a/services/sha_cleanup_worker/src/service/redis/increment.rs b/services/sha_cleanup_worker/src/service/redis/increment.rs
index f2ea3dd355..10c30c222d 100644
--- a/services/sha_cleanup_worker/src/service/redis/increment.rs
+++ b/services/sha_cleanup_worker/src/service/redis/increment.rs
@@ -1,7 +1,8 @@
use anyhow::Context;
-use redis::AsyncCommands;
+use macro_sha_count_client::SHA_REFERENCE_TTL;
-/// Increments the value of a key by the given amount
+/// Adds `value` to the counter and restarts its expiry in one MULTI/EXEC, so a
+/// recounted SHA never leaves a counter without an expiry.
#[tracing::instrument(skip(client))]
pub(in crate::service::redis) async fn increment(
client: &redis::Client,
@@ -13,17 +14,13 @@ pub(in crate::service::redis) async fn increment(
.await
.context("unable to connect to redis")?;
- // Set the key if it doesn't exist
- redis_connection
- .set_nx::<&str, i32, bool>(key, 0)
+ redis::pipe()
+ .atomic()
+ .incr(key, value)
+ .ignore()
+ .expire(key, SHA_REFERENCE_TTL.as_secs() as i64)
+ .ignore()
+ .query_async::<()>(&mut redis_connection)
.await
- .map_err(anyhow::Error::from)?;
-
- // increments sha key by 1
- redis_connection
- .incr::<&str, i64, i32>(key, value)
- .await
- .map_err(anyhow::Error::from)?;
-
- Ok(())
+ .context("unable to set sha count")
}