Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(query-cache): prune tracking of expired entries on write (#127095)

Javier Bahamondes610555acc276
diff --git a/posthog/query_cache/size_tracker.py b/posthog/query_cache/size_tracker.py
index 1a08c2f205e2..aeba52abe0ce 100644
--- a/posthog/query_cache/size_tracker.py
+++ b/posthog/query_cache/size_tracker.py
@@ -71,6 +71,11 @@ CACHE_SIZE_HISTOGRAM = Histogram(
],
)
+# Expiry scores come from the writing host's clock, so keep a margin for clock skew between hosts.
+TRACKING_PRUNE_GRACE_SECONDS = 60
+# Bounds the work of one write. A backlog drains over the next writes.
+TRACKING_PRUNE_BATCH_SIZE = 100
+
# Lua script for atomic cache write tracking
# Handles overwrite detection and counter updates in a single atomic operation
TRACK_CACHE_WRITE_SCRIPT = """
@@ -82,6 +87,20 @@ local cache_key = ARGV[1]
local size_bytes = tonumber(ARGV[2])
local expires_at = tonumber(ARGV[3])
local tracking_ttl = tonumber(ARGV[4])
+local prune_before = tonumber(ARGV[5])
+local prune_batch = tonumber(ARGV[6])
+
+-- The structure TTL refreshes on every write, so without this a team under its limit keeps
+-- tracking every cache_key it ever wrote, long after the entries expired.
+local expired = redis.call('ZRANGEBYSCORE', entries_key, '-inf', prune_before, 'LIMIT', 0, prune_batch)
+for _, expired_key in ipairs(expired) do
+ local expired_size = redis.call('HGET', sizes_key, expired_key)
+ if expired_size then
+ redis.call('HDEL', sizes_key, expired_key)
+ redis.call('INCRBY', total_key, -tonumber(expired_size))
+ end
+ redis.call('ZREM', entries_key, expired_key)
+end
-- Atomically handle overwrite: only decrement if key exists
local old_size = redis.call('HGET', sizes_key, cache_key)
@@ -210,7 +229,7 @@ class TeamCacheSizeTracker:
Tracks cache size per team using Redis data structures.
Redis keys used:
- - posthog:cache_sizes:{team_id} - Sorted set: member=cache_key, score=timestamp (for LRU ordering)
+ - posthog:cache_sizes:{team_id} - Sorted set: member=cache_key, score=entry expiry time (eviction and pruning order)
- posthog:cache_entry_sizes:{team_id} - Hash: field=cache_key, value=size (for O(1) size lookup)
- posthog:cache_total:{team_id} - String counter: total bytes (for O(1) total size lookup)
"""
@@ -299,9 +318,17 @@ class TeamCacheSizeTracker:
def track_cache_write(self, cache_key: str, size_bytes: int, applied_ttl: int) -> "TeamCacheTotals":
"""Track a cache write with its size, returning the team's totals after it. Atomic via Lua script."""
tracking_ttl = settings.CACHED_RESULTS_TTL + 86400
+ now = time.time()
total_bytes, entry_count = self._track_write_script(
keys=[self.entries_key, self.sizes_key, self.total_key],
- args=[cache_key, size_bytes, time.time() + applied_ttl, tracking_ttl],
+ args=[
+ cache_key,
+ size_bytes,
+ now + applied_ttl,
+ tracking_ttl,
+ now - TRACKING_PRUNE_GRACE_SECONDS,
+ TRACKING_PRUNE_BATCH_SIZE,
+ ],
)
return TeamCacheTotals(total_bytes=int(total_bytes), entry_count=int(entry_count))
diff --git a/posthog/query_cache/test/test_size_tracker.py b/posthog/query_cache/test/test_size_tracker.py
index 1b6f75a32d15..031bdda11598 100644
--- a/posthog/query_cache/test/test_size_tracker.py
+++ b/posthog/query_cache/test/test_size_tracker.py
@@ -62,6 +62,21 @@ class TestTeamCacheSizeTracker(BaseTest):
self.tracker.track_cache_write("test_key_1", 500, 300)
self.assertEqual(self.tracker.get_total_size(), 500)
+ @parameterized.expand([("expired_past_grace", -3600, False), ("expired_within_grace", -30, True)])
+ def test_track_cache_write_prunes_tracking_of_expired_entries(
+ self, _name: str, expired_ttl: int, still_tracked: bool
+ ) -> None:
+ self.tracker.track_cache_write("expired_key", 1000, expired_ttl)
+ self.tracker.track_cache_write("real_key", 500, 300)
+
+ self.assertEqual(
+ self.tracker.redis_client.zscore(self.tracker.entries_key, "expired_key") is not None, still_tracked
+ )
+ self.assertEqual(
+ self.tracker.redis_client.hget(self.tracker.sizes_key, "expired_key") is not None, still_tracked
+ )
+ self.assertEqual(self.tracker.get_total_size(), 1500 if still_tracked else 500)
+
def test_get_total_size_returns_correct_value(self):
self.assertEqual(self.tracker.get_total_size(), 0)