Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(flags): push the last_called_at sync checkpoint as a timestamp gauge (#121672)

Phil Haackd37ffe1d5a04
diff --git a/posthog/tasks/tasks.py b/posthog/tasks/tasks.py
index c989d5aae2d1..e2079cd43187 100644
--- a/posthog/tasks/tasks.py
+++ b/posthog/tasks/tasks.py
@@ -1359,12 +1359,22 @@ def sync_feature_flag_last_called(self: PushGatewayTask) -> None:
"Seconds between checkpoint timestamp and current time",
registry=self.metrics_registry,
)
+ checkpoint_timestamp_gauge = Gauge(
+ "posthog_feature_flag_last_called_at_sync_checkpoint_timestamp_seconds",
+ "Unix time of the checkpoint the sync wrote to Redis",
+ registry=self.metrics_registry,
+ )
run_failed = False
try:
redis_client = get_client()
+ def save_checkpoint(checkpoint: datetime) -> None:
+ redis_client.set(FEATURE_FLAG_LAST_CALLED_SYNC_KEY, checkpoint.isoformat())
+ checkpoint_lag_gauge.set((timezone.now() - checkpoint).total_seconds())
+ checkpoint_timestamp_gauge.set(checkpoint.timestamp())
+
# Get last sync timestamp from Redis or use lookback. A Redis error on this read propagates so
# that Celery retries the run. The lookback fallback would rescan up to
# FEATURE_FLAG_LAST_CALLED_AT_SYNC_MAX_LOOKBACK_HOURS of chunks that the checkpoint already covers.
@@ -1569,13 +1579,12 @@ def sync_feature_flag_last_called(self: PushGatewayTask) -> None:
if not merged_results:
# Advance the checkpoint even when no results, so the next run starts from the
# end of the window that was read instead of rescanning it
- redis_client.set(FEATURE_FLAG_LAST_CALLED_SYNC_KEY, checkpoint_timestamp.isoformat())
+ save_checkpoint(checkpoint_timestamp)
# Emit metrics for no-results case
updated_count_gauge.set(0)
events_processed_gauge.set(0)
clickhouse_results_gauge.set(0)
- checkpoint_lag_gauge.set((timezone.now() - checkpoint_timestamp).total_seconds())
logger.info(
"Feature flag sync completed with no events",
@@ -1593,7 +1602,7 @@ def sync_feature_flag_last_called(self: PushGatewayTask) -> None:
flag_updates[(team_id, flag_key)] = ts
if not flag_updates:
- redis_client.set(FEATURE_FLAG_LAST_CALLED_SYNC_KEY, checkpoint_timestamp.isoformat())
+ save_checkpoint(checkpoint_timestamp)
logger.info(
"Feature flag sync: no valid timestamps to update",
chunks_processed=len(chunks),
@@ -1601,7 +1610,6 @@ def sync_feature_flag_last_called(self: PushGatewayTask) -> None:
clickhouse_results_gauge.set(total_clickhouse_results)
updated_count_gauge.set(0)
events_processed_gauge.set(0)
- checkpoint_lag_gauge.set((timezone.now() - checkpoint_timestamp).total_seconds())
return
# Fetch flags matching any (team_id, key) combination from updates.
@@ -1644,17 +1652,15 @@ def sync_feature_flag_last_called(self: PushGatewayTask) -> None:
raise
# Set final checkpoint to the end of the last window that was read successfully
- redis_client.set(FEATURE_FLAG_LAST_CALLED_SYNC_KEY, checkpoint_timestamp.isoformat())
+ save_checkpoint(checkpoint_timestamp)
duration = (timezone.now() - start_time).total_seconds()
processed_events = sum(count for _ts, count in merged_results.values())
# Emit metrics for successful completion
- checkpoint_lag_seconds = (timezone.now() - checkpoint_timestamp).total_seconds()
updated_count_gauge.set(updated_count)
events_processed_gauge.set(processed_events)
clickhouse_results_gauge.set(total_clickhouse_results)
- checkpoint_lag_gauge.set(checkpoint_lag_seconds)
logger.info(
"Feature flag sync completed",
diff --git a/posthog/tasks/test/utils.py b/posthog/tasks/test/utils.py
index 7c652db982ac..73041af60a16 100644
--- a/posthog/tasks/test/utils.py
+++ b/posthog/tasks/test/utils.py
@@ -1,20 +1,29 @@
-from unittest.mock import MagicMock, patch
+from collections.abc import Iterator
+from contextlib import contextmanager
+
+from unittest.mock import patch
from prometheus_client import CollectorRegistry
class PushGatewayTaskTestMixin:
- """Sets up mocked PushGateway context so PushGatewayTask-based tasks can run in tests."""
+ """Sets up mocked PushGateway context so PushGatewayTask-based tasks can run in tests.
+
+ Each task run gets a fresh registry, as in production. `self.registry` holds the registry of the latest run.
+ """
+
+ registry: CollectorRegistry
def setUp(self) -> None:
super().setUp() # type: ignore[misc]
- self.registry = CollectorRegistry()
- self.mock_context = MagicMock()
- self.mock_context.__enter__ = MagicMock(return_value=self.registry)
- self.mock_context.__exit__ = MagicMock(return_value=False)
- self.patcher = patch("posthog.tasks.utils.pushed_metrics_registry", return_value=self.mock_context)
+ self.patcher = patch("posthog.tasks.utils.pushed_metrics_registry", self._pushed_metrics_registry)
self.patcher.start()
+ @contextmanager
+ def _pushed_metrics_registry(self, _job_name: str) -> Iterator[CollectorRegistry]:
+ self.registry = CollectorRegistry()
+ yield self.registry
+
def tearDown(self) -> None:
self.patcher.stop()
super().tearDown() # type: ignore[misc]
diff --git a/products/feature_flags/backend/test/test_feature_flag_sync.py b/products/feature_flags/backend/test/test_feature_flag_sync.py
index 028009eeeea2..e15caa740341 100644
--- a/products/feature_flags/backend/test/test_feature_flag_sync.py
+++ b/products/feature_flags/backend/test/test_feature_flag_sync.py
@@ -12,7 +12,7 @@ from django.utils import timezone as tz
from clickhouse_driver.errors import UnknownPacketFromServerError
from parameterized import parameterized
-from prometheus_client import REGISTRY
+from prometheus_client import REGISTRY, CollectorRegistry
from redis.exceptions import ConnectionError as RedisConnectionError
from posthog.clickhouse.client import sync_execute
@@ -25,6 +25,7 @@ from posthog.models.flag_evaluations.sql import (
FLAG_EVALUATIONS_WRITABLE_TABLE,
)
from posthog.tasks.tasks import sync_feature_flag_last_called
+from posthog.tasks.test.utils import PushGatewayTaskTestMixin
from products.feature_flags.backend.models.feature_flag import FeatureFlag
@@ -39,12 +40,19 @@ def mock_redis_client() -> Mock:
return mock
+def assert_checkpoint_gauge_matches_stored_checkpoint(registry: CollectorRegistry, stored: bytes) -> None:
+ assert (
+ registry.get_sample_value("posthog_feature_flag_last_called_at_sync_checkpoint_timestamp_seconds")
+ == datetime.fromisoformat(stored.decode()).timestamp()
+ )
+
+
# Force single-chunk behavior for existing tests so they don't need to handle multiple calls
@override_settings(
FEATURE_FLAG_LAST_CALLED_AT_SYNC_CHUNK_MINUTES=1440,
FEATURE_FLAG_LAST_CALLED_AT_SYNC_MAX_LOOKBACK_HOURS=24,
)
-class TestSyncFeatureFlagLastCalled(BaseTest):
+class TestSyncFeatureFlagLastCalled(PushGatewayTaskTestMixin, BaseTest):
def setUp(self) -> None:
super().setUp()
# Clear Redis cache before each test
@@ -88,6 +96,7 @@ class TestSyncFeatureFlagLastCalled(BaseTest):
# Verify checkpoint was updated in Redis
checkpoint_key = "posthog:feature_flag_last_called_sync:last_timestamp"
assert checkpoint_key in redis_mock.storage
+ assert_checkpoint_gauge_matches_stored_checkpoint(self.registry, redis_mock.storage[checkpoint_key])
# Verify no flags were updated
self.flag1.refresh_from_db()
@@ -417,6 +426,7 @@ class TestSyncFeatureFlagLastCalled(BaseTest):
assert stored_timestamp is not None
assert b"2024-06-15" in stored_timestamp
assert b"2024-06-15T11:59:00" in stored_timestamp
+ assert_checkpoint_gauge_matches_stored_checkpoint(self.registry, stored_timestamp)
@time_machine.travel("2024-06-15 12:00:00", tick=False)
@patch("posthog.clickhouse.client.sync_execute")
@@ -471,7 +481,7 @@ class TestSyncFeatureFlagLastCalled(BaseTest):
FEATURE_FLAG_LAST_CALLED_AT_SYNC_CHUNK_MINUTES=5,
FEATURE_FLAG_LAST_CALLED_AT_SYNC_MAX_LOOKBACK_HOURS=6,
)
-class TestSyncFeatureFlagLastCalledChunking(BaseTest):
+class TestSyncFeatureFlagLastCalledChunking(PushGatewayTaskTestMixin, BaseTest):
def setUp(self) -> None:
super().setUp()
cache.clear()
@@ -551,6 +561,7 @@ class TestSyncFeatureFlagLastCalledChunking(BaseTest):
# onwards is retried next run rather than skipped
stored = redis_mock.storage.get(checkpoint_key)
assert stored == tz.make_aware(datetime(2024, 6, 15, 11, 50, 0)).isoformat().encode()
+ assert_checkpoint_gauge_matches_stored_checkpoint(self.registry, stored)
@time_machine.travel("2024-06-15 12:00:00", tick=False)
@patch("posthog.clickhouse.client.sync_execute")
@@ -696,6 +707,7 @@ class TestSyncFeatureFlagLastCalledChunking(BaseTest):
stored = redis_mock.storage.get(checkpoint_key)
assert stored is not None
assert b"2024-06-15T11:59:00" in stored
+ assert_checkpoint_gauge_matches_stored_checkpoint(self.registry, stored)
# Flag should remain unchanged
self.flag1.refresh_from_db()