:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(flags): push the last_called_at sync checkpoint as a timestamp gauge (#121672)
Phil Haackd37ffe1d5a04
diff --git a/posthog/tasks/tasks.py b/posthog/tasks/tasks.pyindex c989d5aae2d1..e2079cd43187 100644--- a/posthog/tasks/tasks.py+++ b/posthog/tasks/tasks.py@@ -1359,12 +1359,22 @@ def sync_feature_flag_last_called(self: PushGatewayTask) -> None:"Seconds between checkpoint timestamp and current time",registry=self.metrics_registry,)+ checkpoint_timestamp_gauge = Gauge(+ "posthog_feature_flag_last_called_at_sync_checkpoint_timestamp_seconds",+ "Unix time of the checkpoint the sync wrote to Redis",+ registry=self.metrics_registry,+ )run_failed = Falsetry:redis_client = get_client()+ def save_checkpoint(checkpoint: datetime) -> None:+ redis_client.set(FEATURE_FLAG_LAST_CALLED_SYNC_KEY, checkpoint.isoformat())+ checkpoint_lag_gauge.set((timezone.now() - checkpoint).total_seconds())+ checkpoint_timestamp_gauge.set(checkpoint.timestamp())+# Get last sync timestamp from Redis or use lookback. A Redis error on this read propagates so# that Celery retries the run. The lookback fallback would rescan up to# FEATURE_FLAG_LAST_CALLED_AT_SYNC_MAX_LOOKBACK_HOURS of chunks that the checkpoint already covers.@@ -1569,13 +1579,12 @@ def sync_feature_flag_last_called(self: PushGatewayTask) -> None:if not merged_results:# Advance the checkpoint even when no results, so the next run starts from the# end of the window that was read instead of rescanning it- redis_client.set(FEATURE_FLAG_LAST_CALLED_SYNC_KEY, checkpoint_timestamp.isoformat())+ save_checkpoint(checkpoint_timestamp)# Emit metrics for no-results caseupdated_count_gauge.set(0)events_processed_gauge.set(0)clickhouse_results_gauge.set(0)- checkpoint_lag_gauge.set((timezone.now() - checkpoint_timestamp).total_seconds())logger.info("Feature flag sync completed with no events",@@ -1593,7 +1602,7 @@ def sync_feature_flag_last_called(self: PushGatewayTask) -> None:flag_updates[(team_id, flag_key)] = tsif not flag_updates:- redis_client.set(FEATURE_FLAG_LAST_CALLED_SYNC_KEY, checkpoint_timestamp.isoformat())+ save_checkpoint(checkpoint_timestamp)logger.info("Feature flag sync: no valid timestamps to update",chunks_processed=len(chunks),@@ -1601,7 +1610,6 @@ def sync_feature_flag_last_called(self: PushGatewayTask) -> None:clickhouse_results_gauge.set(total_clickhouse_results)updated_count_gauge.set(0)events_processed_gauge.set(0)- checkpoint_lag_gauge.set((timezone.now() - checkpoint_timestamp).total_seconds())return# Fetch flags matching any (team_id, key) combination from updates.@@ -1644,17 +1652,15 @@ def sync_feature_flag_last_called(self: PushGatewayTask) -> None:raise# Set final checkpoint to the end of the last window that was read successfully- redis_client.set(FEATURE_FLAG_LAST_CALLED_SYNC_KEY, checkpoint_timestamp.isoformat())+ save_checkpoint(checkpoint_timestamp)duration = (timezone.now() - start_time).total_seconds()processed_events = sum(count for _ts, count in merged_results.values())# Emit metrics for successful completion- checkpoint_lag_seconds = (timezone.now() - checkpoint_timestamp).total_seconds()updated_count_gauge.set(updated_count)events_processed_gauge.set(processed_events)clickhouse_results_gauge.set(total_clickhouse_results)- checkpoint_lag_gauge.set(checkpoint_lag_seconds)logger.info("Feature flag sync completed",diff --git a/posthog/tasks/test/utils.py b/posthog/tasks/test/utils.pyindex 7c652db982ac..73041af60a16 100644--- a/posthog/tasks/test/utils.py+++ b/posthog/tasks/test/utils.py@@ -1,20 +1,29 @@-from unittest.mock import MagicMock, patch+from collections.abc import Iterator+from contextlib import contextmanager++from unittest.mock import patchfrom prometheus_client import CollectorRegistryclass PushGatewayTaskTestMixin:- """Sets up mocked PushGateway context so PushGatewayTask-based tasks can run in tests."""+ """Sets up mocked PushGateway context so PushGatewayTask-based tasks can run in tests.++ Each task run gets a fresh registry, as in production. `self.registry` holds the registry of the latest run.+ """++ registry: CollectorRegistrydef setUp(self) -> None:super().setUp() # type: ignore[misc]- self.registry = CollectorRegistry()- self.mock_context = MagicMock()- self.mock_context.__enter__ = MagicMock(return_value=self.registry)- self.mock_context.__exit__ = MagicMock(return_value=False)- self.patcher = patch("posthog.tasks.utils.pushed_metrics_registry", return_value=self.mock_context)+ self.patcher = patch("posthog.tasks.utils.pushed_metrics_registry", self._pushed_metrics_registry)self.patcher.start()+ @contextmanager+ def _pushed_metrics_registry(self, _job_name: str) -> Iterator[CollectorRegistry]:+ self.registry = CollectorRegistry()+ yield self.registry+def tearDown(self) -> None:self.patcher.stop()super().tearDown() # type: ignore[misc]diff --git a/products/feature_flags/backend/test/test_feature_flag_sync.py b/products/feature_flags/backend/test/test_feature_flag_sync.pyindex 028009eeeea2..e15caa740341 100644--- a/products/feature_flags/backend/test/test_feature_flag_sync.py+++ b/products/feature_flags/backend/test/test_feature_flag_sync.py@@ -12,7 +12,7 @@ from django.utils import timezone as tzfrom clickhouse_driver.errors import UnknownPacketFromServerErrorfrom parameterized import parameterized-from prometheus_client import REGISTRY+from prometheus_client import REGISTRY, CollectorRegistryfrom redis.exceptions import ConnectionError as RedisConnectionErrorfrom posthog.clickhouse.client import sync_execute@@ -25,6 +25,7 @@ from posthog.models.flag_evaluations.sql import (FLAG_EVALUATIONS_WRITABLE_TABLE,)from posthog.tasks.tasks import sync_feature_flag_last_called+from posthog.tasks.test.utils import PushGatewayTaskTestMixinfrom products.feature_flags.backend.models.feature_flag import FeatureFlag@@ -39,12 +40,19 @@ def mock_redis_client() -> Mock:return mock+def assert_checkpoint_gauge_matches_stored_checkpoint(registry: CollectorRegistry, stored: bytes) -> None:+ assert (+ registry.get_sample_value("posthog_feature_flag_last_called_at_sync_checkpoint_timestamp_seconds")+ == datetime.fromisoformat(stored.decode()).timestamp()+ )++# Force single-chunk behavior for existing tests so they don't need to handle multiple calls@override_settings(FEATURE_FLAG_LAST_CALLED_AT_SYNC_CHUNK_MINUTES=1440,FEATURE_FLAG_LAST_CALLED_AT_SYNC_MAX_LOOKBACK_HOURS=24,)-class TestSyncFeatureFlagLastCalled(BaseTest):+class TestSyncFeatureFlagLastCalled(PushGatewayTaskTestMixin, BaseTest):def setUp(self) -> None:super().setUp()# Clear Redis cache before each test@@ -88,6 +96,7 @@ class TestSyncFeatureFlagLastCalled(BaseTest):# Verify checkpoint was updated in Redischeckpoint_key = "posthog:feature_flag_last_called_sync:last_timestamp"assert checkpoint_key in redis_mock.storage+ assert_checkpoint_gauge_matches_stored_checkpoint(self.registry, redis_mock.storage[checkpoint_key])# Verify no flags were updatedself.flag1.refresh_from_db()@@ -417,6 +426,7 @@ class TestSyncFeatureFlagLastCalled(BaseTest):assert stored_timestamp is not Noneassert b"2024-06-15" in stored_timestampassert b"2024-06-15T11:59:00" in stored_timestamp+ assert_checkpoint_gauge_matches_stored_checkpoint(self.registry, stored_timestamp)@time_machine.travel("2024-06-15 12:00:00", tick=False)@patch("posthog.clickhouse.client.sync_execute")@@ -471,7 +481,7 @@ class TestSyncFeatureFlagLastCalled(BaseTest):FEATURE_FLAG_LAST_CALLED_AT_SYNC_CHUNK_MINUTES=5,FEATURE_FLAG_LAST_CALLED_AT_SYNC_MAX_LOOKBACK_HOURS=6,)-class TestSyncFeatureFlagLastCalledChunking(BaseTest):+class TestSyncFeatureFlagLastCalledChunking(PushGatewayTaskTestMixin, BaseTest):def setUp(self) -> None:super().setUp()cache.clear()@@ -551,6 +561,7 @@ class TestSyncFeatureFlagLastCalledChunking(BaseTest):# onwards is retried next run rather than skippedstored = redis_mock.storage.get(checkpoint_key)assert stored == tz.make_aware(datetime(2024, 6, 15, 11, 50, 0)).isoformat().encode()+ assert_checkpoint_gauge_matches_stored_checkpoint(self.registry, stored)@time_machine.travel("2024-06-15 12:00:00", tick=False)@patch("posthog.clickhouse.client.sync_execute")@@ -696,6 +707,7 @@ class TestSyncFeatureFlagLastCalledChunking(BaseTest):stored = redis_mock.storage.get(checkpoint_key)assert stored is not Noneassert b"2024-06-15T11:59:00" in stored+ assert_checkpoint_gauge_matches_stored_checkpoint(self.registry, stored)# Flag should remain unchangedself.flag1.refresh_from_db()