Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

chore(flags): remove the rolled-out refresh routing flag check (#119295)

posthog[bot]de0a2223a4cd
diff --git a/products/feature_flags/backend/flags_cache.py b/products/feature_flags/backend/flags_cache.py
index 1300bfd36c52..b40b3598a1bd 100644
--- a/products/feature_flags/backend/flags_cache.py
+++ b/products/feature_flags/backend/flags_cache.py
@@ -1145,18 +1145,17 @@ def get_cache_stats() -> dict[str, Any]:
#
# Transitional surface: KAFKA_ROUTING_FLAG, _evaluate_kafka_routing_flag,
# _route_to_kafka, get_team_primary_flags_writer (and its config binding on
-# FLAGS_HYPERCACHE_MANAGEMENT_CONFIG), REFRESH_ROUTING_FLAG, _refresh_routing_enabled,
-# route_refresh_to_kafka (and its config binding), SHADOW_COMPARE_FLAG,
-# _shadow_compare_enabled, publish_shadow_invalidation, _produce_invalidation,
-# _enqueue_invalidation, and the Kafka branch inside it.
+# FLAGS_HYPERCACHE_MANAGEMENT_CONFIG), route_refresh_to_kafka (and its config
+# binding), SHADOW_COMPARE_FLAG, _shadow_compare_enabled, publish_shadow_invalidation,
+# _produce_invalidation, _enqueue_invalidation, and the Kafka branch inside it.
# The signal handlers themselves stay; their tails simplify at cutover. The one
# call site outside this block is the tail of update_team_service_flags_cache in
# tasks.py, which goes with it.
#
-# At cutover the sweep routes unconditionally: REFRESH_ROUTING_FLAG and its gate go,
-# and the binding is replaced by a hook that always produces, not deleted. The generic
-# surface it uses — route_refresh_fn, the enqueued count and RefreshPacing — keeps its
-# caller and stays.
+# At cutover the sweep routes unconditionally: the FLAGS_CACHE_REFRESH_KAFKA_ENABLED
+# guard goes, and the binding is replaced by a hook that always produces, not deleted.
+# The generic surface it uses — route_refresh_fn, the enqueued count and RefreshPacing —
+# keeps its caller and stays.
# Per-team gate that routes invalidation to Kafka instead of Celery — see
# _enqueue_invalidation for why the two paths are mutually exclusive. The key
@@ -1252,63 +1251,6 @@ def _route_to_kafka(team_id: int) -> bool:
return bool(result)
-# Per-team gate for the hourly expiry refresh sweep. A different flag from
-# KAFKA_ROUTING_FLAG on purpose: that one is pinned at 100% for the edit path, so
-# reusing it would move the sweep the moment this code shipped. The two ramp
-# independently, and setting this one back to 0 returns the next hourly run to the
-# Python builder with no cleanup, because a refresh message already in flight still
-# produces a correct build.
-REFRESH_ROUTING_FLAG = "flags-cache-refresh-kafka"
-
-
-def _refresh_routing_enabled(team_id: int) -> bool:
- """Return True if this team's hourly refresh should be raised as a Kafka
- invalidation instead of being built in the Celery worker.
-
- FLAGS_CACHE_REFRESH_KAFKA_ENABLED is read before the flag, because the flag cannot
- be scoped to one deployment: local evaluation resolves it against the single
- project key set in posthog/apps.py, so raising it raises it in every region at
- once. A region whose flags-cache-builder image predates the `source` field rejects
- the message through deny_unknown_fields, counts it as a parse error and drops it,
- and the dead letter queue holds build failures rather than parse failures, so
- nothing can be replayed. The setting is read per deployment, so a lagging region
- holds routing off on its own until its builder is rolled.
-
- Every failure mode returns False, which keeps the sweep building in Python. That
- is the safe direction: a build that happens is never worse than a message nobody
- consumes.
-
- Unlike `_route_to_kafka`, an unresolvable flag ticks no TOMBSTONE_COUNTER, for the
- reason `_shadow_compare_enabled` gives. Local evaluation cannot resolve
- REFRESH_ROUTING_FLAG for as long as the flag does not exist in PostHog, so a tick
- would fire for every team of every run, which is about 5,000 an hour, and hold a
- constant high rate on a panel that means "rare anomaly". A client that is actually
- broken raises instead, and the warning below records that.
- """
- if not settings.FLAGS_CACHE_REFRESH_KAFKA_ENABLED:
- return False
-
- try:
- return feature_enabled_or_false(
- REFRESH_ROUTING_FLAG,
- f"team-{team_id}",
- groups={"project": str(team_id)},
- group_properties={"project": {"id": str(team_id)}},
- only_evaluate_locally=True,
- send_feature_flag_events=False,
- )
- except Exception:
- # Log so a fleet-wide silent disable is visible in Sentry rather than only as
- # an enqueued gauge that quietly returns to zero mid-ramp.
- logger.warning(
- "flags_cache_refresh_routing_flag_evaluation_failed",
- team_id=team_id,
- flag=REFRESH_ROUTING_FLAG,
- exc_info=True,
- )
- return False
-
-
def route_refresh_to_kafka(team_id: int) -> bool:
"""Routing hook for the expiry refresh sweep, bound on
FLAGS_HYPERCACHE_MANAGEMENT_CONFIG.
@@ -1330,8 +1272,16 @@ def route_refresh_to_kafka(team_id: int) -> bool:
building the team anyway. The sweep skips its own build for a team whose hook
failed. A team whose message is lost stays inside the expiry window, so the next
hourly run raises it again, and the verifier repairs it in the meantime.
+
+ FLAGS_CACHE_REFRESH_KAFKA_ENABLED is read per deployment, so each region turns
+ routing on only after its flags-cache-builder image accepts the `source` field.
+ An older builder rejects the message through deny_unknown_fields, counts it as a
+ parse error and drops it, and the dead letter queue holds build failures rather
+ than parse failures, so nothing can be replayed. With the setting off the sweep
+ keeps building in Python, which is the safe direction: a build that happens is
+ never worse than a message nobody consumes.
"""
- if not _refresh_routing_enabled(team_id):
+ if not settings.FLAGS_CACHE_REFRESH_KAFKA_ENABLED:
return False
_produce_invalidation(team_id, source="refresh", raise_on_error=True)
diff --git a/products/feature_flags/backend/test/test_flags_cache.py b/products/feature_flags/backend/test/test_flags_cache.py
index b8593f1526e2..114241a709d8 100644
--- a/products/feature_flags/backend/test/test_flags_cache.py
+++ b/products/feature_flags/backend/test/test_flags_cache.py
@@ -38,7 +38,6 @@ from products.experiments.backend.models.experiment import Experiment
from products.feature_flags.backend.flags_cache import (
FLAGS_HYPERCACHE_MANAGEMENT_CONFIG,
KAFKA_ROUTING_FLAG,
- REFRESH_ROUTING_FLAG,
SHADOW_COMPARE_FLAG,
_blank_inactive_filters,
_compare_flag_fields,
@@ -1220,28 +1219,13 @@ class TestRefreshRoutingHook(SimpleTestCase):
@override_settings(FLAGS_CACHE_REFRESH_KAFKA_ENABLED=False)
@patch("products.feature_flags.backend.flags_cache._produce_invalidation")
- @patch("products.feature_flags.backend.flags_cache.feature_enabled_or_false", return_value=True)
- def test_the_deployment_switch_off_declines_the_team_without_reading_the_flag(self, mock_gate, mock_produce):
- assert route_refresh_to_kafka(self.TEAM_ID) is False
-
- # Checked before the flag, which resolves against one project key for every
- # region and so cannot hold routing off in the lagging one.
- mock_gate.assert_not_called()
- mock_produce.assert_not_called()
-
- @patch("products.feature_flags.backend.flags_cache._produce_invalidation")
- @patch("products.feature_flags.backend.flags_cache.feature_enabled_or_false", return_value=False)
- def test_gate_off_declines_the_team_and_produces_nothing(self, mock_gate, mock_produce):
+ def test_the_deployment_switch_off_declines_the_team_and_produces_nothing(self, mock_produce):
assert route_refresh_to_kafka(self.TEAM_ID) is False
mock_produce.assert_not_called()
- # The sweep must not read the edit path's flag, which is pinned at 100%.
- assert mock_gate.call_args.args[0] == REFRESH_ROUTING_FLAG
- assert REFRESH_ROUTING_FLAG != KAFKA_ROUTING_FLAG
@patch("products.feature_flags.backend.flags_cache.producer_scope")
- @patch("products.feature_flags.backend.flags_cache.feature_enabled_or_false", return_value=True)
- def test_gate_on_produces_a_message_the_builder_reads_as_a_refresh(self, mock_gate, mock_producer_scope):
+ def test_the_deployment_switch_on_produces_a_message_the_builder_reads_as_a_refresh(self, mock_producer_scope):
mock_producer = MagicMock()
mock_producer_scope.return_value.__enter__.return_value = mock_producer
@@ -1254,25 +1238,8 @@ class TestRefreshRoutingHook(SimpleTestCase):
assert envelope.source == "refresh"
assert envelope.shadow is False
- @patch("products.feature_flags.backend.flags_cache.TOMBSTONE_COUNTER")
- @patch("products.feature_flags.backend.flags_cache._produce_invalidation")
- @patch(
- "products.feature_flags.backend.flags_cache.feature_enabled_or_false",
- side_effect=RuntimeError("posthoganalytics borked"),
- )
- def test_a_broken_gate_leaves_the_build_to_python_without_ticking_the_tombstone(
- self, mock_gate, mock_produce, mock_tombstone
- ):
- assert route_refresh_to_kafka(self.TEAM_ID) is False
-
- mock_produce.assert_not_called()
- # A run evaluates the gate once per team, so a tick here would hold a constant
- # rate on a panel that means "rare anomaly".
- mock_tombstone.labels.assert_not_called()
-
@patch("products.feature_flags.backend.flags_cache.producer_scope")
- @patch("products.feature_flags.backend.flags_cache.feature_enabled_or_false", return_value=True)
- def test_a_produce_failure_is_reported_rather_than_counted_as_enqueued(self, mock_gate, mock_producer_scope):
+ def test_a_produce_failure_is_reported_rather_than_counted_as_enqueued(self, mock_producer_scope):
mock_producer_scope.side_effect = RuntimeError("kafka cluster unreachable")
# Never True: the sweep counts a team as enqueued from this returning True, so
@@ -1419,7 +1386,6 @@ class TestShadowInvalidationPublishing(SimpleTestCase):
# the gate reports its own failure as "shadow off".
assert mock_logger.warning.call_args.args[0] == "flags_cache_shadow_compare_flag_evaluation_failed"
- @override_settings(FLAGS_CACHE_REFRESH_KAFKA_ENABLED=True)
@patch("products.feature_flags.backend.flags_cache._produce_invalidation")
# `_shadow_compare_enabled` reaches the SDK through ph_client rather than this
# module's import. Patching here still covers it, because both names resolve to
@@ -1427,15 +1393,12 @@ class TestShadowInvalidationPublishing(SimpleTestCase):
@patch("products.feature_flags.backend.flags_cache.posthoganalytics.feature_enabled", return_value=False)
def test_every_gate_evaluates_locally_and_captures_nothing(self, mock_feature_enabled, mock_produce):
publish_shadow_invalidation(self.TEAM_ID)
- route_refresh_to_kafka(self.TEAM_ID)
# A remote evaluation would put a blocking flags-API call inside every cache
- # build, and an event capture would bill each rebuild as product usage. The
- # refresh gate runs once per team of every hourly run, so it costs the most.
+ # build, and an event capture would bill each rebuild as product usage.
assert {call.args[0] for call in mock_feature_enabled.call_args_list} == {
KAFKA_ROUTING_FLAG,
SHADOW_COMPARE_FLAG,
- REFRESH_ROUTING_FLAG,
}
for call in mock_feature_enabled.call_args_list:
assert call.kwargs["only_evaluate_locally"] is True
@@ -1492,14 +1455,8 @@ class TestGetTeamPrimaryFlagsWriter(unittest.TestCase):
def test_flags_config_binds_the_refresh_routing_hook(self):
# The lambda on the config is the only wire between the sweep and the hook.
# Unbound, the sweep silently keeps building in Python and the enqueued gauge
- # stays at zero, which looks the same as the flag being off.
- with (
- patch("products.feature_flags.backend.flags_cache._produce_invalidation") as mock_produce,
- patch(
- "products.feature_flags.backend.flags_cache.feature_enabled_or_false",
- return_value=True,
- ),
- ):
+ # stays at zero, which looks the same as the setting being off.
+ with patch("products.feature_flags.backend.flags_cache._produce_invalidation") as mock_produce:
assert FLAGS_HYPERCACHE_MANAGEMENT_CONFIG.route_refresh_fn is not None
assert FLAGS_HYPERCACHE_MANAGEMENT_CONFIG.route_refresh_fn(42) is True