:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(experiments): name the srm segment cause for every reader (#127850)
Marcel Poelkere375fa85f2e9
diff --git a/products/experiments/backend/health/rollout.py b/products/experiments/backend/health/rollout.pydeleted file mode 100644index 5e736f152ff3..000000000000--- a/products/experiments/backend/health/rollout.py+++ /dev/null@@ -1,45 +0,0 @@-import posthoganalytics--from posthog.clickhouse.query_tagging import get_query_tags-from posthog.models.team.team import Team-from posthog.models.user import User-from posthog.utils import flag_identity--# pinned: FEATURE_FLAGS.EXPERIMENT_HEALTH_FINDINGS in the frontend, which shows the findings.-HEALTH_FINDINGS_FLAG = "experiment-health-findings"---def runs_for_impersonated_session() -> bool:- """- Whether a staff member started this run in an impersonated session. A query runner has no session, so this reads- the tag of the request, which follows an async run into the worker. The tag reads only the session cookie, so an- OAuth token that was minted during impersonation is not an impersonated session here.- """- return bool(get_query_tags().is_impersonated)---def shows_health_findings(user: User, team: Team) -> bool:- """- Whether the web app shows the health findings to the user in the team. An impersonated session always shows them,- because the user here is the customer, whom the flag can leave out. Otherwise it evaluates the app's flag locally,- from the person and group inputs that the app sends to /flags, so it never waits on the network. An evaluation- that is undecided or fails is False.- """- if runs_for_impersonated_session():- return True- identity = flag_identity(user, team, team.organization)- if identity is None:- return False- try:- enabled = posthoganalytics.feature_enabled(- HEALTH_FINDINGS_FLAG,- identity.distinct_id,- groups=identity.groups,- person_properties=identity.person_properties,- group_properties=identity.group_properties,- only_evaluate_locally=True,- send_feature_flag_events=False,- )- except Exception:- return False- return bool(enabled)diff --git a/products/experiments/backend/health/test/test_rollout.py b/products/experiments/backend/health/test/test_rollout.pydeleted file mode 100644index 58d2f45b82ff..000000000000--- a/products/experiments/backend/health/test/test_rollout.py+++ /dev/null@@ -1,64 +0,0 @@-from contextlib import AbstractContextManager--from posthog.test.base import BaseTest-from unittest.mock import patch--import posthoganalytics-from parameterized import parameterized-from posthoganalytics import Posthog--from products.experiments.backend.health.rollout import HEALTH_FINDINGS_FLAG, shows_health_findings--# The shape of a definition from /api/feature_flag/local_evaluation, with the rollout by email domain.-HEALTH_FINDINGS_DEFINITION = {- "id": 1,- "key": HEALTH_FINDINGS_FLAG,- "active": True,- "deleted": False,- "ensure_experience_continuity": False,- "filters": {- "groups": [- {- "properties": [{"key": "email", "type": "person", "value": "@example.com", "operator": "icontains"}],- "rollout_percentage": 100,- }- ],- "multivariate": None,- },-}---class TestShowsHealthFindings(BaseTest):- def setUp(self) -> None:- super().setUp()- self.client_with_definition = Posthog("phc_test", personal_api_key="phx_test", sync_mode=True)- self.client_with_definition.feature_flags = [HEALTH_FINDINGS_DEFINITION]- self.client_with_definition.group_type_mapping = {"0": "organization", "1": "project"}-- @parameterized.expand(- [- ("email_in_the_rollout", "[email protected]", False, True),- ("email_outside_the_rollout", "[email protected]", False, False),- ("anonymized", "[email protected]", True, False),- ]- )- def test_evaluates_the_apps_flag_from_the_users_identity(- self, _name: str, email: str, anonymize_data: bool, expected: bool- ) -> None:- self.user.email = email- self.user.anonymize_data = anonymize_data-- with self._client_with_definition():- self.assertEqual(shows_health_findings(self.user, self.team), expected)-- def test_shows_nothing_to_a_user_without_a_distinct_id(self) -> None:- self.user.email = "[email protected]"- self.user.distinct_id = None-- with self._client_with_definition():- self.assertFalse(shows_health_findings(self.user, self.team))-- def _client_with_definition(self) -> AbstractContextManager[None]:- # The module functions apply the module settings to the default client on every call, and the test settings- # disable it.- return patch.multiple(posthoganalytics, default_client=self.client_with_definition, disabled=False)diff --git a/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py b/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.pyindex 1a9044e223f7..9c545900e64a 100644--- a/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py+++ b/products/experiments/backend/hogql_queries/experiment_exposures_query_runner.py@@ -68,7 +68,6 @@ from products.experiments.backend.health.registry import (EXPOSURE_HEALTH_CHECKS,evaluate as evaluate_health,)-from products.experiments.backend.health.rollout import runs_for_impersonated_session, shows_health_findingsfrom products.experiments.backend.hogql_queries import MULTIPLE_VARIANT_KEYfrom products.experiments.backend.hogql_queries.base_query_utils import analysis_window, analysis_window_endfrom products.experiments.backend.hogql_queries.error_handling import experiment_error_handler@@ -96,9 +95,9 @@ QUERY_ROW_LIMIT = 5000 # The result has one row per (day, variant) pair# The queries that explain a health finding scan the exposure events. A timeout keeps the finding without the# explanation, so it must not hold up the exposure answer as long as its own query can.HEALTH_QUERY_TIMEOUT_SECONDS = 60-# The page recalculates an experiment with few exposures on every entry, and the count without the test account filter-# must not repeat its scan each time.-EXPOSED_WITHOUT_FILTER_CACHE_SECONDS = 60 * 60+# The page recalculates the exposures on a forced refresh, and on every entry while few users are exposed. The queries+# that explain a finding must not repeat their scan each time.+HEALTH_QUERY_CACHE_SECONDS = 60 * 60def build_experiment_exposure_query(experiment: Experiment) -> ExperimentExposureQuery:@@ -135,7 +134,6 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):self.error_event_context = error_event_context# The user id and whether that user may read the flag's history.self._flag_history_access: tuple[int, bool] | None = None- self._health_findings_shown: tuple[int, bool] | None = Noneself._reads_precomputed_exposures = Falseif not self.query.experiment_id:@@ -332,17 +330,6 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):self._flag_history_access = (self.user.pk, readable)return self._flag_history_access[1]- def _runs_segments_query(self) -> bool:- # The segments only explain a finding and the query scans the exposure events, so it runs only for a reader- # who sees the findings. A userless run sees none. The flag's answer stays out of the cache key: an async run- # computes the key again in a worker, from that worker's flag definitions, and a key that disagrees with the web- # request's makes every page load miss the cache.- if not isinstance(self.user, User):- return False- if self._health_findings_shown is None or self._health_findings_shown[0] != self.user.pk:- self._health_findings_shown = (self.user.pk, shows_health_findings(self.user, self.team))- return self._health_findings_shown[1]-def _flag_history(self) -> FlagHistory | FlagHistoryNotChecked | None:try:restrictions = get_restricted_properties_with_group_type_index_for_team(@@ -364,11 +351,12 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):context = load_health_context(self.experiment, exposures)if reads_flag_history(context):context = replace(context, flag_history=self._flag_history())- if self._runs_segments_query() and reads_segments(context):+ # These queries run for every reader, a userless run included. The findings depend on them, and the cache key+ # doesn't say who computed the answer, so every reader with the same key shares it.+ if reads_segments(context):segments = self._segments(exposure_params, segments_first_exposed_after(context))exposures = replace(exposures, segments=segments)context = replace(context, exposures=exposures)- # This count runs for every reader. The findings depend on it, and every reader shares the cached answer.if exposures.filters_test_accounts and (reads_exposed_without_test_account_filter(context) or reads_share_kept_by_test_account_filter(context)):@@ -379,10 +367,24 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):def _segments(self, exposure_params: ExposureQueryParams, first_exposed_after: date | None) -> tuple[ExposureSegment, ...] | None:- query = self._query_builder(exposure_params).get_exposure_segments_query(- TOP_SEGMENT_VALUES, first_exposed_after+ cache_key = self._health_query_cache_key(+ "exposure_segments",+ {+ "variants": self.variants,+ "group_type_index": self.group_type_index,+ "top_values": TOP_SEGMENT_VALUES,+ "first_exposed_after": first_exposed_after.isoformat() if first_exposed_after else None,+ },)- rows = self._run_health_query("ExperimentExposureSegmentsQuery", query)+ cached = get_safe_cache(cache_key)+ if cached is not None:+ rows = cached["rows"]+ else:+ query = self._query_builder(exposure_params).get_exposure_segments_query(+ TOP_SEGMENT_VALUES, first_exposed_after+ )+ rows = self._run_health_query("ExperimentExposureSegmentsQuery", query)+ safe_cache_set(cache_key, {"rows": rows}, HEALTH_QUERY_CACHE_SECONDS)if rows is None:return Noneexposed: dict[tuple[str, str, str], dict[str, int]] = {}@@ -399,15 +401,9 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):)def _exposed_without_test_account_filter(self, exposure_params: ExposureQueryParams) -> int | None:- # A cache hit skips the masking of the query, so the key needs the reader's property access from the payload.- cache_key = self._cache_key_for(- {- **self.get_cache_payload(),- "exposed_without_test_account_filter": {- "variants": self.variants,- "group_type_index": self.group_type_index,- },- }+ cache_key = self._health_query_cache_key(+ "exposed_without_test_account_filter",+ {"variants": self.variants, "group_type_index": self.group_type_index},)cached = get_safe_cache(cache_key)if cached is not None:@@ -416,9 +412,13 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):rows = self._run_health_query("ExperimentExposuresWithoutTestAccountFilterQuery", query)exposed = None if rows is None else int(rows[0][0]) if rows else 0# The cache keeps a failed count too, so a project that hits the timeout doesn't wait for it on every entry.- safe_cache_set(cache_key, {"exposed": exposed}, EXPOSED_WITHOUT_FILTER_CACHE_SECONDS)+ safe_cache_set(cache_key, {"exposed": exposed}, HEALTH_QUERY_CACHE_SECONDS)return exposed+ def _health_query_cache_key(self, name: str, inputs: dict[str, Any]) -> str:+ # A cache hit skips the masking of the query, so the key needs the reader's property access from the payload.+ return self._cache_key_for({**self.get_cache_payload(), name: inputs})+def _run_health_query(self, query_type: str, query: ast.SelectQuery) -> list[Any] | None:"""The rows of a query that only explains a health finding. None when it fails, so that the finding keeps thewording without the explanation and the exposure answer doesn't fail."""@@ -618,8 +618,4 @@ class ExperimentExposuresQueryRunner(ExperimentResultsCacheMixin, QueryRunner):# The test account filter finding asks the user to edit these filters, so an edit must not serve the old answer.if applies_test_account_filters(self.team, self.exposure_criteria):payload["test_account_filters"] = self.team.test_account_filters- # An impersonated session reads the segments whether or not the customer has the flag, so it must not get the- # customer's cached answer without them. The query tag follows an async run, so a worker computes the same key.- if runs_for_impersonated_session():- payload["impersonated_session"] = Truereturn payloaddiff --git a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.pyindex 425746867fdf..1d23db7c25a1 100644--- a/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py+++ b/products/experiments/backend/hogql_queries/test/experiment_query_runner/test_experiment_exposures_query_runner.py@@ -15,7 +15,6 @@ from rest_framework.exceptions import ValidationErrorfrom posthog.schema import (ActionsNode,- CachedExperimentExposureQueryResponse,ExperimentEventExposureConfig,ExperimentExposureCriteria,ExperimentExposureHealthFindingCode,@@ -27,15 +26,13 @@ from posthog.schema import (from posthog.hogql.errors import ExposedHogQLErrorfrom posthog.hogql.query import execute_hogql_query-from posthog.clickhouse.query_tagging import AccessMethod, reset_query_tags, tags_context+from posthog.clickhouse.query_tagging import AccessMethod, tags_contextfrom posthog.constants import AvailableFeaturefrom posthog.exceptions import ClickHouseQueryTimeOut-from posthog.hogql_queries.query_runner import ExecutionModefrom posthog.hogql_queries.utils.breakdowns import BREAKDOWN_OTHER_STRING_LABELfrom posthog.models import PropertyDefinitionfrom posthog.models.activity_logging.activity_log import ActivityLogfrom posthog.models.user import User-from posthog.tasks.tasks import process_query_taskfrom posthog.test.persons import create_group_type_mappingfrom posthog.test.test_journeys import journeys_for@@ -1631,10 +1628,9 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):flush_persons_and_events()self._log_split_change_from_90_10_on_2024_01_03()- with patch(f"{ExperimentExposuresQueryRunner.__module__}.shows_health_findings", return_value=True):- response = ExperimentExposuresQueryRunner(- team=self.team, query=self._running_exposure_query(), user=self.user- ).calculate()+ response = ExperimentExposuresQueryRunner(+ team=self.team, query=self._running_exposure_query(), user=self.user+ ).calculate()[finding] = response.health_findings or []versions = sorted(@@ -1673,72 +1669,10 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):created_at=datetime(2024, 1, 3, tzinfo=UTC),)- @parameterized.expand(- [- (- "reader_who_sees_the_findings",- True,- 'The mismatch comes from users whose first exposure was sent by version "4.2.1" of the "posthog-node" '- "library.",- "library_version",- {- "located_dimensions": "library,library_version",- "library_1": "web",- "library_1_exposed_control": 100,- "library_1_exposed_test": 100,- "library_2": "posthog-node",- "library_2_exposed_control": 5,- "library_2_exposed_test": 120,- "library_3": "posthog-ios",- "library_3_exposed_control": 10,- "library_3_exposed_test": 10,- "library_version_1": "1.200.0",- "library_version_1_library": "web",- "library_version_1_exposed_control": 100,- "library_version_1_exposed_test": 100,- "library_version_2": "4.2.1",- "library_version_2_library": "posthog-node",- "library_version_2_exposed_control": 5,- "library_version_2_exposed_test": 120,- "library_version_3": "3.0.0",- "library_version_3_library": "posthog-ios",- "library_version_3_exposed_control": 10,- "library_version_3_exposed_test": 10,- "page_1": "/",- "page_1_exposed_control": 34,- "page_1_exposed_test": 34,- "page_2": "/pricing",- "page_2_exposed_control": 20,- "page_2_exposed_test": 20,- "page_3": "/docs/:page",- "page_3_exposed_control": 15,- "page_3_exposed_test": 15,- "page_4": "/blog",- "page_4_exposed_control": 12,- "page_4_exposed_test": 12,- "page_5": "/about",- "page_5_exposed_control": 10,- "page_5_exposed_test": 10,- "page_6": BREAKDOWN_OTHER_STRING_LABEL,- "page_6_exposed_control": 9,- "page_6_exposed_test": 9,- "screen_1": "Checkout",- "screen_1_exposed_control": 10,- "screen_1_exposed_test": 10,- },- ),- (- "reader_who_does_not_see_them",- False,- "This may indicate issues with randomization or data collection.",- None,- {},- ),- ]- )+ @parameterized.expand([("signed_in_reader", True), ("userless_run", False)])@time_machine.travel("2024-01-07T12:00:00Z", tick=False)def test_srm_splits_the_exposures_by_the_library_and_page_of_the_first_exposure(- self, _name: str, findings_shown: bool, detail: str, reason: str | None, evidence: dict[str, str | int]+ self, _name: str, as_user: bool) -> None:def exposure(variant: str, timestamp: str, library: str, version: str, **place: str | None) -> dict[str, Any]:return {@@ -1795,18 +1729,67 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):journeys_for({**web_first, **server_only, **mobile}, self.team)flush_persons_and_events()- with patch(f"{ExperimentExposuresQueryRunner.__module__}.shows_health_findings", return_value=findings_shown):- response = ExperimentExposuresQueryRunner(- team=self.team, query=self._running_exposure_query(), user=self.user- ).calculate()+ response = ExperimentExposuresQueryRunner(+ team=self.team, query=self._running_exposure_query(), user=self.user if as_user else None+ ).calculate()[finding] = response.health_findings or []self.assertEqual(- (finding.code, finding.subcode, finding.reason), (ExperimentExposureHealthFindingCode.SRM, None, reason)+ (finding.code, finding.subcode, finding.reason),+ (ExperimentExposureHealthFindingCode.SRM, None, "library_version"),+ )+ self.assertIn(+ 'The mismatch comes from users whose first exposure was sent by version "4.2.1" of the "posthog-node" '+ "library.",+ finding.detail,)- self.assertIn(detail, finding.detail)self.assertEqual(- {key: value for key, value in finding.evidence.items() if not key.endswith("p_value")}, evidence+ {key: value for key, value in finding.evidence.items() if not key.endswith("p_value")},+ {+ "located_dimensions": "library,library_version",+ "library_1": "web",+ "library_1_exposed_control": 100,+ "library_1_exposed_test": 100,+ "library_2": "posthog-node",+ "library_2_exposed_control": 5,+ "library_2_exposed_test": 120,+ "library_3": "posthog-ios",+ "library_3_exposed_control": 10,+ "library_3_exposed_test": 10,+ "library_version_1": "1.200.0",+ "library_version_1_library": "web",+ "library_version_1_exposed_control": 100,+ "library_version_1_exposed_test": 100,+ "library_version_2": "4.2.1",+ "library_version_2_library": "posthog-node",+ "library_version_2_exposed_control": 5,+ "library_version_2_exposed_test": 120,+ "library_version_3": "3.0.0",+ "library_version_3_library": "posthog-ios",+ "library_version_3_exposed_control": 10,+ "library_version_3_exposed_test": 10,+ "page_1": "/",+ "page_1_exposed_control": 34,+ "page_1_exposed_test": 34,+ "page_2": "/pricing",+ "page_2_exposed_control": 20,+ "page_2_exposed_test": 20,+ "page_3": "/docs/:page",+ "page_3_exposed_control": 15,+ "page_3_exposed_test": 15,+ "page_4": "/blog",+ "page_4_exposed_control": 12,+ "page_4_exposed_test": 12,+ "page_5": "/about",+ "page_5_exposed_control": 10,+ "page_5_exposed_test": 10,+ "page_6": BREAKDOWN_OTHER_STRING_LABEL,+ "page_6_exposed_control": 9,+ "page_6_exposed_test": 9,+ "screen_1": "Checkout",+ "screen_1_exposed_control": 10,+ "screen_1_exposed_test": 10,+ },)@parameterized.expand(@@ -1870,10 +1853,9 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):)flush_persons_and_events()- with patch(f"{ExperimentExposuresQueryRunner.__module__}.shows_health_findings", return_value=True):- response = ExperimentExposuresQueryRunner(- team=self.team, query=self._running_exposure_query(), user=self.user- ).calculate()+ response = ExperimentExposuresQueryRunner(+ team=self.team, query=self._running_exposure_query(), user=self.user+ ).calculate()[finding] = [finding@@ -1928,20 +1910,26 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):runner_module = ExperimentExposuresQueryRunner.__module__with (patch(f"{runner_module}.execute_hogql_query", side_effect=execute),- patch(f"{runner_module}.shows_health_findings", return_value=True),patch(f"{runner_module}.logger") as logger,patch(f"{runner_module}.capture_exception") as capture_exception,):- response = ExperimentExposuresQueryRunner(- team=self.team, query=self._running_exposure_query(), user=self.user- ).calculate()+ query = self._running_exposure_query()+ responses = [+ ExperimentExposuresQueryRunner(team=self.team, query=query, user=self.user).calculate()+ for _ in range(2)+ ]- self.assertIn("ExperimentExposureSegmentsQuery", query_types)+ self.assertEqual(query_types.count("ExperimentExposureSegmentsQuery"), 1)self.assertEqual((logger.warning.called, capture_exception.called), (logged, not logged))- self.assertEqual(response.total_exposures, {"control": 90, "test": 10})+ self.assertEqual([response.total_exposures for response in responses], [{"control": 90, "test": 10}] * 2)self.assertEqual(- [(finding.code, finding.detail, set(finding.evidence)) for finding in response.health_findings or []],[+ (finding.code, finding.detail, set(finding.evidence))+ for response in responses+ for finding in response.health_findings or []+ ],+ 2+ * [(ExperimentExposureHealthFindingCode.SRM,"The distribution of users across variants doesn't match your configured rollout percentages "@@ -1951,81 +1939,6 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):],)- def _expose_a_mismatch_located_in_one_library_version(self) -> None:- ff_property = f"$feature/{self.feature_flag.key}"- journeys_for(- {- f"user_{library}_{variant}_{i}": [- {- "event": "$feature_flag_called",- "timestamp": "2024-01-02",- "properties": {- "$feature_flag_response": variant,- ff_property: variant,- "$feature_flag": self.feature_flag.key,- "$lib": library,- "$lib_version": "1.0.0",- },- }- ]- for library, counts in (- ("web", (("control", 60), ("test", 60))),- ("posthog-node", (("control", 15), ("test", 105))),- )- for variant, count in counts- for i in range(count)- },- self.team,- )- flush_persons_and_events()-- @time_machine.travel("2024-01-07T12:00:00Z", tick=False)- def test_impersonated_session_reads_the_segments_and_keeps_its_answer_apart_from_the_customers(self) -> None:- self._expose_a_mismatch_located_in_one_library_version()- query = self._running_exposure_query()-- def cached_and_reason() -> tuple[bool | None, str | None]:- response = ExperimentExposuresQueryRunner(team=self.team, query=query, user=self.user).run(- ExecutionMode.RECENT_CACHE_CALCULATE_BLOCKING_IF_STALE- )- assert isinstance(response, CachedExperimentExposureQueryResponse)- [finding] = response.health_findings or []- return response.is_cached, finding.reason-- customer = cached_and_reason()- with tags_context(is_impersonated=True):- impersonated = cached_and_reason()- customer_again = cached_and_reason()-- self.assertEqual(- [customer, impersonated, customer_again],- [(False, None), (False, "library_version"), (True, None)],- )-- @time_machine.travel("2024-01-07T12:00:00Z", tick=False)- def test_async_run_of_an_impersonated_session_fills_the_entry_the_web_request_reads(self) -> None:- self._expose_a_mismatch_located_in_one_library_version()- query = self._running_exposure_query()-- def run(execution_mode: ExecutionMode) -> Any:- return ExperimentExposuresQueryRunner(team=self.team, query=query, user=self.user).run(- execution_mode, user=self.user- )-- with tags_context(is_impersonated=True), patch("posthog.tasks.tasks.process_query_task.si") as queued_task:- run(ExecutionMode.CALCULATE_ASYNC_ALWAYS)-- # A worker starts without the tags of the request, so only what the queued task carries reaches it.- with tags_context():- reset_query_tags()- process_query_task.apply(args=queued_task.call_args.args, kwargs=queued_task.call_args.kwargs).get()-- with tags_context(is_impersonated=True):- response = run(ExecutionMode.CACHE_ONLY_NEVER_CALCULATE)-- [finding] = response.health_findings or []- self.assertEqual(finding.reason, "library_version")-def test_cache_key_keeps_the_flag_history_from_a_reader_who_cannot_view_the_flag(self) -> None:self.organization.available_product_features = [{"key": AvailableFeature.ACCESS_CONTROL, "name": AvailableFeature.ACCESS_CONTROL}@@ -2116,7 +2029,6 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):0,2,LEAVES_OUT_TEST_ACCOUNTS,- True,"2024-01-07T12:00:00Z",[(@@ -2132,7 +2044,6 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):0,0,LEAVES_OUT_TEST_ACCOUNTS,- True,"2024-01-07T12:00:00Z",[(@@ -2143,44 +2054,11 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):)],),- (- "test_accounts_exposed_for_a_reader_who_does_not_see_the_findings",- 0,- 2,- LEAVES_OUT_TEST_ACCOUNTS,- False,- "2024-01-07T12:00:00Z",- [- (- ExperimentExposureHealthFindingCode.ZERO_EXPOSURES,- "calls_not_counted",- "No users exposed: the test account filter removes all of them",- 2,- )- ],- ),- (- "few_users_counted_for_a_reader_who_does_not_see_the_findings",- 3,- 100,- KEEPS_MATCHING_USERS,- False,- "2024-01-07T12:00:00Z",- [- (- ExperimentExposureHealthFindingCode.TEST_ACCOUNT_FILTER_REMOVES_EXPOSURES,- None,- "The test account filter removes most exposed users",- 103,- )- ],- ),("few_users_counted",3,100,KEEPS_MATCHING_USERS,- True,"2024-01-07T12:00:00Z",[(@@ -2191,9 +2069,9 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):)],),- ("few_users_counted_in_the_first_day", 3, 100, KEEPS_MATCHING_USERS, True, "2024-01-01T23:00:00Z", []),- ("few_users_counted_with_only_negative_filters", 3, 100, LEAVES_OUT_TEST_ACCOUNTS, True, LATER, []),- ("few_users_counted_with_only_a_hogql_filter", 3, 100, HOGQL_FILTER, True, LATER, []),+ ("few_users_counted_in_the_first_day", 3, 100, KEEPS_MATCHING_USERS, "2024-01-01T23:00:00Z", []),+ ("few_users_counted_with_only_negative_filters", 3, 100, LEAVES_OUT_TEST_ACCOUNTS, LATER, []),+ ("few_users_counted_with_only_a_hogql_filter", 3, 100, HOGQL_FILTER, LATER, []),])def test_counts_the_users_that_the_test_account_filter_removes(@@ -2202,7 +2080,6 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):users: int,test_accounts: int,test_account_filter: dict[str, Any],- findings_shown: bool,now: str,expected: list[tuple[ExperimentExposureHealthFindingCode, str | None, str, int | None]],) -> None:@@ -2217,7 +2094,6 @@ class TestExperimentExposuresQueryRunner(ExperimentQueryRunnerBaseTest):with (time_machine.travel(now, tick=False),patch(f"{runner_module}.execute_hogql_query", side_effect=execute),- patch(f"{runner_module}.shows_health_findings", return_value=findings_shown),):response = ExperimentExposuresQueryRunner(team=self.team, query=self._running_exposure_query(), user=self.user