:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(cohorts): reject a static cohort query patch while a population runs (#121510)
Gustavo H. Strassburger49d57df0b5d7
diff --git a/frontend/src/lib/components/DefinitionPopover/definitionPopoverLogic.test.ts b/frontend/src/lib/components/DefinitionPopover/definitionPopoverLogic.test.tsindex 8f150c13c548..00da3f744108 100644--- a/frontend/src/lib/components/DefinitionPopover/definitionPopoverLogic.test.ts+++ b/frontend/src/lib/components/DefinitionPopover/definitionPopoverLogic.test.ts@@ -163,6 +163,7 @@ describe('definitionPopoverLogic', () => {type: TaxonomicFilterGroupType | stringdefinition: TaxonomicDefinitionTypesurl?: string+ payload?: Record<string, unknown>dispatchActions: any[]}[] = [{@@ -208,6 +209,8 @@ describe('definitionPopoverLogic', () => {type: TaxonomicFilterGroupType.Cohorts,definition: mockCohort,url: `api/projects/${MOCK_TEAM_ID}/cohorts/${mockCohort.id}`,+ // The full cohort carries its query, which would re-populate a static cohort+ payload: { description: 'new and improved description' },dispatchActions: [cohortsModel, ['updateCohort']],},{@@ -240,7 +243,7 @@ describe('definitionPopoverLogic', () => {if (group.url) {expect(api.update).toHaveBeenCalledWith(group.url,- expect.objectContaining({ description: 'new and improved description' })+ group.payload ?? expect.objectContaining({ description: 'new and improved description' }))}})diff --git a/frontend/src/lib/components/DefinitionPopover/definitionPopoverLogic.ts b/frontend/src/lib/components/DefinitionPopover/definitionPopoverLogic.tsindex f60fb8491ce8..1ee0a97ea562 100644--- a/frontend/src/lib/components/DefinitionPopover/definitionPopoverLogic.ts+++ b/frontend/src/lib/components/DefinitionPopover/definitionPopoverLogic.ts@@ -260,7 +260,7 @@ export const definitionPopoverLogic = kea<definitionPopoverLogicType>([// nosemgrep: prefer-codegen-api -- Legacy raw API call with a hand-written URL and an unchecked response type. Use cohortsPartialUpdate() from 'products/cohorts/frontend/generated/api' instead.definition = await api.update(`api/projects/${values.currentProjectId}/cohorts/${_cohort.id}`,- _cohort+ { description: _cohort.description })cohortsModel.findMounted()?.actions.updateCohort(definition as CohortType)}diff --git a/frontend/src/lib/components/SaveToCohortModalContent/SaveToCohortModalContent.tsx b/frontend/src/lib/components/SaveToCohortModalContent/SaveToCohortModalContent.tsxindex 3a33afeab3d9..9b9d3d3116e0 100644--- a/frontend/src/lib/components/SaveToCohortModalContent/SaveToCohortModalContent.tsx+++ b/frontend/src/lib/components/SaveToCohortModalContent/SaveToCohortModalContent.tsx@@ -51,6 +51,11 @@ export function SaveToCohortModalContent({ closeModal, query }: SaveToCohortModa<LemonButtonsize="xsmall"type="primary"+ disabledReason={+ cohort.is_calculating+ ? 'This cohort is still being populated. Try again when it finishes.'+ : undefined+ }onClick={() => {saveQueryToCohort(cohort, query)closeModal()diff --git a/frontend/src/lib/components/SaveToCohortModalContent/saveToCohortModalContentLogic.ts b/frontend/src/lib/components/SaveToCohortModalContent/saveToCohortModalContentLogic.tsindex 33be0bb953b9..2a68eb2f00d1 100644--- a/frontend/src/lib/components/SaveToCohortModalContent/saveToCohortModalContentLogic.ts+++ b/frontend/src/lib/components/SaveToCohortModalContent/saveToCohortModalContentLogic.ts@@ -177,10 +177,10 @@ export const saveToCohortModalContentLogic = kea<saveToCohortModalContentLogicTyawait delay(500) // just in case the toast is too fastlemonToast.dismiss(toastId)cohortSavedToast(cohort, 'query_results', `${toastId}-success`)- } catch (error) {+ } catch (error: any) {console.error('Save to cohort failed:', error)lemonToast.dismiss(toastId)- lemonToast.error('Save to cohort failed')+ lemonToast.error(error.detail || 'Save to cohort failed')}},})),diff --git a/posthog/api/cohort.py b/posthog/api/cohort.pyindex efeceedb6d12..c35e6397d2c3 100644--- a/posthog/api/cohort.py+++ b/posthog/api/cohort.py@@ -10,6 +10,7 @@ from copy import deepcopyfrom typing import Annotated, Any, ClassVar, Literal, NoReturn, Optional, Union, castfrom django.conf import settings+from django.db import transactionfrom django.db.models import OuterRef, Q, QuerySet, Subqueryfrom django.utils import timezone@@ -1834,6 +1835,41 @@ class CohortSerializer(SearchMatchTypeSerializerMixin, serializers.ModelSerializdef _user_access_control(self) -> Optional[UserAccessControl]:return getattr(self.context.get("view"), "user_access_control", None)+ def _claim_static_population(self, cohort: Cohort, source_field: Literal["query", "csv"]) -> None:+ # A query population can run on ClickHouse for hours, and each PATCH would otherwise queue another full run.+ # The claim sets is_calculating, and a query or CSV PATCH gets a 400 until it is cleared. A CSV import takes+ # the claim too, because it clears is_calculating when it finishes and would release a running population.+ # The conditional update locks the row, so two concurrent requests cannot both claim it. The task clears+ # is_calculating when it finishes, and the stuck-cohort sweep clears it if the task dies.+ from posthog.tasks.calculate_cohort import HAS_STATIC_POPULATION_SOURCE++ claimable = Q(is_calculating=False)+ if source_field == "csv":+ # Only a cohort with a query or filters can have a population that a CSV import would release, and the+ # sourced sweep frees that cohort if this request dies before it enqueues. The sourceless sweep cannot+ # tell an abandoned claim from a queued upload, so a CSV-only cohort skips the check, and the next upload+ # recovers it.+ claimable |= ~HAS_STATIC_POPULATION_SOURCE+ claimed = Cohort.objects.filter(claimable, pk=cohort.pk, team_id=cohort.team_id).update(is_calculating=True)+ if not claimed:+ raise ValidationError({source_field: ["This cohort is still being populated. Try again when it finishes."]})+ # cohort.save() writes every field, so the in-memory value must match the claim.+ cohort.is_calculating = True++ def _enqueue_query_population(self, cohort: Cohort) -> None:+ from posthog.tasks.calculate_cohort import insert_cohort_from_query++ try:+ insert_cohort_from_query.delay(cohort.pk, self.context["team_id"])+ except Exception as e:+ # Without a release, every retry is rejected for a run that was never queued.+ Cohort.objects.filter(pk=cohort.pk, team_id=cohort.team_id).update(is_calculating=False)+ cohort.is_calculating = False+ capture_exception(e, additional_properties={"cohort_id": cohort.pk, "team_id": cohort.team_id})+ raise ValidationError(+ {"query": ["Couldn't start populating this cohort. Try again in a few minutes."]}+ ) from e+def update(self, cohort: Cohort, validated_data: dict, *args: Any, **kwargs: Any) -> Cohort: # type: ignorerequest = self.context["request"]existing_has_criteria = cohort_filters_have_values(cohort.filters)@@ -2025,16 +2061,33 @@ class CohortSerializer(SearchMatchTypeSerializerMixin, serializers.ModelSerializif will_create_loops(cohort):raise ValidationError("Cohorts cannot reference other cohorts in a loop.")- cohort.save()+ import_from_csv = not deleted_state and cohort.is_static and parsed_csv is not None+ populate_from_query = (+ not deleted_state and cohort.is_static and parsed_csv is None and bool(validated_data.get("query"))+ )+ with transaction.atomic():+ if import_from_csv or populate_from_query:+ self._claim_static_population(cohort, "csv" if import_from_csv else "query")+ cohort.save()+ elif cohort.is_static:+ # A concurrent query PATCH can claim the cohort after this request loaded it. Writing this request's+ # stale is_calculating would release that claim and let a duplicate population through.+ cohort.save(+ update_fields=[+ field.name+ for field in Cohort._meta.concrete_fields+ if not field.primary_key and field.name != "is_calculating"+ ]+ )+ else:+ cohort.save()if not deleted_state:- from posthog.tasks.calculate_cohort import insert_cohort_from_query-- if cohort.is_static and parsed_csv is not None:+ if import_from_csv:# You can't update a static cohort using the trend/stickiness thingself._enqueue_csv_ids(parsed_csv, cohort)- elif cohort.is_static and validated_data.get("query"):- insert_cohort_from_query.delay(cohort.pk, self.context["team_id"])+ elif populate_from_query:+ self._enqueue_query_population(cohort)elif not cohort.is_static:cohort.enqueue_calculation(initiating_user=request.user)diff --git a/posthog/api/test/test_cohort.py b/posthog/api/test/test_cohort.pyindex 31daaa777349..c0c0284da274 100644--- a/posthog/api/test/test_cohort.py+++ b/posthog/api/test/test_cohort.py@@ -1,4 +1,5 @@import json+from collections.abc import Callablefrom datetime import datetime, timedeltafrom typing import Any, Optionalfrom uuid import uuid4@@ -26,7 +27,7 @@ from rest_framework import statusfrom posthog.schema import PersonsOnEventsMode, PropertyOperator-from posthog.api.cohort import COHORT_USED_IN_PAGE_SIZE, CohortFilters+from posthog.api.cohort import COHORT_USED_IN_PAGE_SIZE, CohortFilters, CohortViewSetfrom posthog.clickhouse.client.execute import sync_executefrom posthog.constants import AvailableFeaturefrom posthog.models import User@@ -670,6 +671,85 @@ [email protected]# Verify the persons were actually added to the cohortself.assertEqual(count_cohort_members(cohort.team_id, cohort.pk), 2)+ @patch("posthog.tasks.calculate_cohort.calculate_cohort_from_list.delay")+ @patch("posthog.tasks.calculate_cohort.insert_cohort_from_query.delay")+ def test_static_cohort_patch_with_query_rejected_while_population_in_flight(+ self, mock_delay: MagicMock, mock_csv_delay: MagicMock+ ):+ query = {"kind": "ActorsQuery", "properties": []}+ cohort = Cohort.objects.create(team=self.team, name="static", is_static=True, query=query)+ url = f"/api/projects/{self.team.id}/cohorts/{cohort.pk}"+ stale_for_rename = Cohort.objects.get(pk=cohort.pk)+ stale_for_query = Cohort.objects.get(pk=cohort.pk)+ stale_for_csv = Cohort.objects.get(pk=cohort.pk)+ csv = SimpleUploadedFile("example.csv", b"distinct_id\n123\n", content_type="application/csv")++ first = self.client.patch(url, data={"query": query})+ with patch.object(CohortViewSet, "get_object", return_value=stale_for_rename):+ rename = self.client.patch(url, data={"name": "renamed"})+ with patch.object(CohortViewSet, "get_object", return_value=stale_for_query):+ second = self.client.patch(url, data={"query": query})+ with patch.object(CohortViewSet, "get_object", return_value=stale_for_csv):+ csv_upload = self.client.patch(url, data={"csv": csv}, format="multipart")++ self.assertEqual(first.status_code, 200, first.json())+ self.assertEqual(rename.status_code, 200, rename.json())+ self.assertEqual(second.status_code, 400, second.json())+ self.assertEqual(second.json()["attr"], "query")+ self.assertEqual(csv_upload.status_code, 400, csv_upload.json())+ self.assertEqual(+ (csv_upload.json()["attr"], csv_upload.json()["detail"]),+ ("csv", "This cohort is still being populated. Try again when it finishes."),+ )+ mock_delay.assert_called_once_with(cohort.pk, self.team.id)+ mock_csv_delay.assert_not_called()++ @parameterized.expand(+ [+ (+ "query",+ "posthog.tasks.calculate_cohort.insert_cohort_from_query.delay",+ lambda: {"data": {"query": {"kind": "ActorsQuery", "properties": []}}},+ ),+ (+ "csv",+ "posthog.tasks.calculate_cohort.calculate_cohort_from_list.delay",+ lambda: {+ "data": {+ "csv": SimpleUploadedFile("example.csv", b"distinct_id\n123\n", content_type="application/csv")+ },+ "format": "multipart",+ },+ ),+ ]+ )+ def test_static_cohort_patch_releases_claim_when_enqueue_fails(+ self, source_field: str, delay_path: str, request_kwargs: Callable[[], dict[str, Any]]+ ):+ cohort = Cohort.objects.create(team=self.team, name="static", is_static=True)+ url = f"/api/projects/{self.team.id}/cohorts/{cohort.pk}"++ with patch(delay_path, side_effect=[ConnectionError("broker down"), None]) as mock_delay:+ failed = self.client.patch(url, **request_kwargs())+ retried = self.client.patch(url, **request_kwargs())++ self.assertEqual(failed.status_code, 400, failed.json())+ self.assertEqual(failed.json()["attr"], source_field)+ self.assertEqual(retried.status_code, 200, retried.json())+ self.assertEqual(mock_delay.call_count, 2)++ @patch("posthog.tasks.calculate_cohort.calculate_cohort_from_list.delay")+ def test_csv_patch_recovers_csv_only_cohort_left_calculating(self, mock_csv_delay: MagicMock):+ cohort = Cohort.objects.create(team=self.team, name="static", is_static=True, is_calculating=True)+ csv = SimpleUploadedFile("example.csv", b"distinct_id\n123\n", content_type="application/csv")++ response = self.client.patch(+ f"/api/projects/{self.team.id}/cohorts/{cohort.pk}", data={"csv": csv}, format="multipart"+ )++ self.assertEqual(response.status_code, 200, response.json())+ mock_csv_delay.assert_called_once()+@patch("posthog.tasks.calculate_cohort.insert_cohort_from_filters.delay",side_effect=insert_cohort_from_filters,diff --git a/posthog/tasks/calculate_cohort.py b/posthog/tasks/calculate_cohort.pyindex 3dae3a43ce1c..4844cc6bd89b 100644--- a/posthog/tasks/calculate_cohort.py+++ b/posthog/tasks/calculate_cohort.py@@ -233,6 +233,7 @@ def get_stuck_static_cohort_candidates_queryset() -> QuerySet:- AND it has not errored within the last hour. A population task stamps last_error_at whenit schedules a retry, and reset_stuck_cohorts stamps it when it re-dispatches, so a recenterror means the run is alive and resetting it would dispatch a duplicate population.+ A cohort at the error cap is still released, so a query or CSV update can claim it again, but it is not re-run."""one_hour_ago = timezone.now() - relativedelta(hours=1)return (@@ -240,7 +241,6 @@ def get_stuck_static_cohort_candidates_queryset() -> QuerySet:is_static=True,is_calculating=True,deleted=False,- errors_calculating__lt=MAX_ERRORS_CALCULATING,).filter(Q(last_calculation__isnull=True, created_at__lte=one_hour_ago)diff --git a/posthog/tasks/test/test_calculate_cohort.py b/posthog/tasks/test/test_calculate_cohort.pyindex 6eadc356bc6c..c4b20eed9fda 100644--- a/posthog/tasks/test/test_calculate_cohort.py+++ b/posthog/tasks/test/test_calculate_cohort.py@@ -601,10 +601,7 @@ def calculate_cohort_test_factory(event_factory: Callable, person_factory: Callareset_stuck_cohorts()stuck_static_cohort.refresh_from_db()- # Should NOT be picked up because errors_calculating is at the max- self.assertTrue(stuck_static_cohort.is_calculating)-- # Verify insert_cohort_from_query was NOT called+ self.assertFalse(stuck_static_cohort.is_calculating)mock_insert_cohort_from_query.delay.assert_not_called()@patch("posthog.tasks.calculate_cohort.increment_version_and_enqueue_calculate_cohort")diff --git a/products/cohorts/mcp/tools.yaml b/products/cohorts/mcp/tools.yamlindex 3a2e95a08fb8..74bbedc4f003 100644--- a/products/cohorts/mcp/tools.yaml+++ b/products/cohorts/mcp/tools.yaml@@ -158,6 +158,11 @@ tools:ActorsQuery through its matching 'query-*-actors' tool (e.g. 'query-trends-actors') — and only save one thatreturned successfully. Prefer this over 'cohorts-add-persons-to-static-cohort-partial-update' for anythingbeyond a handful (~20) of people.+++ An update that sets 'query' returns a 400 while the cohort's 'is_calculating' is true. That includes the+ first population right after 'cohorts-create' with a query. Before you send the next query, call+ 'cohorts-retrieve' and wait until 'is_calculating' is false.exclude_params:- groups- versiondiff --git a/services/mcp/schema/generated-tool-definitions.json b/services/mcp/schema/generated-tool-definitions.jsonindex 86da8a35d6b0..58df8e167c9a 100644--- a/services/mcp/schema/generated-tool-definitions.json+++ b/services/mcp/schema/generated-tool-definitions.json@@ -2956,7 +2956,7 @@}},"cohorts-partial-update": {- "description": "Update an existing cohort's name, description, or filters. Changing filters on a dynamic cohort triggers recalculation. To soft-delete a cohort, set 'deleted: true'.\n\nTo add more people to an existing STATIC cohort in bulk, set a 'query' (HogQLQuery or ActorsQuery) on it — the server runs the query and inserts every newly matched person (existing members are skipped), with no row limit. The query must expose the person id under one of these column names, checked in order: 'person_id', 'actor_id', 'id', or 'distinct_id' (selecting from the 'events' or 'persons' tables resolves the actor automatically). Validate the query before saving it — run a HogQLQuery through 'execute-sql' and an ActorsQuery through its matching 'query-*-actors' tool (e.g. 'query-trends-actors') — and only save one that returned successfully. Prefer this over 'cohorts-add-persons-to-static-cohort-partial-update' for anything beyond a handful (~20) of people.",+ "description": "Update an existing cohort's name, description, or filters. Changing filters on a dynamic cohort triggers recalculation. To soft-delete a cohort, set 'deleted: true'.\n\nTo add more people to an existing STATIC cohort in bulk, set a 'query' (HogQLQuery or ActorsQuery) on it — the server runs the query and inserts every newly matched person (existing members are skipped), with no row limit. The query must expose the person id under one of these column names, checked in order: 'person_id', 'actor_id', 'id', or 'distinct_id' (selecting from the 'events' or 'persons' tables resolves the actor automatically). Validate the query before saving it — run a HogQLQuery through 'execute-sql' and an ActorsQuery through its matching 'query-*-actors' tool (e.g. 'query-trends-actors') — and only save one that returned successfully. Prefer this over 'cohorts-add-persons-to-static-cohort-partial-update' for anything beyond a handful (~20) of people.\n\nAn update that sets 'query' returns a 400 while the cohort's 'is_calculating' is true. That includes the first population right after 'cohorts-create' with a query. Before you send the next query, call 'cohorts-retrieve' and wait until 'is_calculating' is false.","category": "Cohorts","feature": "cohorts","summary": "Update cohort",diff --git a/services/mcp/schema/tool-definitions-all.json b/services/mcp/schema/tool-definitions-all.jsonindex 7982898575ee..136e7ca88600 100644--- a/services/mcp/schema/tool-definitions-all.json+++ b/services/mcp/schema/tool-definitions-all.json@@ -2971,7 +2971,7 @@}},"cohorts-partial-update": {- "description": "Update an existing cohort's name, description, or filters. Changing filters on a dynamic cohort triggers recalculation. To soft-delete a cohort, set 'deleted: true'.\n\nTo add more people to an existing STATIC cohort in bulk, set a 'query' (HogQLQuery or ActorsQuery) on it — the server runs the query and inserts every newly matched person (existing members are skipped), with no row limit. The query must expose the person id under one of these column names, checked in order: 'person_id', 'actor_id', 'id', or 'distinct_id' (selecting from the 'events' or 'persons' tables resolves the actor automatically). Validate the query before saving it — run a HogQLQuery through 'execute-sql' and an ActorsQuery through its matching 'query-*-actors' tool (e.g. 'query-trends-actors') — and only save one that returned successfully. Prefer this over 'cohorts-add-persons-to-static-cohort-partial-update' for anything beyond a handful (~20) of people.",+ "description": "Update an existing cohort's name, description, or filters. Changing filters on a dynamic cohort triggers recalculation. To soft-delete a cohort, set 'deleted: true'.\n\nTo add more people to an existing STATIC cohort in bulk, set a 'query' (HogQLQuery or ActorsQuery) on it — the server runs the query and inserts every newly matched person (existing members are skipped), with no row limit. The query must expose the person id under one of these column names, checked in order: 'person_id', 'actor_id', 'id', or 'distinct_id' (selecting from the 'events' or 'persons' tables resolves the actor automatically). Validate the query before saving it — run a HogQLQuery through 'execute-sql' and an ActorsQuery through its matching 'query-*-actors' tool (e.g. 'query-trends-actors') — and only save one that returned successfully. Prefer this over 'cohorts-add-persons-to-static-cohort-partial-update' for anything beyond a handful (~20) of people.\n\nAn update that sets 'query' returns a 400 while the cohort's 'is_calculating' is true. That includes the first population right after 'cohorts-create' with a query. Before you send the next query, call 'cohorts-retrieve' and wait until 'is_calculating' is false.","category": "Cohorts","feature": "cohorts","summary": "Update cohort",