:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(cohorts): bound csv upload size for static cohorts (#119686)
Patricio772fc9b1a7d1
diff --git a/posthog/api/cohort.py b/posthog/api/cohort.pyindex c9eb6d2fdce5..798a1fbc4b97 100644--- a/posthog/api/cohort.py+++ b/posthog/api/cohort.py@@ -1,3 +1,4 @@+import ioimport csvimport jsonimport time@@ -8,6 +9,7 @@ from collections.abc import Iterable, Iterator, Sequencefrom copy import deepcopyfrom typing import Annotated, Any, ClassVar, Literal, Optional, Union, cast+from django.conf import settingsfrom django.db.models import OuterRef, Q, QuerySet, Subqueryfrom django.utils import timezone@@ -797,6 +799,40 @@ class CSVConfig:ENCODING_ERROR = "CSV file encoding is not supported. Please save your file as UTF-8 and try again."FORMAT_ERROR = "CSV file format is invalid. Please check your file format and try again."GENERIC_ERROR = "An error occurred while processing your CSV file. Please try again or contact support if the problem persists."+ FILE_TOO_LARGE = "CSV file is larger than the limit of {limit}. Split the file into smaller parts and upload them one at a time."+ TOO_MANY_ROWS = (+ "CSV file has more than {limit} rows. Split the file into smaller parts and upload them one at a time."+ )++ @staticmethod+ def file_too_large_message() -> str:+ limit = settings.COHORT_CSV_UPLOAD_MAX_BYTES+ if limit < 1024 * 1024:+ readable = f"{limit:,} bytes"+ else:+ readable = f"{limit / (1024 * 1024):.1f}".removesuffix(".0") + " MB"+ return CSVConfig.ErrorMessages.FILE_TOO_LARGE.format(limit=readable)++ @staticmethod+ def too_many_rows_message() -> str:+ return CSVConfig.ErrorMessages.TOO_MANY_ROWS.format(limit=f"{settings.COHORT_CSV_UPLOAD_MAX_ROWS:,}")+++CsvIdType = Literal["distinct_id", "person_id", "email"]+++@frozen+class ParsedCsvIds:+ ids: list[str]+ id_type: CsvIdType+ email_property_key: str | None+++@frozen+class _CsvIdColumn:+ index: int+ id_type: CsvIdType+ header: str@extend_schema_field(CohortFilters) # type: ignore[arg-type]@@ -1142,6 +1178,7 @@ class CohortSerializer(SearchMatchTypeSerializerMixin, serializers.ModelSerializcontext: dict,validated_data: dict,person_ids: list[str] | None,+ parsed_csv: ParsedCsvIds | None,) -> None:from posthog.tasks.calculate_cohort import (insert_cohort_from_feature_flag,@@ -1149,13 +1186,12 @@ class CohortSerializer(SearchMatchTypeSerializerMixin, serializers.ModelSerializinsert_cohort_from_query,)- request = self.context["request"]- if request.FILES.get("csv") or person_ids:+ if parsed_csv is not None or person_ids:if person_ids:uuids = validate_person_uuids_exist(self.context["team_id"], person_ids)cohort.insert_users_list_by_uuid(uuids, team_id=self.context["team_id"])- if request.FILES.get("csv"):- self._calculate_static_by_csv(request.FILES["csv"], cohort)+ if parsed_csv is not None:+ self._enqueue_csv_ids(parsed_csv, cohort)elif context.get("from_feature_flag_key"):insert_cohort_from_feature_flag.delay(cohort.pk, context["from_feature_flag_key"], self.context["team_id"])elif validated_data.get("query"):@@ -1194,6 +1230,7 @@ class CohortSerializer(SearchMatchTypeSerializerMixin, serializers.ModelSerializvalidated_data["cohort_type"] = computed_cohort_typeperson_ids = validated_data.pop("_create_static_person_ids", None)+ parsed_csv = validated_data.pop("_parsed_csv", None)cohort = Cohort.objects.create(team_id=self.context["team_id"], **validated_data)if cohort.is_static:@@ -1206,7 +1243,7 @@ class CohortSerializer(SearchMatchTypeSerializerMixin, serializers.ModelSerializcohort.is_calculating = Truecohort.save(update_fields=["is_calculating"])- self._handle_static(cohort, self.context, validated_data, person_ids)+ self._handle_static(cohort, self.context, validated_data, person_ids, parsed_csv)# Refresh from DB to get updated count field set by _insert_users_list_with_batchingcohort.refresh_from_db()elif cohort.query is not None:@@ -1225,8 +1262,8 @@ class CohortSerializer(SearchMatchTypeSerializerMixin, serializers.ModelSerializdef _parse_csv_file(self, file) -> tuple[list[str], Iterator[list[str]]]:"""Handle file reading and CSV parsing with error handling"""- decoded_file = file.read().decode(CSVConfig.ENCODING).splitlines()- reader = csv.reader(decoded_file)+ # Decodes row by row, so the upload is never held whole in memory as bytes, text, or a list of lines.+ reader = csv.reader(io.TextIOWrapper(file, encoding=CSVConfig.ENCODING, newline=""))# Skip empty rows at the beginningfirst_row: list[str] = []@@ -1251,7 +1288,7 @@ class CohortSerializer(SearchMatchTypeSerializerMixin, serializers.ModelSerializ"""Check if header indicates email column"""return CSVConfig.normalize_header(header) in CSVConfig.EMAIL_HEADERS- def _find_id_column(self, headers: list[str]) -> tuple[int, str, str] | None:+ def _find_id_column(self, headers: list[str]) -> _CsvIdColumn | None:"""Find the index, type, and actual column name of the ID column in headers, with preference order: person_id > distinct_id > email"""normalized_headers = [h.strip() for h in headers]header_keys = [CSVConfig.normalize_header(h) for h in headers]@@ -1259,20 +1296,25 @@ class CohortSerializer(SearchMatchTypeSerializerMixin, serializers.ModelSerializ# First, look for person_id columns (preferred)for i, key in enumerate(header_keys):if key in CSVConfig.PERSON_ID_HEADERS:- return i, "person_id", normalized_headers[i]+ return _CsvIdColumn(index=i, id_type="person_id", header=normalized_headers[i])# Then, look for distinct_id columnsfor i, key in enumerate(header_keys):if key in CSVConfig.DISTINCT_ID_HEADERS:- return i, "distinct_id", normalized_headers[i]+ return _CsvIdColumn(index=i, id_type="distinct_id", header=normalized_headers[i])# Finally, look for email columnsfor i, key in enumerate(header_keys):if key in CSVConfig.EMAIL_HEADERS:- return i, "email", normalized_headers[i]+ return _CsvIdColumn(index=i, id_type="email", header=normalized_headers[i])return None+ def _append_id_within_row_limit(self, ids: list[str], id_value: str) -> None:+ if len(ids) >= settings.COHORT_CSV_UPLOAD_MAX_ROWS:+ raise ValidationError({"csv": [CSVConfig.too_many_rows_message()]})+ ids.append(id_value)+def _extract_ids_single_column(self,first_row: list[str],@@ -1280,22 +1322,22 @@ class CohortSerializer(SearchMatchTypeSerializerMixin, serializers.ModelSerializskip_header: bool = False,) -> list[str]:"""Process single-column CSV format"""- distinct_ids = []+ distinct_ids: list[str] = []# Include first row only if it's not a headerif not skip_header and first_row and first_row[0].strip() != "":- distinct_ids.append(first_row[0].strip())+ self._append_id_within_row_limit(distinct_ids, first_row[0].strip())for row in reader:if len(row) > 0:stripped_id = row[0].strip()if stripped_id != "":- distinct_ids.append(stripped_id)+ self._append_id_within_row_limit(distinct_ids, stripped_id)return distinct_ids- def _extract_ids_multi_column(self, reader: Iterator[list[str]], id_col: int, cohort_pk: int) -> list[str]:+ def _extract_ids_multi_column(self, reader: Iterator[list[str]], id_col: int) -> list[str]:"""Process multi-column CSV format with robust error handling"""- ids = []+ ids: list[str] = []skipped_rows = 0for row in reader:@@ -1307,77 +1349,23 @@ class CohortSerializer(SearchMatchTypeSerializerMixin, serializers.ModelSerializ# Extract ID if present and non-emptyid_value = row[id_col].strip()if id_value != "":- ids.append(id_value)+ self._append_id_within_row_limit(ids, id_value)if skipped_rows > 0:- logger.info(f"Skipped {skipped_rows} rows with incorrect column count in CSV for cohort {cohort_pk}")-- return ids-- def _validate_and_process_ids(- self, ids: list[str], id_type: str, cohort: Cohort, email_property_key: str | None = None- ) -> None:- """Final validation and task scheduling"""- from posthog.tasks.calculate_cohort import calculate_cohort_from_list-- if not ids:- raise ValidationError({"csv": [CSVConfig.ErrorMessages.NO_VALID_IDS]})-- logger.info(f"Processing CSV upload for cohort {cohort.pk} with {len(ids)} {id_type}s")- # A killed import leaves its history row open. The stuck-cohort sweep resets a cohort whose newest row stays- # open past its cutoff. That row stays the newest until the task of this upload starts. A run that is still- # alive overwrites this stamp when it finishes.- close_open_population_attempts(cohort.pk, finished_at=timezone.now())- calculate_cohort_from_list.delay(- cohort.pk,- ids,- team_id=self.context["team_id"],- id_type=id_type,- email_property_key=email_property_key,- )-- def _handle_csv_errors(self, e: Exception, cohort: Cohort) -> None:- """Centralized error handling with consistent exception capture"""-- # Reset calculating flag on error- cohort.is_calculating = False- cohort.save(update_fields=["is_calculating"])-- if isinstance(e, UnicodeDecodeError):- raise ValidationError({"csv": [CSVConfig.ErrorMessages.ENCODING_ERROR]})- elif isinstance(e, csv.Error):- capture_exception(- e,- additional_properties={- "cohort_id": cohort.pk,- "team_id": self.context["team_id"],- },- )- raise ValidationError({"csv": [CSVConfig.ErrorMessages.FORMAT_ERROR]})- elif isinstance(e, ValidationError):- # If it's already a ValidationError, just re-raise it to preserve format- raise- else:- capture_exception(- e,- additional_properties={- "cohort_id": cohort.pk,- "team_id": self.context["team_id"],- },+ logger.info(+ f"Skipped {skipped_rows} rows with incorrect column count in CSV for team {self.context['team_id']}")- raise ValidationError({"csv": [CSVConfig.ErrorMessages.GENERIC_ERROR]})- def _calculate_static_by_csv(self, file, cohort: Cohort) -> None:- """Main orchestration method for CSV processing - clear high-level flow"""- # Set calculating flag immediately so UI shows loading state- cohort.is_calculating = True- cohort.save(update_fields=["is_calculating"])+ return ids+ def _parse_csv_ids(self, file) -> ParsedCsvIds:+ """Parse the uploaded CSV into IDs, raising a ValidationError on the csv field for any problem"""try:first_row, reader = self._parse_csv_file(file)+ id_type: CsvIdType+ email_property_key: str | None = Noneif self._is_single_column_format(first_row):- email_property_key: str | None = Noneif first_row and self._is_person_id_header(first_row[0]):ids = self._extract_ids_single_column(first_row, reader, skip_header=True)id_type = "person_id"@@ -1389,8 +1377,6 @@ class CohortSerializer(SearchMatchTypeSerializerMixin, serializers.ModelSerializ# Single column format treated as distinct_ids for backwards compatibilityids = self._extract_ids_single_column(first_row, reader, skip_header=False)id_type = "distinct_id"-- self._validate_and_process_ids(ids, id_type, cohort, email_property_key)else:result = self._find_id_column(first_row)@@ -1408,12 +1394,59 @@ class CohortSerializer(SearchMatchTypeSerializerMixin, serializers.ModelSerializ})- id_col, id_type, actual_column_name = result- ids = self._extract_ids_multi_column(reader, id_col, cohort.pk)- self._validate_and_process_ids(ids, id_type, cohort, actual_column_name if id_type == "email" else None)+ ids = self._extract_ids_multi_column(reader, result.index)+ id_type = result.id_type+ if id_type == "email":+ email_property_key = result.header+ except ValidationError:+ raise+ except UnicodeDecodeError as e:+ raise ValidationError({"csv": [CSVConfig.ErrorMessages.ENCODING_ERROR]}) from e+ except Exception as e:+ capture_exception(e, additional_properties=self._csv_error_properties())+ message = (+ CSVConfig.ErrorMessages.FORMAT_ERROR+ if isinstance(e, csv.Error)+ else CSVConfig.ErrorMessages.GENERIC_ERROR+ )+ raise ValidationError({"csv": [message]}) from e++ if not ids:+ raise ValidationError({"csv": [CSVConfig.ErrorMessages.NO_VALID_IDS]})+ return ParsedCsvIds(ids=ids, id_type=id_type, email_property_key=email_property_key)++ def _csv_error_properties(self) -> dict[str, Any]:+ return {+ "cohort_id": self.instance.pk if self.instance is not None else None,+ "team_id": self.context["team_id"],+ }++ def _enqueue_csv_ids(self, parsed: ParsedCsvIds, cohort: Cohort) -> None:+ """Schedule the import of IDs that _parse_csv_ids already validated"""+ from posthog.tasks.calculate_cohort import calculate_cohort_from_list++ # Set calculating flag immediately so UI shows loading state+ cohort.is_calculating = True+ cohort.save(update_fields=["is_calculating"])+ try:+ logger.info(f"Processing CSV upload for cohort {cohort.pk} with {len(parsed.ids)} {parsed.id_type}s")+ # A killed import leaves its history row open. The stuck-cohort sweep resets a cohort whose newest row stays+ # open past its cutoff. That row stays the newest until the task of this upload starts. A run that is still+ # alive overwrites this stamp when it finishes.+ close_open_population_attempts(cohort.pk, finished_at=timezone.now())+ calculate_cohort_from_list.delay(+ cohort.pk,+ parsed.ids,+ team_id=self.context["team_id"],+ id_type=parsed.id_type,+ email_property_key=parsed.email_property_key,+ )except Exception as e:- self._handle_csv_errors(e, cohort)+ cohort.is_calculating = False+ cohort.save(update_fields=["is_calculating"])+ capture_exception(e, additional_properties={"cohort_id": cohort.pk, "team_id": self.context["team_id"]})+ raise ValidationError({"csv": [CSVConfig.ErrorMessages.GENERIC_ERROR]}) from edef validate_query(self, query: Optional[dict]) -> Optional[dict]:if not query:@@ -1621,6 +1654,9 @@ class CohortSerializer(SearchMatchTypeSerializerMixin, serializers.ModelSerializ# instance's preserved behavioral filters against the feature-flag rule.attrs = super().validate(attrs)+ parsed_csv = self._parse_csv_upload(attrs)+ if parsed_csv is not None:+ attrs["_parsed_csv"] = parsed_csvself._validate_cohort_references(attrs)self._validate_warehouse_access(attrs)self._validate_no_new_hidden_event_criteria(attrs)@@ -1635,6 +1671,21 @@ class CohortSerializer(SearchMatchTypeSerializerMixin, serializers.ModelSerializreturn attrs+ def _parse_csv_upload(self, attrs: dict) -> ParsedCsvIds | None:+ # Runs in validate() so that every CSV rejection, the size and row limits included, happens before+ # create() or update() writes the cohort. create() and update() only enqueue the parsed IDs.+ csv_file = self.context["request"].FILES.get("csv")+ if csv_file is None:+ return None+ # Mirrors the conditions under which create() and update() use the file, so an ignored file is not parsed.+ is_static = attrs.get("is_static", self.instance.is_static if self.instance is not None else False)+ if not is_static or attrs.get("deleted"):+ return None+ # `size` is what the upload handler received, not the Content-Length header.+ if csv_file.size > settings.COHORT_CSV_UPLOAD_MAX_BYTES:+ raise ValidationError({"csv": [CSVConfig.file_too_large_message()]})+ return self._parse_csv_ids(csv_file)+def _validate_no_new_hidden_event_criteria(self, attrs: dict) -> None:if self.context.get(ALLOW_HIDDEN_EVENT_CRITERIA_CONTEXT_KEY):return@@ -1739,6 +1790,7 @@ class CohortSerializer(SearchMatchTypeSerializerMixin, serializers.ModelSerializ_coerce_stored_filter_values(cohort.filters))+ parsed_csv = validated_data.pop("_parsed_csv", None)create_in_folder = validated_data.pop("_create_in_folder", None)if create_in_folder is not None:cohort._create_in_folder = create_in_folder or None@@ -1926,9 +1978,9 @@ class CohortSerializer(SearchMatchTypeSerializerMixin, serializers.ModelSerializif not deleted_state:from posthog.tasks.calculate_cohort import insert_cohort_from_query- if cohort.is_static and request.FILES.get("csv"):+ if cohort.is_static and parsed_csv is not None:# You can't update a static cohort using the trend/stickiness thing- self._calculate_static_by_csv(request.FILES["csv"], cohort)+ self._enqueue_csv_ids(parsed_csv, cohort)elif cohort.is_static and validated_data.get("query"):insert_cohort_from_query.delay(cohort.pk, self.context["team_id"])elif not cohort.is_static:diff --git a/posthog/api/test/test_cohort.py b/posthog/api/test/test_cohort.pyindex df765e6b33fa..dbe3c1812a37 100644--- a/posthog/api/test/test_cohort.py+++ b/posthog/api/test/test_cohort.py@@ -1488,6 +1488,112 @@ Jane Smith,25self.assertIn("empty", response_data["detail"])self.assertEqual(patch_calculate_cohort_from_list.call_count, 0)+ @parameterized.expand([("create",), ("update",)])+ @patch("posthog.tasks.calculate_cohort.calculate_cohort_from_list.delay")+ def test_static_cohort_csv_upload_rejects_file_over_size_limit(self, action, patch_calculate_cohort_from_list):+ csv = SimpleUploadedFile("ids.csv", b"user_1\nuser_2\n", content_type="application/csv")++ with override_settings(COHORT_CSV_UPLOAD_MAX_BYTES=4):+ if action == "create":+ response = self.client.post(+ f"/api/projects/{self.team.id}/cohorts/",+ {"name": "test_size", "csv": csv, "is_static": True},+ format="multipart",+ )+ self.assertFalse(Cohort.objects.filter(team=self.team, name="test_size").exists())+ else:+ cohort = Cohort.objects.create(team=self.team, name="before", is_static=True)+ response = self.client.patch(+ f"/api/projects/{self.team.id}/cohorts/{cohort.id}",+ {"name": "after", "csv": csv},+ format="multipart",+ )+ cohort.refresh_from_db()+ self.assertEqual(cohort.name, "before")+ self.assertFalse(cohort.is_calculating)++ self.assertEqual(response.status_code, 400)+ self.assertEqual(response.json()["attr"], "csv")+ self.assertIn("larger than the limit of 4 bytes", response.json()["detail"])+ self.assertEqual(patch_calculate_cohort_from_list.call_count, 0)++ @parameterized.expand(+ [+ ("single_column_over_limit", b"user_1\nuser_2\nuser_3\n", 400),+ ("multi_column_over_limit", b"name,distinct_id\na,user_1\nb,user_2\nc,user_3\n", 400),+ ("single_column_at_limit", b"user_1\nuser_2\n", 201),+ ("multi_column_at_limit", b"name,distinct_id\na,user_1\nb,user_2\n", 201),+ ]+ )+ @patch("posthog.tasks.calculate_cohort.calculate_cohort_from_list.delay")+ def test_static_cohort_csv_upload_row_limit(+ self, _name, content, expected_status, patch_calculate_cohort_from_list+ ):+ csv = SimpleUploadedFile("ids.csv", content, content_type="application/csv")++ with override_settings(COHORT_CSV_UPLOAD_MAX_ROWS=2):+ response = self.client.post(+ f"/api/projects/{self.team.id}/cohorts/",+ {"name": "test_rows", "csv": csv, "is_static": True},+ format="multipart",+ )++ self.assertEqual(response.status_code, expected_status)+ if expected_status == 400:+ self.assertEqual(response.json()["attr"], "csv")+ self.assertIn("more than 2 rows", response.json()["detail"])+ self.assertFalse(Cohort.objects.filter(team=self.team, name="test_rows").exists())+ self.assertEqual(patch_calculate_cohort_from_list.call_count, 0)+ else:+ patch_calculate_cohort_from_list.assert_called_once_with(+ response.json()["id"],+ ["user_1", "user_2"],+ team_id=self.team.id,+ id_type="distinct_id",+ email_property_key=None,+ )++ @patch("posthog.tasks.calculate_cohort.calculate_cohort_from_list.delay")+ def test_static_cohort_csv_upload_rejects_invalid_utf8_after_first_chunk(self, patch_calculate_cohort_from_list):+ # The reader decodes in chunks of about 8 KB, so this byte fails inside the ID extraction loop.+ content = b"".join(f"user_{i}\n".encode() for i in range(3000)) + b"caf\xe9\n"+ csv = SimpleUploadedFile("ids.csv", content, content_type="application/csv")++ response = self.client.post(+ f"/api/projects/{self.team.id}/cohorts/",+ {"name": "test_encoding", "csv": csv, "is_static": True},+ format="multipart",+ )++ self.assertEqual(response.status_code, 400)+ self.assertEqual(response.json()["attr"], "csv")+ self.assertIn("UTF-8", response.json()["detail"])+ self.assertFalse(Cohort.objects.filter(team=self.team, name="test_encoding").exists())+ self.assertEqual(patch_calculate_cohort_from_list.call_count, 0)++ @patch("posthog.tasks.calculate_cohort.calculate_cohort_from_list.delay")+ def test_static_cohort_csv_upload_reads_file_spooled_to_disk(self, patch_calculate_cohort_from_list):+ csv = SimpleUploadedFile(+ "ids.csv", "\ufeffemail\[email protected]\[email protected]\n".encode(), content_type="application/csv"+ )++ # A file above FILE_UPLOAD_MAX_MEMORY_SIZE reaches the serializer as a TemporaryUploadedFile.+ with override_settings(FILE_UPLOAD_MAX_MEMORY_SIZE=1):+ response = self.client.post(+ f"/api/projects/{self.team.id}/cohorts/",+ {"name": "test_disk", "csv": csv, "is_static": True},+ format="multipart",+ )++ self.assertEqual(response.status_code, 201)+ patch_calculate_cohort_from_list.assert_called_once_with(+ response.json()["id"],+ ["[email protected]", "[email protected]"],+ team_id=self.team.id,+ id_type="email",+ email_property_key="email",+ )+@patch("posthog.tasks.calculate_cohort.calculate_cohort_from_list.delay")def test_static_cohort_csv_upload_no_valid_ids_fails(self, patch_calculate_cohort_from_list):"""Test that CSV with no valid distinct IDs fails with clear error"""diff --git a/posthog/api/utils.py b/posthog/api/utils.pyindex 795fe350af31..e7b72f2f1412 100644--- a/posthog/api/utils.py+++ b/posthog/api/utils.py@@ -77,6 +77,7 @@ class ServiceRequest:self.successful_authenticator = Noneself.path = "/"self.data: dict = {}+ self.FILES: dict = {}self.GET: dict = {}self.META: dict = {}self.headers: dict = {}diff --git a/posthog/settings/cohorts.py b/posthog/settings/cohorts.pyindex 8be611a89873..d37217c575ff 100644--- a/posthog/settings/cohorts.py+++ b/posthog/settings/cohorts.py@@ -113,3 +113,8 @@ BEHAVIORAL_BACKFILL_FINALIZER_MAX_RUNS_PER_PASS: int = get_from_env(COHORT_DEPENDENCY_INCREMENTAL_MAINTENANCE: bool = get_from_env("COHORT_DEPENDENCY_INCREMENTAL_MAINTENANCE", True, type_cast=str_to_bool)+# Bounds one static cohort CSV upload. The IDs are collected in memory and sent to Celery in one+# message, so the byte size and the row count both need a ceiling. 50 MB matches MAX_UPLOAD_SIZE_BYTES+# in products/warehouse_sources/backend/file_uploads.py.+COHORT_CSV_UPLOAD_MAX_BYTES: int = get_from_env("COHORT_CSV_UPLOAD_MAX_BYTES", 50 * 1024 * 1024, type_cast=int)+COHORT_CSV_UPLOAD_MAX_ROWS: int = get_from_env("COHORT_CSV_UPLOAD_MAX_ROWS", 1_000_000, type_cast=int)