:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(cohorts): cap static cohort csv uploads at 20 mb (#122853)
Patricio7a8530f5953d
diff --git a/frontend/src/scenes/cohorts/CohortCsvDropzone.tsx b/frontend/src/scenes/cohorts/CohortCsvDropzone.tsxindex bdbc04b99dff..5e24c3c2dd6a 100644--- a/frontend/src/scenes/cohorts/CohortCsvDropzone.tsx+++ b/frontend/src/scenes/cohorts/CohortCsvDropzone.tsx@@ -1,7 +1,11 @@+import { useValues } from 'kea'+import { IconUpload } from '@posthog/icons'import { LemonFileInput } from 'lib/lemon-ui/LemonFileInput'+import { preflightLogic } from 'lib/logic/preflightLogic'import { cn } from 'lib/utils/css-classes'+import { cohortCsvMaxBytes, formatCohortCsvLimit } from 'scenes/cohorts/cohortUtils'export const COHORT_CSV_HELP ="Upload a CSV file to add users to your cohort. For single-column files, include one distinct ID per row (all rows will be processed as data). For multi-column files, include a header row with a 'person_id', 'distinct_id', or 'email' column containing the user identifiers."@@ -13,6 +17,7 @@ export function CohortCsvDropzone({value: File | nullonChange: (file: File | null) => void}): JSX.Element {+ const { preflight } = useValues(preflightLogic)return (<LemonFileInputaccept=".csv"@@ -43,7 +48,9 @@ export function CohortCsvDropzone({) : (<><div>Drag a file here or click to browse for a file</div>- <div className="text-secondary text-xs">Accepts .csv files only</div>+ <div className="text-secondary text-xs">+ {`Accepts .csv files up to ${formatCohortCsvLimit(cohortCsvMaxBytes(preflight))}`}+ </div></>)}</div>diff --git a/frontend/src/scenes/cohorts/cohortEditLogic.test.ts b/frontend/src/scenes/cohorts/cohortEditLogic.test.tsindex 0b5c05f4fab6..86e64efae657 100644--- a/frontend/src/scenes/cohorts/cohortEditLogic.test.ts+++ b/frontend/src/scenes/cohorts/cohortEditLogic.test.ts@@ -5,9 +5,11 @@ import { expectLogic, partial } from 'kea-test-utils'import posthog from 'posthog-js'import { v4 as uuidv4 } from 'uuid'+import { ApiError } from 'lib/api-error'import { TaxonomicFilterGroupType } from 'lib/components/TaxonomicFilter/types'import { scrollToFormError } from 'lib/forms/scrollToFormError'import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast'+import { preflightLogic } from 'lib/logic/preflightLogic'import { cohortBlockingWorkflowsModalLogic } from 'scenes/cohorts/cohortBlockingWorkflowsModalLogic'import { CohortLogicProps, REALTIME_POLL_INTERVAL_MS, cohortEditLogic } from 'scenes/cohorts/cohortEditLogic'import { CRITERIA_VALIDATIONS, NEW_CRITERIA, ROWS } from 'scenes/cohorts/CohortFilters/constants'@@ -1184,6 +1186,44 @@ describe('cohortEditLogic', () => {expect(api.update).toHaveBeenCalledTimes(0)})+ const MB = 1024 * 1024++ it.each([+ { case: 'over the default limit', isStatic: true, configuredMax: undefined, blocked: true },+ { case: 'under a larger configured limit', isStatic: true, configuredMax: 50 * MB, blocked: false },+ { case: 'left on a cohort switched to dynamic', isStatic: false, configuredMax: undefined, blocked: false },+ ])('csv size check: $case', async ({ isStatic, configuredMax, blocked }) => {+ await initCohortLogic({ id: 1 })+ preflightLogic.actions.loadPreflightSuccess({+ ...preflightLogic.values.preflight,+ cohort_csv_upload_max_bytes: configuredMax,+ } as any)+ const csv = new File(['user_1'], 'ids.csv')+ Object.defineProperty(csv, 'size', { value: 20 * MB + 1 })+ logic.actions.setCohort({ ...mockCohort, is_static: isStatic, csv })+ expect(logic.values.cohortValidationErrors.csv).toEqual(+ blocked+ ? 'CSV file is larger than the limit of 20 MB. Split the file into smaller parts and upload them one at a time.'+ : undefined+ )+ })++ it('shows the size limit when the save is rejected with a 413', async () => {+ await initCohortLogic({ id: 1 })+ ;(api.update as jest.Mock).mockRejectedValueOnce(new ApiError('Payload too large', 413))+ await expectLogic(logic, async () => {+ logic.actions.setCohort({+ ...mockCohort,+ is_static: true,+ groups: [],+ csv: new File(['user_1'], 'ids.csv'),+ })+ logic.actions.setStaticCohortMode('people')+ logic.actions.submitCohort()+ }).toDispatchActions(['submitCohort', 'saveCohortSuccess'])+ expect(lemonToast.error).toHaveBeenCalledWith(expect.stringContaining('larger than the limit of 20 MB'))+ })+it('calls scrollToFormError with fallback message on submitCohortFailure', async () => {await initCohortLogic({ id: 1 })const mockScrollToFormError = scrollToFormError as jest.Mockdiff --git a/frontend/src/scenes/cohorts/cohortEditLogic.ts b/frontend/src/scenes/cohorts/cohortEditLogic.tsindex 271eb31032f2..beb975e7d157 100644--- a/frontend/src/scenes/cohorts/cohortEditLogic.ts+++ b/frontend/src/scenes/cohorts/cohortEditLogic.ts@@ -27,6 +27,7 @@ import { SetupTaskId, globalSetupLogic } from 'lib/components/ProductSetup'import { ENTITY_MATCH_TYPE } from 'lib/constants'import { scrollToFormError } from 'lib/forms/scrollToFormError'import { lemonToast } from 'lib/lemon-ui/LemonToast/LemonToast'+import { preflightLogic } from 'lib/logic/preflightLogic'import { eventUsageLogic } from 'lib/utils/eventUsageLogic'import { getCurrentTeamId } from 'lib/utils/getAppContext'import { objectsEqual } from 'lib/utils/objects'@@ -45,6 +46,8 @@ import {createCohortFormData,isCohortCriteriaGroup,validateGroup,+ cohortCsvMaxBytes,+ cohortCsvTooLargeMessage,} from 'scenes/cohorts/cohortUtils'import { teamLogic } from 'scenes/teamLogic'import { urls } from 'scenes/urls'@@ -78,6 +81,7 @@ import type {import { personsLogic } from 'products/persons/frontend/logics/personsLogic'import type { UserBasicType } from '../../types'+import type { PreflightStatus } from '../../types'export type CohortLogicProps = {id?: CohortType['id']@@ -92,6 +96,8 @@ const isCohortEditTab = (value: unknown): value is CohortEditTab => value === 'o// While history is being built, flags can't target the cohort yet, so the scene keeps asking. The// build takes tens of minutes, so this is slow polling, unlike the 1s loop the daily calculation uses.export const REALTIME_POLL_INTERVAL_MS = 15_000++const PAYLOAD_TOO_LARGE_STATUS = 413const BUILDING_STATES: CohortRealtimeStateEnumApi[] = ['building', 'rebuilding']const isBuildingHistory = (cohort: CohortType): boolean =>@@ -125,6 +131,7 @@ const inferStaticCohortMode = (cohort: CohortType): StaticCohortMode =>// Generated by kea-typegen. Update if you're an agent, ignore if you're human.export interface cohortEditLogicValues {+ preflight: PreflightStatus | null // preflightLogiccurrentProjectId: number | string // teamLogicactiveTab: CohortEditTabcanRemovePersonFromCohort: boolean | undefined@@ -140,6 +147,7 @@ export interface cohortEditLogicValues {cohortTouches: Record<string, boolean>cohortValidationErrors: DeepPartialMap<CohortType, ValidationErrorType>creationPersonQuery: ActorsQuery+ csvMaxBytes: numberduplicatedCohort: CohortType | nullduplicatedCohortLoading: booleaneffectiveQuery: DataTableNode@@ -532,6 +540,7 @@ export interface cohortEditLogicMeta {effectiveQuery: (query: DataTableNode, persistedColumns: string[] | null) => DataTableNodecanRemovePersonFromCohort: (cohort: CohortType) => boolean | undefinedisPendingCalculation: (cohort: CohortType) => boolean+ csvMaxBytes: (preflight: PreflightStatus | null) => numberisCalculatingOrPending: (cohort: CohortType, isPendingCalculation: boolean) => boolean}}@@ -556,7 +565,7 @@ export const cohortEditLogic = kea<cohortEditLogicType>([key((props) => (props.id === 'new' || !props.id ? 'new' : props.id)),path(['scenes', 'cohorts', 'cohortLogicEdit']),connect(() => ({- values: [teamLogic, ['currentProjectId']],+ values: [teamLogic, ['currentProjectId'], preflightLogic, ['preflight']],actions: [eventUsageLogic,['reportExperimentExposureCohortEdited'],@@ -840,6 +849,7 @@ export const cohortEditLogic = kea<cohortEditLogicType>([},],isPendingCalculation: [(s) => [s.cohort], (cohort: CohortType) => checkIsPendingCalculation(cohort)],+ csvMaxBytes: [(s) => [s.preflight], (preflight: PreflightStatus | null) => cohortCsvMaxBytes(preflight)],isCalculatingOrPending: [(s) => [s.cohort, s.isPendingCalculation],(cohort: CohortType, isPendingCalculation: boolean) => {@@ -851,9 +861,12 @@ export const cohortEditLogic = kea<cohortEditLogicType>([forms(({ actions, values }) => ({cohort: {defaults: NEW_COHORT,- errors: ({ name, is_static, filters }) => ({+ errors: ({ name, is_static, filters, csv }) => ({name: !name ? 'Cohort name cannot be empty' : undefined,- csv: undefined,+ csv:+ is_static && csv && csv.size > values.csvMaxBytes+ ? cohortCsvTooLargeMessage(values.csvMaxBytes)+ : undefined,filters: {properties: {values:@@ -1011,6 +1024,11 @@ export const cohortEditLogic = kea<cohortEditLogicType>([}}+ if (error.status === PAYLOAD_TOO_LARGE_STATUS) {+ lemonToast.error(cohortCsvTooLargeMessage(values.csvMaxBytes))+ return values.cohort+ }+lemonToast.error(error.detail || 'Failed to save cohort')return values.cohort}diff --git a/frontend/src/scenes/cohorts/cohortUtils.tsx b/frontend/src/scenes/cohorts/cohortUtils.tsxindex cbbe7f5ab691..154c02477c1a 100644--- a/frontend/src/scenes/cohorts/cohortUtils.tsx+++ b/frontend/src/scenes/cohorts/cohortUtils.tsx@@ -49,6 +49,7 @@ import {PropertyFilterType,PropertyFilterValue,PropertyOperator,+ PreflightStatus,PropertyType,TimeUnitType,} from '~/types'@@ -144,6 +145,21 @@ export function isValidCohortGroup(criteria: AnyCohortGroupType): boolean {)}+// Mirrors the COHORT_CSV_UPLOAD_MAX_BYTES default, for when preflight has not loaded.+const DEFAULT_COHORT_CSV_MAX_BYTES = 20 * 1024 * 1024++export function cohortCsvMaxBytes(preflight: PreflightStatus | null | undefined): number {+ return preflight?.cohort_csv_upload_max_bytes ?? DEFAULT_COHORT_CSV_MAX_BYTES+}++export function formatCohortCsvLimit(maxBytes: number): string {+ return `${Math.round((maxBytes / (1024 * 1024)) * 10) / 10} MB`+}++export function cohortCsvTooLargeMessage(maxBytes: number): string {+ return `CSV file is larger than the limit of ${formatCohortCsvLimit(maxBytes)}. Split the file into smaller parts and upload them one at a time.`+}+export function createCohortFormData(cohort: CohortType,{ preserveStaticFilters = false }: { preserveStaticFilters?: boolean } = {}diff --git a/frontend/src/types.ts b/frontend/src/types.tsindex ac74923a4cb9..bc37d1e184e5 100644--- a/frontend/src/types.ts+++ b/frontend/src/types.ts@@ -4890,6 +4890,7 @@ export interface PreflightStatus {site_url?: stringinstance_preferences?: InstancePreferencesInterfacebuffer_conversion_seconds?: number+ cohort_csv_upload_max_bytes?: number/** Whether the instance has an MCP server that the WebMCP proxy can reach. */webmcp_available?: booleanobject_storage: booleandiff --git a/posthog/api/test/test_preflight.py b/posthog/api/test/test_preflight.pyindex 7e68a970813b..1ca15ed27937 100644--- a/posthog/api/test/test_preflight.py+++ b/posthog/api/test/test_preflight.py@@ -129,6 +129,7 @@ class TestPreflight(APIBaseTest, QueryMatchingTest):"instance_preferences": {"debug_queries": True, "disable_paid_fs": False},"object_storage": False,"buffer_conversion_seconds": 60,+ "cohort_csv_upload_max_bytes": 20 * 1024 * 1024,# we calculate this here because otherwise it is non-deterministic when running locally# it can be overridden in tests by passing in options"openai_available": bool(os.environ.get("OPENAI_API_KEY")),diff --git a/posthog/settings/cohorts.py b/posthog/settings/cohorts.pyindex d37217c575ff..c2ccdd862e58 100644--- a/posthog/settings/cohorts.py+++ b/posthog/settings/cohorts.py@@ -114,7 +114,6 @@ COHORT_DEPENDENCY_INCREMENTAL_MAINTENANCE: bool = get_from_env("COHORT_DEPENDENCY_INCREMENTAL_MAINTENANCE", True, type_cast=str_to_bool)# Bounds one static cohort CSV upload. The IDs are collected in memory and sent to Celery in one-# message, so the byte size and the row count both need a ceiling. 50 MB matches MAX_UPLOAD_SIZE_BYTES-# in products/warehouse_sources/backend/file_uploads.py.-COHORT_CSV_UPLOAD_MAX_BYTES: int = get_from_env("COHORT_CSV_UPLOAD_MAX_BYTES", 50 * 1024 * 1024, type_cast=int)+# message, so the byte size and the row count both need a ceiling. 20 MB stays under proxy body limits.+COHORT_CSV_UPLOAD_MAX_BYTES: int = get_from_env("COHORT_CSV_UPLOAD_MAX_BYTES", 20 * 1024 * 1024, type_cast=int)COHORT_CSV_UPLOAD_MAX_ROWS: int = get_from_env("COHORT_CSV_UPLOAD_MAX_ROWS", 1_000_000, type_cast=int)diff --git a/posthog/views.py b/posthog/views.pyindex 0708a6796419..a0264e60396a 100644--- a/posthog/views.py+++ b/posthog/views.py@@ -297,6 +297,7 @@ def preflight_check(request: HttpRequest) -> JsonResponse:"site_url": settings.SITE_URL,"instance_preferences": settings.INSTANCE_PREFERENCES,"buffer_conversion_seconds": settings.BUFFER_CONVERSION_SECONDS,+ "cohort_csv_upload_max_bytes": settings.COHORT_CSV_UPLOAD_MAX_BYTES,"webmcp_available": webmcp_api.is_available(),}