:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(retention): add the exclude incomplete periods option to retention (#119651)
Sam Penningtond159450f30ba
diff --git a/posthog/hogql_queries/utils/query_date_range.py b/posthog/hogql_queries/utils/query_date_range.pyindex 99434a145d78..a26acc667f41 100644--- a/posthog/hogql_queries/utils/query_date_range.py+++ b/posthog/hogql_queries/utils/query_date_range.py@@ -572,7 +572,9 @@ class QueryDateRangeWithIntervals(QueryDateRange):# if date_from is present in retention query then use itif self._date_range and self._date_range.date_from:- date_from = super().date_from()+ # Unclipped: the start is already widened to a whole interval, so the base class's+ # excludeIncompletePeriods start clip would drop a complete cohort.+ date_from = self._date_from_unclipped()return date_to_start_of_interval(date_from, self._interval, self._team)# otherwise calculate from date_to and lookaheaddiff --git a/posthog/hogql_queries/utils/test/test_query_date_range.py b/posthog/hogql_queries/utils/test/test_query_date_range.pyindex f4c3d12aac71..5160ecd64ca1 100644--- a/posthog/hogql_queries/utils/test/test_query_date_range.py+++ b/posthog/hogql_queries/utils/test/test_query_date_range.py@@ -799,6 +799,25 @@ class TestQueryDateRangeWithIntervals(APIBaseTest):self.assertEqual(query.intervals_between, _count_intervals_by_stepping(query))+ @parameterized.expand(+ [+ ("daily", IntervalType.DAY, "-7d", "2021-08-18T00:00:00Z", "2021-08-25T00:00:00Z"),+ ("weekly_from_mid_week", IntervalType.WEEK, "-4w", "2021-07-25T00:00:00Z", "2021-08-22T00:00:00Z"),+ ]+ )+ def test_exclude_incomplete_periods_drops_only_the_current_cohort(+ self, _name: str, interval: IntervalType, date_from: str, expected_from: str, expected_to: str+ ):+ now = parser.isoparse("2021-08-25T12:00:00Z")+ included = QueryDateRangeWithIntervals(DateRange(date_from=date_from), 8, self.team, interval, now)+ excluded = QueryDateRangeWithIntervals(+ DateRange(date_from=date_from, excludeIncompletePeriods=True), 8, self.team, interval, now+ )++ self.assertEqual(excluded.date_from(), parser.isoparse(expected_from))+ self.assertEqual(excluded.date_to(), parser.isoparse(expected_to))+ self.assertEqual(excluded.intervals_between, included.intervals_between - 1)+def test_determine_time_delta_valid(self):delta = QueryDateRangeWithIntervals.determine_time_delta(5, "day")self.assertEqual(delta, timedelta(days=5))diff --git a/products/product_analytics/backend/hogql_queries/retention/retention_query_runner.py b/products/product_analytics/backend/hogql_queries/retention/retention_query_runner.pyindex 4a584c89d854..c12ff5d7770f 100644--- a/products/product_analytics/backend/hogql_queries/retention/retention_query_runner.py+++ b/products/product_analytics/backend/hogql_queries/retention/retention_query_runner.py@@ -393,8 +393,13 @@ class RetentionQueryRunner(AnalyticsQueryRunner[RetentionQueryResponse]):else DEFAULT_INTERVAL)+ date_range = self.query.dateRange+ if self.is_24h_window_calculation and date_range and date_range.excludeIncompletePeriods:+ # Rolling 24-hour windows end mid-day, so clipping at the period boundary would drop completed returns+ date_range = date_range.model_copy(update={"excludeIncompletePeriods": None})+return QueryDateRangeWithIntervals(- date_range=self.query.dateRange,+ date_range=date_range,total_intervals=total_intervals,team=self.team,interval=interval,diff --git a/products/product_analytics/backend/hogql_queries/retention/test/test_retention_query_runner_filters.py b/products/product_analytics/backend/hogql_queries/retention/test/test_retention_query_runner_filters.pyindex ef0dc702cdc6..931d86c0e5cc 100644--- a/products/product_analytics/backend/hogql_queries/retention/test/test_retention_query_runner_filters.py+++ b/products/product_analytics/backend/hogql_queries/retention/test/test_retention_query_runner_filters.py@@ -1,8 +1,11 @@+from datetime import datetime++import time_machinefrom posthog.test.base import BaseTestfrom parameterized import parameterized-from posthog.schema import DashboardFilter, IntervalType, RetentionFilter, RetentionQuery+from posthog.schema import DashboardFilter, DateRange, IntervalType, RetentionFilter, RetentionQueryfrom products.product_analytics.backend.hogql_queries.retention.retention_query_runner import RetentionQueryRunner@@ -35,3 +38,22 @@ class TestRetentionDashboardFilters(BaseTest):runner.apply_dashboard_filters(DashboardFilter(filterTestAccounts=dashboard_filter))assert runner.query.filterTestAccounts is expected+++class TestRetentionExcludeIncompletePeriods(BaseTest):+ @parameterized.expand(+ [+ ("calendar_dates_clip_the_current_day", "strict_calendar_dates", True),+ ("rolling_windows_ignore_the_exclusion", "24_hour_windows", False),+ ]+ )+ @time_machine.travel("2024-01-10T12:00:00Z", tick=False)+ def test_exclude_incomplete_periods(self, _name: str, time_window_mode: str, clipped: bool) -> None:+ def date_to(exclude: bool) -> datetime:+ query = RetentionQuery(+ retentionFilter=RetentionFilter(timeWindowMode=time_window_mode),+ dateRange=DateRange(date_from="-7d", excludeIncompletePeriods=exclude),+ )+ return RetentionQueryRunner(query=query, team=self.team).query_date_range.date_to()++ assert (date_to(True) < date_to(False)) is clippeddiff --git a/products/product_analytics/frontend/insights/retention/filters/RetentionDatePicker.tsx b/products/product_analytics/frontend/insights/retention/filters/RetentionDatePicker.tsxindex aaab037043ca..52a1dce1a0ed 100644--- a/products/product_analytics/frontend/insights/retention/filters/RetentionDatePicker.tsx+++ b/products/product_analytics/frontend/insights/retention/filters/RetentionDatePicker.tsx@@ -4,14 +4,21 @@ import { IconCalendar, IconInfo } from '@posthog/icons'import { Tooltip } from '@posthog/lemon-ui'import { DateFilter } from 'lib/components/DateFilter/DateFilter'+import { type DateFilterExclusions } from 'lib/components/DateFilter/DateFilterExclusionsControl'+import { eventUsageLogic } from 'lib/utils/eventUsageLogic'import { insightLogic } from 'scenes/insights/insightLogic'+import { NodeKind } from '~/queries/schema/schema-general'+import { retentionLogic } from '../retentionLogic'export function RetentionDatePicker(): JSX.Element {const { insightProps, editingDisabledReason } = useValues(insightLogic)- const { dateRange, dateMappings } = useValues(retentionLogic(insightProps))+ const { dateRange, dateMappings, retentionFilter } = useValues(retentionLogic(insightProps))const { updateDateRange } = useActions(retentionLogic(insightProps))+ const { reportInsightDateExclusionsChanged } = useActions(eventUsageLogic)++ const exclusions: DateFilterExclusions = { days: [], incomplete: !!dateRange?.excludeIncompletePeriods }return (<DateFilter@@ -21,6 +28,14 @@ export function RetentionDatePicker(): JSX.Element {updateDateRange({ date_from, date_to, explicitDate: explicit_date })}}dateOptions={dateMappings}+ exclusions={exclusions}+ onExclusionsChange={({ incomplete }) => {+ if (incomplete !== exclusions.incomplete) {+ updateDateRange({ excludeIncompletePeriods: incomplete ? true : null }, true)+ reportInsightDateExclusionsChanged(NodeKind.RetentionQuery, incomplete, 0)+ }+ }}+ showIncompletePeriodExclusion={retentionFilter?.timeWindowMode !== '24_hour_windows'}makeLabel={(key) => (<><IconCalendar /> {key}diff --git a/products/product_analytics/frontend/insights/retention/retentionGraphLogic.ts b/products/product_analytics/frontend/insights/retention/retentionGraphLogic.tsindex 58eaa803ee45..21a206133299 100644--- a/products/product_analytics/frontend/insights/retention/retentionGraphLogic.ts+++ b/products/product_analytics/frontend/insights/retention/retentionGraphLogic.ts@@ -44,6 +44,7 @@ export interface retentionGraphLogicValues {| null // insightVizDataLogicretentionFilter: RetentionFilter | null // insightVizDataLogicbreakdownDisplayNames: Record<string, string> // retentionLogic+ excludeIncompletePeriods: boolean // retentionLogicfilteredResults: ProcessedRetentionPayload[] // retentionLogicgetRetentionColor: (rawBreakdownValue: number | string | null | undefined,@@ -98,7 +99,8 @@ export interface retentionGraphLogicMeta {dateRange: DateRange | null | undefined,retentionFilter: RetentionFilter | null,trendSeries: RetentionTrendPayload[],- timezone: string+ timezone: string,+ excludeIncompletePeriods: boolean) => numbershouldShowMeanPerBreakdown: (hasValidBreakdown: boolean,@@ -155,6 +157,7 @@ export const retentionGraphLogic = kea<retentionGraphLogicType>(['breakdownDisplayNames','isPropertyValueAggregation','getRetentionColor',+ 'excludeIncompletePeriods',],teamLogic,['timezone'],@@ -283,18 +286,19 @@ export const retentionGraphLogic = kea<retentionGraphLogicType>([],incompletenessOffsetFromEnd: [- (s) => [s.dateRange, s.retentionFilter, s.trendSeries, s.timezone],+ (s) => [s.dateRange, s.retentionFilter, s.trendSeries, s.timezone, s.excludeIncompletePeriods],(dateRange: DateRange | null | undefined,retentionFilter: RetentionFilter | null | undefined,trendSeries: RetentionTrendPayload[],- timezone: string+ timezone: string,+ excludeIncompletePeriods: boolean) => {const { date_to } = dateRange || {}const { period } = retentionFilter || {}// Returns negative number of points to paint over starting from end of array- if (!trendSeries?.[0]?.days) {+ if (!trendSeries?.[0]?.days || excludeIncompletePeriods) {return 0} else if (!date_to) {return -1diff --git a/products/product_analytics/frontend/insights/retention/retentionLogic.test.ts b/products/product_analytics/frontend/insights/retention/retentionLogic.test.tsindex c3beb5e0b177..5b4391ac28f6 100644--- a/products/product_analytics/frontend/insights/retention/retentionLogic.test.ts+++ b/products/product_analytics/frontend/insights/retention/retentionLogic.test.ts@@ -1,5 +1,6 @@import { expectLogic } from 'kea-test-utils'+import { dayjs } from 'lib/dayjs'import { teamLogic } from 'scenes/teamLogic'import { useMocks } from '~/mocks/jest'@@ -127,4 +128,74 @@ describe('retentionLogic', () => {expect(logic.values.allCohortsEmpty).toBe(allCohortsEmpty)expect(logic.values.hasEntityPropertyFilters).toBe(hasEntityPropertyFilters)})++ const daysAgo = (days: number): string => dayjs().utc().startOf('day').subtract(days, 'day').toISOString()++ it.each([+ {+ name: 'drops the current period when excluded',+ query: { ...retentionQuery(), dateRange: { excludeIncompletePeriods: true } },+ rows: [cohortRow(daysAgo(2), [100, 50, 20])],+ expectedCounts: [100, 50],+ },+ {+ name: 'keeps the current period when not excluded',+ query: retentionQuery(),+ rows: [cohortRow(daysAgo(2), [100, 50, 20])],+ expectedCounts: [100, 50, 20],+ },+ {+ name: 'drops a custom bracket that has not ended',+ query: {+ ...retentionQuery(),+ retentionFilter: { ...retentionQuery().retentionFilter, retentionCustomBrackets: [7, 7] },+ dateRange: { excludeIncompletePeriods: true },+ },+ rows: [cohortRow(daysAgo(10), [100, 40, 30])],+ expectedCounts: [100, 40],+ },+ {+ name: 'treats empty custom brackets as ordinary retention',+ query: {+ ...retentionQuery(),+ retentionFilter: { ...retentionQuery().retentionFilter, retentionCustomBrackets: [] },+ dateRange: { excludeIncompletePeriods: true },+ },+ rows: [cohortRow(daysAgo(2), [100, 50, 20])],+ expectedCounts: [100, 50],+ },+ {+ name: 'ignores the exclusion for 24-hour windows',+ query: {+ ...retentionQuery(),+ retentionFilter: { ...retentionQuery().retentionFilter, timeWindowMode: '24_hour_windows' as const },+ dateRange: { excludeIncompletePeriods: true },+ },+ rows: [cohortRow(daysAgo(2), [100, 50, 20])],+ expectedCounts: [100, 50, 20],+ },+ ])('$name', async ({ query, rows, expectedCounts }) => {+ await loadResults(query, rows)++ expect(logic.values.results[0].values.map((value) => value.count)).toEqual(expectedCounts)+ })++ it('drops the current week of a Monday-start project on a Sunday', async () => {+ jest.useFakeTimers({ advanceTimers: true })+ jest.setSystemTime(new Date('2024-01-14T12:00:00Z'))+ try {+ await loadResults(+ {+ ...retentionQuery(),+ retentionFilter: { ...retentionQuery().retentionFilter, period: RetentionPeriod.Week },+ dateRange: { excludeIncompletePeriods: true },+ },+ [cohortRow('2024-01-01T00:00:00Z', [100, 50, 20])]+ )++ expect(logic.values.results[0].values.map((value) => value.count)).toEqual([100])+ } finally {+ jest.useRealTimers()+ }+ })})diff --git a/products/product_analytics/frontend/insights/retention/retentionLogic.ts b/products/product_analytics/frontend/insights/retention/retentionLogic.tsindex 44dc61aeb15c..5a3161a445cf 100644--- a/products/product_analytics/frontend/insights/retention/retentionLogic.ts+++ b/products/product_analytics/frontend/insights/retention/retentionLogic.ts@@ -84,6 +84,7 @@ export interface retentionLogicValues {breakdownDisplayNames: Record<string, string>breakdownValues: (number | string | null | undefined)[]dateMappings: DateMappingOption[]+ excludeIncompletePeriods: booleanfilteredResults: ProcessedRetentionPayload[]getRetentionColor: (rawBreakdownValue: number | string | null | undefined,@@ -147,12 +148,17 @@ export interface retentionLogicMeta {isPropertyValueAggregation: (retentionFilter: RetentionFilter | null) => booleanhasEntityPropertyFilters: (retentionFilter: RetentionFilter | null) => booleanallCohortsEmpty: (results: ProcessedRetentionPayload[]) => boolean+ excludeIncompletePeriods: (+ dateRange: DateRange | null | undefined,+ retentionFilter: RetentionFilter | null+ ) => booleanresults: (insightQuery: DataNode<Record<string, any>>,insightData: Record<string, any>,retentionFilter: RetentionFilter | null,timezone: string,- isPropertyValueAggregation: boolean+ isPropertyValueAggregation: boolean,+ excludeIncompletePeriods: boolean) => ProcessedRetentionPayload[]filteredResults: (results: ProcessedRetentionPayload[],@@ -353,19 +359,43 @@ export const retentionLogic = kea<retentionLogicType>([(results: ProcessedRetentionPayload[]): boolean =>results.length > 0 && results.every((result) => (result.values[0]?.count ?? 0) === 0),],+ excludeIncompletePeriods: [+ (s) => [s.dateRange, s.retentionFilter],+ // Rolling 24-hour windows end mid-day, so clipping at the period boundary would drop completed returns+ (dateRange: DateRange | null | undefined, retentionFilter: RetentionFilter | null): boolean =>+ !!dateRange?.excludeIncompletePeriods && retentionFilter?.timeWindowMode !== '24_hour_windows',+ ],results: [- (s) => [s.insightQuery, s.insightData, s.retentionFilter, s.timezone, s.isPropertyValueAggregation],+ (s) => [+ s.insightQuery,+ s.insightData,+ s.retentionFilter,+ s.timezone,+ s.isPropertyValueAggregation,+ s.excludeIncompletePeriods,+ ],(insightQuery: import('~/queries/schema/schema-general').DataNode<Record<string, any>>,insightData: Record<string, any>,retentionFilter: RetentionFilter | null,timezone: string,- isPropertyValueAggregation: boolean+ isPropertyValueAggregation: boolean,+ excludeIncompletePeriods: boolean): ProcessedRetentionPayload[] => {if (isPropertyValueAggregation && !retentionFilter?.aggregationProperty) {return []}+ const periodUnit = (retentionFilter?.period ?? RetentionPeriod.Day).toLowerCase() as dayjs.UnitTypeLong+ const now = dayjs().tz(timezone)+ const brackets = retentionFilter?.retentionCustomBrackets+ // A cell is complete once its last period ends; custom bracket cells span several periods+ const isIncomplete = (cohortDate: dayjs.ConfigType, index: number): boolean =>+ dayjs(cohortDate)+ .tz(timezone)+ .add(brackets?.length ? sum(brackets.slice(0, index)) + 1 : index + 1, periodUnit)+ .isAfter(now)+const rawResults = isRetentionQuery(insightQuery) ? (insightData?.result ?? []) : []const results: ProcessedRetentionPayload[] = rawResults.map((result: RetentionResult) => ({@@ -378,11 +408,7 @@ export const retentionLogic = kea<retentionLogicType>([retentionFilter?.retentionReference === 'previous' ? previousCount : totalCountconst percentage = referenceCount > 0 ? (value['count'] / referenceCount) * 100 : 0- const periodUnit = (- retentionFilter?.period ?? RetentionPeriod.Day- ).toLowerCase() as dayjs.UnitTypeLongconst cellDate = dayjs(result.date).tz(timezone).add(index, periodUnit)- const now = dayjs().tz(timezone)return {...value,@@ -394,7 +420,7 @@ export const retentionLogic = kea<retentionLogicType>([}),}))- // Filter out future values and handle breakdown label+ // Filter out future values (and the current period when excluded) and handle breakdown labelreturn results.map((result) => ({...result,date: dayjs(result.date).tz(timezone),@@ -406,7 +432,10 @@ export const retentionLogic = kea<retentionLogicType>([// Dashboard breakdown color configs store the raw value, so keep it// alongside the display-mapped one.rawBreakdownValue: result.breakdown_value ?? null,- values: result.values.filter((value) => !value.isFuture),+ values: result.values.filter(+ (value, index) =>+ !value.isFuture && !(excludeIncompletePeriods && isIncomplete(result.date, index))+ ),}))},],