Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(retention): add the exclude incomplete periods option to retention (#119651)

Sam Penningtond159450f30ba
diff --git a/posthog/hogql_queries/utils/query_date_range.py b/posthog/hogql_queries/utils/query_date_range.py
index 99434a145d78..a26acc667f41 100644
--- a/posthog/hogql_queries/utils/query_date_range.py
+++ b/posthog/hogql_queries/utils/query_date_range.py
@@ -572,7 +572,9 @@ class QueryDateRangeWithIntervals(QueryDateRange):
# if date_from is present in retention query then use it
if self._date_range and self._date_range.date_from:
- date_from = super().date_from()
+ # Unclipped: the start is already widened to a whole interval, so the base class's
+ # excludeIncompletePeriods start clip would drop a complete cohort.
+ date_from = self._date_from_unclipped()
return date_to_start_of_interval(date_from, self._interval, self._team)
# otherwise calculate from date_to and lookahead
diff --git a/posthog/hogql_queries/utils/test/test_query_date_range.py b/posthog/hogql_queries/utils/test/test_query_date_range.py
index f4c3d12aac71..5160ecd64ca1 100644
--- a/posthog/hogql_queries/utils/test/test_query_date_range.py
+++ b/posthog/hogql_queries/utils/test/test_query_date_range.py
@@ -799,6 +799,25 @@ class TestQueryDateRangeWithIntervals(APIBaseTest):
self.assertEqual(query.intervals_between, _count_intervals_by_stepping(query))
+ @parameterized.expand(
+ [
+ ("daily", IntervalType.DAY, "-7d", "2021-08-18T00:00:00Z", "2021-08-25T00:00:00Z"),
+ ("weekly_from_mid_week", IntervalType.WEEK, "-4w", "2021-07-25T00:00:00Z", "2021-08-22T00:00:00Z"),
+ ]
+ )
+ def test_exclude_incomplete_periods_drops_only_the_current_cohort(
+ self, _name: str, interval: IntervalType, date_from: str, expected_from: str, expected_to: str
+ ):
+ now = parser.isoparse("2021-08-25T12:00:00Z")
+ included = QueryDateRangeWithIntervals(DateRange(date_from=date_from), 8, self.team, interval, now)
+ excluded = QueryDateRangeWithIntervals(
+ DateRange(date_from=date_from, excludeIncompletePeriods=True), 8, self.team, interval, now
+ )
+
+ self.assertEqual(excluded.date_from(), parser.isoparse(expected_from))
+ self.assertEqual(excluded.date_to(), parser.isoparse(expected_to))
+ self.assertEqual(excluded.intervals_between, included.intervals_between - 1)
+
def test_determine_time_delta_valid(self):
delta = QueryDateRangeWithIntervals.determine_time_delta(5, "day")
self.assertEqual(delta, timedelta(days=5))
diff --git a/products/product_analytics/backend/hogql_queries/retention/retention_query_runner.py b/products/product_analytics/backend/hogql_queries/retention/retention_query_runner.py
index 4a584c89d854..c12ff5d7770f 100644
--- a/products/product_analytics/backend/hogql_queries/retention/retention_query_runner.py
+++ b/products/product_analytics/backend/hogql_queries/retention/retention_query_runner.py
@@ -393,8 +393,13 @@ class RetentionQueryRunner(AnalyticsQueryRunner[RetentionQueryResponse]):
else DEFAULT_INTERVAL
)
+ date_range = self.query.dateRange
+ if self.is_24h_window_calculation and date_range and date_range.excludeIncompletePeriods:
+ # Rolling 24-hour windows end mid-day, so clipping at the period boundary would drop completed returns
+ date_range = date_range.model_copy(update={"excludeIncompletePeriods": None})
+
return QueryDateRangeWithIntervals(
- date_range=self.query.dateRange,
+ date_range=date_range,
total_intervals=total_intervals,
team=self.team,
interval=interval,
diff --git a/products/product_analytics/backend/hogql_queries/retention/test/test_retention_query_runner_filters.py b/products/product_analytics/backend/hogql_queries/retention/test/test_retention_query_runner_filters.py
index ef0dc702cdc6..931d86c0e5cc 100644
--- a/products/product_analytics/backend/hogql_queries/retention/test/test_retention_query_runner_filters.py
+++ b/products/product_analytics/backend/hogql_queries/retention/test/test_retention_query_runner_filters.py
@@ -1,8 +1,11 @@
+from datetime import datetime
+
+import time_machine
from posthog.test.base import BaseTest
from parameterized import parameterized
-from posthog.schema import DashboardFilter, IntervalType, RetentionFilter, RetentionQuery
+from posthog.schema import DashboardFilter, DateRange, IntervalType, RetentionFilter, RetentionQuery
from products.product_analytics.backend.hogql_queries.retention.retention_query_runner import RetentionQueryRunner
@@ -35,3 +38,22 @@ class TestRetentionDashboardFilters(BaseTest):
runner.apply_dashboard_filters(DashboardFilter(filterTestAccounts=dashboard_filter))
assert runner.query.filterTestAccounts is expected
+
+
+class TestRetentionExcludeIncompletePeriods(BaseTest):
+ @parameterized.expand(
+ [
+ ("calendar_dates_clip_the_current_day", "strict_calendar_dates", True),
+ ("rolling_windows_ignore_the_exclusion", "24_hour_windows", False),
+ ]
+ )
+ @time_machine.travel("2024-01-10T12:00:00Z", tick=False)
+ def test_exclude_incomplete_periods(self, _name: str, time_window_mode: str, clipped: bool) -> None:
+ def date_to(exclude: bool) -> datetime:
+ query = RetentionQuery(
+ retentionFilter=RetentionFilter(timeWindowMode=time_window_mode),
+ dateRange=DateRange(date_from="-7d", excludeIncompletePeriods=exclude),
+ )
+ return RetentionQueryRunner(query=query, team=self.team).query_date_range.date_to()
+
+ assert (date_to(True) < date_to(False)) is clipped
diff --git a/products/product_analytics/frontend/insights/retention/filters/RetentionDatePicker.tsx b/products/product_analytics/frontend/insights/retention/filters/RetentionDatePicker.tsx
index aaab037043ca..52a1dce1a0ed 100644
--- a/products/product_analytics/frontend/insights/retention/filters/RetentionDatePicker.tsx
+++ b/products/product_analytics/frontend/insights/retention/filters/RetentionDatePicker.tsx
@@ -4,14 +4,21 @@ import { IconCalendar, IconInfo } from '@posthog/icons'
import { Tooltip } from '@posthog/lemon-ui'
import { DateFilter } from 'lib/components/DateFilter/DateFilter'
+import { type DateFilterExclusions } from 'lib/components/DateFilter/DateFilterExclusionsControl'
+import { eventUsageLogic } from 'lib/utils/eventUsageLogic'
import { insightLogic } from 'scenes/insights/insightLogic'
+import { NodeKind } from '~/queries/schema/schema-general'
+
import { retentionLogic } from '../retentionLogic'
export function RetentionDatePicker(): JSX.Element {
const { insightProps, editingDisabledReason } = useValues(insightLogic)
- const { dateRange, dateMappings } = useValues(retentionLogic(insightProps))
+ const { dateRange, dateMappings, retentionFilter } = useValues(retentionLogic(insightProps))
const { updateDateRange } = useActions(retentionLogic(insightProps))
+ const { reportInsightDateExclusionsChanged } = useActions(eventUsageLogic)
+
+ const exclusions: DateFilterExclusions = { days: [], incomplete: !!dateRange?.excludeIncompletePeriods }
return (
<DateFilter
@@ -21,6 +28,14 @@ export function RetentionDatePicker(): JSX.Element {
updateDateRange({ date_from, date_to, explicitDate: explicit_date })
}}
dateOptions={dateMappings}
+ exclusions={exclusions}
+ onExclusionsChange={({ incomplete }) => {
+ if (incomplete !== exclusions.incomplete) {
+ updateDateRange({ excludeIncompletePeriods: incomplete ? true : null }, true)
+ reportInsightDateExclusionsChanged(NodeKind.RetentionQuery, incomplete, 0)
+ }
+ }}
+ showIncompletePeriodExclusion={retentionFilter?.timeWindowMode !== '24_hour_windows'}
makeLabel={(key) => (
<>
<IconCalendar /> {key}
diff --git a/products/product_analytics/frontend/insights/retention/retentionGraphLogic.ts b/products/product_analytics/frontend/insights/retention/retentionGraphLogic.ts
index 58eaa803ee45..21a206133299 100644
--- a/products/product_analytics/frontend/insights/retention/retentionGraphLogic.ts
+++ b/products/product_analytics/frontend/insights/retention/retentionGraphLogic.ts
@@ -44,6 +44,7 @@ export interface retentionGraphLogicValues {
| null // insightVizDataLogic
retentionFilter: RetentionFilter | null // insightVizDataLogic
breakdownDisplayNames: Record<string, string> // retentionLogic
+ excludeIncompletePeriods: boolean // retentionLogic
filteredResults: ProcessedRetentionPayload[] // retentionLogic
getRetentionColor: (
rawBreakdownValue: number | string | null | undefined,
@@ -98,7 +99,8 @@ export interface retentionGraphLogicMeta {
dateRange: DateRange | null | undefined,
retentionFilter: RetentionFilter | null,
trendSeries: RetentionTrendPayload[],
- timezone: string
+ timezone: string,
+ excludeIncompletePeriods: boolean
) => number
shouldShowMeanPerBreakdown: (
hasValidBreakdown: boolean,
@@ -155,6 +157,7 @@ export const retentionGraphLogic = kea<retentionGraphLogicType>([
'breakdownDisplayNames',
'isPropertyValueAggregation',
'getRetentionColor',
+ 'excludeIncompletePeriods',
],
teamLogic,
['timezone'],
@@ -283,18 +286,19 @@ export const retentionGraphLogic = kea<retentionGraphLogicType>([
],
incompletenessOffsetFromEnd: [
- (s) => [s.dateRange, s.retentionFilter, s.trendSeries, s.timezone],
+ (s) => [s.dateRange, s.retentionFilter, s.trendSeries, s.timezone, s.excludeIncompletePeriods],
(
dateRange: DateRange | null | undefined,
retentionFilter: RetentionFilter | null | undefined,
trendSeries: RetentionTrendPayload[],
- timezone: string
+ timezone: string,
+ excludeIncompletePeriods: boolean
) => {
const { date_to } = dateRange || {}
const { period } = retentionFilter || {}
// Returns negative number of points to paint over starting from end of array
- if (!trendSeries?.[0]?.days) {
+ if (!trendSeries?.[0]?.days || excludeIncompletePeriods) {
return 0
} else if (!date_to) {
return -1
diff --git a/products/product_analytics/frontend/insights/retention/retentionLogic.test.ts b/products/product_analytics/frontend/insights/retention/retentionLogic.test.ts
index c3beb5e0b177..5b4391ac28f6 100644
--- a/products/product_analytics/frontend/insights/retention/retentionLogic.test.ts
+++ b/products/product_analytics/frontend/insights/retention/retentionLogic.test.ts
@@ -1,5 +1,6 @@
import { expectLogic } from 'kea-test-utils'
+import { dayjs } from 'lib/dayjs'
import { teamLogic } from 'scenes/teamLogic'
import { useMocks } from '~/mocks/jest'
@@ -127,4 +128,74 @@ describe('retentionLogic', () => {
expect(logic.values.allCohortsEmpty).toBe(allCohortsEmpty)
expect(logic.values.hasEntityPropertyFilters).toBe(hasEntityPropertyFilters)
})
+
+ const daysAgo = (days: number): string => dayjs().utc().startOf('day').subtract(days, 'day').toISOString()
+
+ it.each([
+ {
+ name: 'drops the current period when excluded',
+ query: { ...retentionQuery(), dateRange: { excludeIncompletePeriods: true } },
+ rows: [cohortRow(daysAgo(2), [100, 50, 20])],
+ expectedCounts: [100, 50],
+ },
+ {
+ name: 'keeps the current period when not excluded',
+ query: retentionQuery(),
+ rows: [cohortRow(daysAgo(2), [100, 50, 20])],
+ expectedCounts: [100, 50, 20],
+ },
+ {
+ name: 'drops a custom bracket that has not ended',
+ query: {
+ ...retentionQuery(),
+ retentionFilter: { ...retentionQuery().retentionFilter, retentionCustomBrackets: [7, 7] },
+ dateRange: { excludeIncompletePeriods: true },
+ },
+ rows: [cohortRow(daysAgo(10), [100, 40, 30])],
+ expectedCounts: [100, 40],
+ },
+ {
+ name: 'treats empty custom brackets as ordinary retention',
+ query: {
+ ...retentionQuery(),
+ retentionFilter: { ...retentionQuery().retentionFilter, retentionCustomBrackets: [] },
+ dateRange: { excludeIncompletePeriods: true },
+ },
+ rows: [cohortRow(daysAgo(2), [100, 50, 20])],
+ expectedCounts: [100, 50],
+ },
+ {
+ name: 'ignores the exclusion for 24-hour windows',
+ query: {
+ ...retentionQuery(),
+ retentionFilter: { ...retentionQuery().retentionFilter, timeWindowMode: '24_hour_windows' as const },
+ dateRange: { excludeIncompletePeriods: true },
+ },
+ rows: [cohortRow(daysAgo(2), [100, 50, 20])],
+ expectedCounts: [100, 50, 20],
+ },
+ ])('$name', async ({ query, rows, expectedCounts }) => {
+ await loadResults(query, rows)
+
+ expect(logic.values.results[0].values.map((value) => value.count)).toEqual(expectedCounts)
+ })
+
+ it('drops the current week of a Monday-start project on a Sunday', async () => {
+ jest.useFakeTimers({ advanceTimers: true })
+ jest.setSystemTime(new Date('2024-01-14T12:00:00Z'))
+ try {
+ await loadResults(
+ {
+ ...retentionQuery(),
+ retentionFilter: { ...retentionQuery().retentionFilter, period: RetentionPeriod.Week },
+ dateRange: { excludeIncompletePeriods: true },
+ },
+ [cohortRow('2024-01-01T00:00:00Z', [100, 50, 20])]
+ )
+
+ expect(logic.values.results[0].values.map((value) => value.count)).toEqual([100])
+ } finally {
+ jest.useRealTimers()
+ }
+ })
})
diff --git a/products/product_analytics/frontend/insights/retention/retentionLogic.ts b/products/product_analytics/frontend/insights/retention/retentionLogic.ts
index 44dc61aeb15c..5a3161a445cf 100644
--- a/products/product_analytics/frontend/insights/retention/retentionLogic.ts
+++ b/products/product_analytics/frontend/insights/retention/retentionLogic.ts
@@ -84,6 +84,7 @@ export interface retentionLogicValues {
breakdownDisplayNames: Record<string, string>
breakdownValues: (number | string | null | undefined)[]
dateMappings: DateMappingOption[]
+ excludeIncompletePeriods: boolean
filteredResults: ProcessedRetentionPayload[]
getRetentionColor: (
rawBreakdownValue: number | string | null | undefined,
@@ -147,12 +148,17 @@ export interface retentionLogicMeta {
isPropertyValueAggregation: (retentionFilter: RetentionFilter | null) => boolean
hasEntityPropertyFilters: (retentionFilter: RetentionFilter | null) => boolean
allCohortsEmpty: (results: ProcessedRetentionPayload[]) => boolean
+ excludeIncompletePeriods: (
+ dateRange: DateRange | null | undefined,
+ retentionFilter: RetentionFilter | null
+ ) => boolean
results: (
insightQuery: DataNode<Record<string, any>>,
insightData: Record<string, any>,
retentionFilter: RetentionFilter | null,
timezone: string,
- isPropertyValueAggregation: boolean
+ isPropertyValueAggregation: boolean,
+ excludeIncompletePeriods: boolean
) => ProcessedRetentionPayload[]
filteredResults: (
results: ProcessedRetentionPayload[],
@@ -353,19 +359,43 @@ export const retentionLogic = kea<retentionLogicType>([
(results: ProcessedRetentionPayload[]): boolean =>
results.length > 0 && results.every((result) => (result.values[0]?.count ?? 0) === 0),
],
+ excludeIncompletePeriods: [
+ (s) => [s.dateRange, s.retentionFilter],
+ // Rolling 24-hour windows end mid-day, so clipping at the period boundary would drop completed returns
+ (dateRange: DateRange | null | undefined, retentionFilter: RetentionFilter | null): boolean =>
+ !!dateRange?.excludeIncompletePeriods && retentionFilter?.timeWindowMode !== '24_hour_windows',
+ ],
results: [
- (s) => [s.insightQuery, s.insightData, s.retentionFilter, s.timezone, s.isPropertyValueAggregation],
+ (s) => [
+ s.insightQuery,
+ s.insightData,
+ s.retentionFilter,
+ s.timezone,
+ s.isPropertyValueAggregation,
+ s.excludeIncompletePeriods,
+ ],
(
insightQuery: import('~/queries/schema/schema-general').DataNode<Record<string, any>>,
insightData: Record<string, any>,
retentionFilter: RetentionFilter | null,
timezone: string,
- isPropertyValueAggregation: boolean
+ isPropertyValueAggregation: boolean,
+ excludeIncompletePeriods: boolean
): ProcessedRetentionPayload[] => {
if (isPropertyValueAggregation && !retentionFilter?.aggregationProperty) {
return []
}
+ const periodUnit = (retentionFilter?.period ?? RetentionPeriod.Day).toLowerCase() as dayjs.UnitTypeLong
+ const now = dayjs().tz(timezone)
+ const brackets = retentionFilter?.retentionCustomBrackets
+ // A cell is complete once its last period ends; custom bracket cells span several periods
+ const isIncomplete = (cohortDate: dayjs.ConfigType, index: number): boolean =>
+ dayjs(cohortDate)
+ .tz(timezone)
+ .add(brackets?.length ? sum(brackets.slice(0, index)) + 1 : index + 1, periodUnit)
+ .isAfter(now)
+
const rawResults = isRetentionQuery(insightQuery) ? (insightData?.result ?? []) : []
const results: ProcessedRetentionPayload[] = rawResults.map((result: RetentionResult) => ({
@@ -378,11 +408,7 @@ export const retentionLogic = kea<retentionLogicType>([
retentionFilter?.retentionReference === 'previous' ? previousCount : totalCount
const percentage = referenceCount > 0 ? (value['count'] / referenceCount) * 100 : 0
- const periodUnit = (
- retentionFilter?.period ?? RetentionPeriod.Day
- ).toLowerCase() as dayjs.UnitTypeLong
const cellDate = dayjs(result.date).tz(timezone).add(index, periodUnit)
- const now = dayjs().tz(timezone)
return {
...value,
@@ -394,7 +420,7 @@ export const retentionLogic = kea<retentionLogicType>([
}),
}))
- // Filter out future values and handle breakdown label
+ // Filter out future values (and the current period when excluded) and handle breakdown label
return results.map((result) => ({
...result,
date: dayjs(result.date).tz(timezone),
@@ -406,7 +432,10 @@ export const retentionLogic = kea<retentionLogicType>([
// Dashboard breakdown color configs store the raw value, so keep it
// alongside the display-mapped one.
rawBreakdownValue: result.breakdown_value ?? null,
- values: result.values.filter((value) => !value.isFuture),
+ values: result.values.filter(
+ (value, index) =>
+ !value.isFuture && !(excludeIncompletePeriods && isIncomplete(result.date, index))
+ ),
}))
},
],