Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(logs): bill custom retention one month at a time (#109787)

Frank Hamand6e20a50ad8cd
diff --git a/nodejs/src/common/services/app-metrics-aggregator.test.ts b/nodejs/src/common/services/app-metrics-aggregator.test.ts
index f8eced70b277..d67acc99d532 100644
--- a/nodejs/src/common/services/app-metrics-aggregator.test.ts
+++ b/nodejs/src/common/services/app-metrics-aggregator.test.ts
@@ -1,3 +1,5 @@
+import { DateTime } from 'luxon'
+
import { APP_METRICS_OUTPUT, AppMetricsOutput } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { parseJSON } from '~/common/utils/json-parse'
@@ -129,4 +131,26 @@ describe('AppMetricsAggregator', () => {
await agg.flush()
expect(queueMessagesMock).not.toHaveBeenCalled()
})
+
+ it('writes an explicit timestamp truncated to the hour', async () => {
+ const agg = new AppMetricsAggregator(outputs)
+ agg.queue(input({ timestamp: DateTime.fromISO('2027-03-04T05:06:07.890Z') }))
+ await agg.flush()
+
+ expect(getRows()[0].timestamp).toBe('2027-03-04 05:00:00.000')
+ })
+
+ it('sums entries in the same hour and keeps different hours apart', async () => {
+ const agg = new AppMetricsAggregator(outputs)
+ agg.queue(input({ count: 1, timestamp: DateTime.fromISO('2027-03-04T05:10:00Z') }))
+ agg.queue(input({ count: 2, timestamp: DateTime.fromISO('2027-03-04T05:50:00Z') }))
+ agg.queue(input({ count: 4, timestamp: DateTime.fromISO('2027-04-03T05:10:00Z') }))
+ agg.queue(input({ count: 8 }))
+ await agg.flush()
+
+ const countsByTimestamp = Object.fromEntries(getRows().map((r) => [r.timestamp, r.count]))
+ expect(Object.keys(countsByTimestamp)).toHaveLength(3)
+ expect(countsByTimestamp['2027-03-04 05:00:00.000']).toBe(3)
+ expect(countsByTimestamp['2027-04-03 05:00:00.000']).toBe(4)
+ })
})
diff --git a/nodejs/src/common/services/app-metrics-aggregator.ts b/nodejs/src/common/services/app-metrics-aggregator.ts
index 5f9e4db9e9e7..fcd77f7fdfb8 100644
--- a/nodejs/src/common/services/app-metrics-aggregator.ts
+++ b/nodejs/src/common/services/app-metrics-aggregator.ts
@@ -1,11 +1,12 @@
+import { DateTime } from 'luxon'
import { Counter } from 'prom-client'
import { APP_METRICS_OUTPUT, AppMetricsOutput } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { safeClickhouseString } from '~/common/utils/db/utils'
-import { castTimestampOrNow } from '~/common/utils/utils'
+import { castTimestampOrNow, castTimestampToClickhouseFormat } from '~/common/utils/utils'
-import { TimestampFormat } from '../../types'
+import { ClickHouseTimestamp, TimestampFormat } from '../../types'
const appMetricsAggregatorQueuedCounter = new Counter({
name: 'app_metrics_aggregator_queued_total',
@@ -21,8 +22,9 @@ const appMetricsAggregatorFlushedCounter = new Counter({
/**
* One v2 app metric row, matching the ClickHouse `app_metrics2` schema.
- * Aggregation key is the six identity fields — entries sharing them have
- * their `count` summed in-memory and emitted as one Kafka message on `flush`.
+ * Aggregation key is the six identity fields plus the timestamp hour — entries
+ * sharing them have their `count` summed in-memory and emitted as one Kafka
+ * message on `flush`.
*/
export interface AppMetricInput {
team_id: number
@@ -32,6 +34,17 @@ export interface AppMetricInput {
metric_kind: string
metric_name: string
count: number
+ /**
+ * Defaults to the flush time. Set it to date the row at another hour, e.g. a future
+ * month of logs retention that should bill when that month starts. Truncated to the hour,
+ * the same grain `app_metrics2` aggregates at.
+ */
+ timestamp?: DateTime
+}
+
+type BufferedAppMetric = Omit<AppMetricInput, 'instance_id' | 'timestamp'> & {
+ instance_id: string
+ timestamp: ClickHouseTimestamp | null
}
/**
@@ -44,18 +57,26 @@ export interface AppMetricInput {
* whatever batch / cycle they want metrics emitted for.
*/
export class AppMetricsAggregator {
- private buffer = new Map<string, AppMetricInput & { instance_id: string }>()
+ private buffer = new Map<string, BufferedAppMetric>()
constructor(private readonly outputs: IngestionOutputs<AppMetricsOutput>) {}
queue(metric: AppMetricInput): void {
appMetricsAggregatorQueuedCounter.inc({ app_source: metric.app_source })
- const key = makeKey(metric)
+ const { timestamp, ...rest } = metric
+ const row: BufferedAppMetric = {
+ ...rest,
+ instance_id: metric.instance_id ?? '',
+ timestamp: timestamp
+ ? castTimestampToClickhouseFormat(timestamp.toUTC().startOf('hour'), TimestampFormat.ClickHouse)
+ : null,
+ }
+ const key = makeKey(row)
const existing = this.buffer.get(key)
if (existing) {
existing.count += metric.count
} else {
- this.buffer.set(key, { ...metric, instance_id: metric.instance_id ?? '' })
+ this.buffer.set(key, row)
}
}
@@ -66,7 +87,7 @@ export class AppMetricsAggregator {
const drained = [...this.buffer.values()]
this.buffer.clear()
- const timestamp = castTimestampOrNow(null, TimestampFormat.ClickHouse)
+ const flushTimestamp = castTimestampOrNow(null, TimestampFormat.ClickHouse)
// No partition key — rows are re-aggregated by ClickHouse, ordering is
// irrelevant, and round-robin distributes load evenly across partitions.
const messages = drained.map((m) => ({
@@ -77,7 +98,7 @@ export class AppMetricsAggregator {
safeClickhouseString(
JSON.stringify({
team_id: m.team_id,
- timestamp,
+ timestamp: m.timestamp ?? flushTimestamp,
app_source: m.app_source,
app_source_id: m.app_source_id,
instance_id: m.instance_id,
@@ -99,6 +120,6 @@ export class AppMetricsAggregator {
}
}
-function makeKey(m: AppMetricInput): string {
- return `${m.team_id}:${m.app_source}:${m.app_source_id}:${m.instance_id ?? ''}:${m.metric_kind}:${m.metric_name}`
+function makeKey(m: BufferedAppMetric): string {
+ return `${m.team_id}:${m.app_source}:${m.app_source_id}:${m.instance_id}:${m.metric_kind}:${m.metric_name}:${m.timestamp ?? ''}`
}
diff --git a/nodejs/src/logs/logs-ingestion-consumer.test.ts b/nodejs/src/logs/logs-ingestion-consumer.test.ts
index 2065524d9420..f15ca58ebe1f 100644
--- a/nodejs/src/logs/logs-ingestion-consumer.test.ts
+++ b/nodejs/src/logs/logs-ingestion-consumer.test.ts
@@ -53,6 +53,7 @@ import type { MetricRulesCache } from './metrics-rules/metric-rules-cache'
import type { LogsMetricsEmitter } from './metrics-rules/metrics-emitter'
import { LOGS_DLQ_OUTPUT, LOGS_OUTPUT, LogsDlqOutput, LogsOutput } from './outputs/outputs'
import type { PatternMessageKeysCache } from './pattern-message-keys-cache'
+import { MAX_RETENTION_DAYS } from './retention/evaluate-retention'
import { DEFAULT_TRACES_RETENTION_DAYS } from './retention/tracing-config-cache'
import { compileRuleSet } from './sampling/compile-rules'
import type { SamplingRulesCache } from './sampling/sampling-rules-cache'
@@ -1576,13 +1577,17 @@ describe('LogsIngestionConsumer', () => {
})
it.each([
- { retentionDays: 14, tierMetric: 'bytes_ingested_retention_14d', byteDays: undefined },
- { retentionDays: 30, tierMetric: 'bytes_ingested_retention_30d', byteDays: 500 * 30 },
- { retentionDays: 90, tierMetric: 'bytes_ingested_retention_90d', byteDays: 500 * 90 },
- { retentionDays: 45, tierMetric: null, byteDays: 500 * 45 },
+ { retentionDays: 14, tierMetric: 'bytes_ingested_retention_14d', byteDayCounts: [] },
+ { retentionDays: 30, tierMetric: 'bytes_ingested_retention_30d', byteDayCounts: [500 * 30] },
+ {
+ retentionDays: 90,
+ tierMetric: 'bytes_ingested_retention_90d',
+ byteDayCounts: [500 * 30, 500 * 30, 500 * 30],
+ },
+ { retentionDays: 45, tierMetric: null, byteDayCounts: [500 * 30, 500 * 15] },
])(
- 'should emit retention_byte_days as $byteDays for $retentionDays days alongside the per-tier metric',
- async ({ retentionDays, tierMetric, byteDays }) => {
+ 'should emit one retention_byte_days row per month for $retentionDays days alongside the per-tier metric',
+ async ({ retentionDays, tierMetric, byteDayCounts }) => {
const usageStats = new Map([
[
team.id,
@@ -1605,7 +1610,10 @@ describe('LogsIngestionConsumer', () => {
const messages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)
const parsed = messages.map((m) => parseMetricValue(m.value))
- expect(parsed.find((m) => m?.metric_name === 'retention_byte_days')?.count).toBe(byteDays)
+ const byteDayRows = parsed
+ .filter((m) => m?.metric_name === 'retention_byte_days')
+ .sort((a, b) => a.timestamp.localeCompare(b.timestamp))
+ expect(byteDayRows.map((m) => m.count)).toEqual(byteDayCounts)
const tierMetrics = parsed.filter((m) => m?.metric_name?.startsWith('bytes_ingested_retention_'))
if (tierMetric) {
@@ -1618,6 +1626,73 @@ describe('LogsIngestionConsumer', () => {
}
)
+ it('should date each retention_byte_days row at the start of its month', async () => {
+ const usageStats = new Map([
+ [
+ team.id,
+ {
+ bytesReceived: 500,
+ recordsReceived: 5,
+ bytesAllowed: 500,
+ recordsAllowed: 5,
+ bytesAllowedRecords: 0,
+ bytesDropped: 0,
+ recordsDropped: 0,
+ piiReplacements: 0,
+ retentionDays: 90,
+ },
+ ],
+ ])
+ // Mid-hour, so the rows show the truncation to the hour.
+ jest.spyOn(Date, 'now').mockReturnValue(DateTime.fromISO('2025-01-01T05:37:12Z').toMillis())
+
+ await consumer['emitUsageMetrics'](usageStats)
+
+ const timestamps = getProducedKafkaMessages()
+ .filter((m) => m.topic === KAFKA_APP_METRICS_2)
+ .map((m) => parseMetricValue(m.value))
+ .filter((m) => m?.metric_name === 'retention_byte_days')
+ .map((m) => m.timestamp)
+ .sort()
+
+ expect(timestamps).toEqual([
+ '2025-01-01 05:00:00.000',
+ '2025-01-31 05:00:00.000',
+ '2025-03-02 05:00:00.000',
+ ])
+ })
+
+ it('should keep the longest retention within ClickHouse partitions-per-insert limit', async () => {
+ // app_metrics2 is partitioned by month, and ClickHouse fails an insert block that touches
+ // more than 100 partitions by default. One flush at the maximum retention must stay well
+ // under that, with room for the current month and a block that crosses a month boundary.
+ const usageStats = new Map([
+ [
+ team.id,
+ {
+ bytesReceived: 500,
+ recordsReceived: 5,
+ bytesAllowed: 500,
+ recordsAllowed: 5,
+ bytesAllowedRecords: 0,
+ bytesDropped: 0,
+ recordsDropped: 0,
+ piiReplacements: 0,
+ retentionDays: MAX_RETENTION_DAYS,
+ },
+ ],
+ ])
+
+ await consumer['emitUsageMetrics'](usageStats)
+
+ const months = new Set(
+ getProducedKafkaMessages()
+ .filter((m) => m.topic === KAFKA_APP_METRICS_2)
+ .map((m) => parseMetricValue(m.value)?.timestamp.slice(0, 7))
+ )
+ expect(months.size).toBeLessThanOrEqual(95)
+ })
+
it('should skip zero-count metrics', async () => {
const usageStats = new Map([
[
@@ -1695,13 +1770,14 @@ describe('LogsIngestionConsumer', () => {
const messages = getProducedKafkaMessages().filter((m) => m.topic === KAFKA_APP_METRICS_2)
- // 4 base metrics + per-tier retention + retention_byte_days per team (no dropped) = 12 total.
- expect(messages).toHaveLength(12)
+ // 4 base metrics + per-tier retention + one retention_byte_days row per 30-day month
+ // (no dropped): 6 for the 30-day team and 8 for the 90-day team.
+ expect(messages).toHaveLength(14)
const team1Messages = messages.filter((m) => parseMetricValue(m.value)?.team_id === team.id)
const team2Messages = messages.filter((m) => parseMetricValue(m.value)?.team_id === team2.id)
expect(team1Messages).toHaveLength(6)
- expect(team2Messages).toHaveLength(6)
+ expect(team2Messages).toHaveLength(8)
})
})
diff --git a/nodejs/src/logs/logs-ingestion-consumer.ts b/nodejs/src/logs/logs-ingestion-consumer.ts
index 2a93d131a17f..b19c84dcde07 100644
--- a/nodejs/src/logs/logs-ingestion-consumer.ts
+++ b/nodejs/src/logs/logs-ingestion-consumer.ts
@@ -1,4 +1,5 @@
import { trace } from '@opentelemetry/api'
+import { DateTime } from 'luxon'
import { Message } from 'node-rdkafka'
import pLimit from 'p-limit'
import { Counter, Histogram } from 'prom-client'
@@ -48,7 +49,7 @@ import { LOGS_DLQ_OUTPUT, LOGS_OUTPUT, LogsDlqOutput, LogsOutput } from './outpu
import { PatternMessageKeysCache } from './pattern-message-keys-cache'
import { EMPTY_STAGE_DROP_STATS, type PipelineStage } from './pipeline/log-processing-pipeline'
import type { RetentionRuleSource } from './retention/compile-retention-rules'
-import type { CompiledRetentionRuleSet } from './retention/evaluate-retention'
+import { type CompiledRetentionRuleSet, RETENTION_MONTH_DAYS } from './retention/evaluate-retention'
import {
RETENTION_EXPIRED_STAGE_NAME,
canHoldExpiredRow,
@@ -1292,13 +1293,13 @@ export class LogsIngestionConsumer {
if (retentionMetric) {
this.queueUsageMetric(teamId, retentionMetric, stats.bytesAllowed)
}
- // Byte-days: ingested bytes weighted by the full retention day count, only for teams that
- // chose a retention longer than the default. The default tier is covered by `bytes_ingested`,
+ // Byte-days: ingested bytes weighted by the retention day count, only for teams that
+ // chose a retention other than the default. The default tier is covered by `bytes_ingested`,
// so it must not be billed a second time here. Uses credit-adjusted `bytesAllowed` to
// reconcile with `bytes_ingested`. Runs beside the per-tier metric above until billing
// leaves fixed tiers.
if (stats.retentionDays !== DEFAULT_LOGS_RETENTION_DAYS) {
- this.queueUsageMetric(teamId, 'retention_byte_days', stats.bytesAllowed * stats.retentionDays)
+ this.queueRetentionByteDays(teamId, stats.bytesAllowed, stats.retentionDays)
}
const source = this.appSource === 'traces' ? 'apm_traces' : 'logs'
// These records are per-flush aggregates, not one per billed thing, so there is no
@@ -1327,7 +1328,7 @@ export class LogsIngestionConsumer {
])
}
- private queueUsageMetric(teamId: number, metricName: string, count: number): void {
+ private queueUsageMetric(teamId: number, metricName: string, count: number, timestamp?: DateTime): void {
if (count === 0) {
return
}
@@ -1339,9 +1340,24 @@ export class LogsIngestionConsumer {
metric_kind: 'usage',
metric_name: metricName,
count,
+ timestamp,
})
}
+ /**
+ * Writes one `retention_byte_days` row per 30-day month of retention, each dated at the start of
+ * its month, so each month of storage bills when it starts instead of the whole period billing
+ * on the ingest day. The usage report sums these rows by timestamp, and the `app_metrics2` TTL
+ * counts from each row's own timestamp, so future rows stay until they are billed.
+ */
+ private queueRetentionByteDays(teamId: number, bytes: number, retentionDays: number): void {
+ const now = DateTime.utc()
+ for (let offsetDays = 0; offsetDays < retentionDays; offsetDays += RETENTION_MONTH_DAYS) {
+ const days = Math.min(RETENTION_MONTH_DAYS, retentionDays - offsetDays)
+ this.queueUsageMetric(teamId, 'retention_byte_days', bytes * days, now.plus({ days: offsetDays }))
+ }
+ }
+
/**
* Per-rule head sampling drops in app_metrics2 (`sampling_records_dropped_by_rule`).
* Team-level sampling volume in CH is `sum(count)` over those rows (no separate aggregate metric).
diff --git a/posthog/data_freshness.py b/posthog/data_freshness.py
index 5af9428368d4..c48992bf386e 100644
--- a/posthog/data_freshness.py
+++ b/posthog/data_freshness.py
@@ -293,6 +293,8 @@ def _probe_app_metrics(specs: tuple[DataSourceSpec, ...], teams: list[Team], win
"""Every `app_metrics2`-backed product, in one query.
That table is already a per-team, per-product activity ledger sorted on `team_id` first.
+ `retention_byte_days` rows are excluded: the logs consumer dates them at future billing
+ months, so they would make a project look live long after its ingestion stopped.
"""
products_by_app_source = {spec.app_metrics_source: spec.product for spec in specs if spec.app_metrics_source}
if not products_by_app_source:
@@ -309,6 +311,7 @@ def _probe_app_metrics(specs: tuple[DataSourceSpec, ...], teams: list[Team], win
FROM app_metrics2
WHERE team_id IN %(team_ids)s
AND app_source IN %(app_sources)s
+ AND metric_name != 'retention_byte_days'
AND timestamp >= %(cutoff)s
AND timestamp <= %(horizon)s
GROUP BY team_id, app_source
diff --git a/posthog/tasks/test/test_usage_report.py b/posthog/tasks/test/test_usage_report.py
index df93f659b59b..1bb4f4be60b8 100644
--- a/posthog/tasks/test/test_usage_report.py
+++ b/posthog/tasks/test/test_usage_report.py
@@ -74,6 +74,7 @@ from posthog.tasks.usage_report import (
get_teams_with_billable_event_count_in_period,
get_teams_with_event_count_with_groups_in_period,
get_teams_with_logs_distinct_patterns_in_period,
+ get_teams_with_logs_retention_byte_days_in_period,
get_teams_with_posthog_code_credits_used_in_period,
get_teams_with_query_metric,
get_teams_with_sdk_logs_records_in_period,
@@ -2395,6 +2396,7 @@ class TestHasNonZeroUsage(SimpleTestCase):
("empty", None),
("events", "event_count_in_period"),
("logs_bytes", "logs_bytes_in_period"),
+ ("logs_retention_mb_days", "logs_retention_mb_days_in_period"),
("signals_credits", "signals_credits_used_in_period"),
("posthog_code_credits", "posthog_code_credits_used_in_period"),
]
@@ -3790,6 +3792,31 @@ class TestHogFunctionUsageReports(ClickhouseDestroyTablesMixin, TestCase, Clickh
assert dict(get_teams_with_logs_distinct_patterns_in_period(begin, end)) == {team_id: 2, other_team_id: 1}
+ @parameterized.expand([("ingest_day", 0, 1_000), ("second_month", 30, 2_000)])
+ def test_logs_retention_byte_days_bill_each_month_on_its_own_day(
+ self, _name: str, period_offset_days: int, expected_count: int
+ ) -> None:
+ team_id = 1
+ ingest_day = now().replace(hour=0, minute=0, second=0, microsecond=0)
+ # The consumer dates each later month of retention in the future, so only the row dated
+ # in the report period may bill there.
+ for offset_days, count in ((0, 1_000), (30, 2_000)):
+ create_app_metric2(
+ team_id=team_id,
+ app_source="logs",
+ timestamp=ingest_day + timedelta(days=offset_days),
+ app_source_id="",
+ instance_id="",
+ metric_kind="usage",
+ metric_name="retention_byte_days",
+ count=count,
+ )
+
+ begin = ingest_day + timedelta(days=period_offset_days)
+ assert get_teams_with_logs_retention_byte_days_in_period(begin, begin + timedelta(days=1)) == [
+ (team_id, expected_count)
+ ]
+
@parameterized.expand(
[
# Per-signal MB is floored to whole decimal MB, and the billable combined metric is
diff --git a/posthog/tasks/usage_report.py b/posthog/tasks/usage_report.py
index 5cf7aa0ca5fb..a2b6ea58b774 100644
--- a/posthog/tasks/usage_report.py
+++ b/posthog/tasks/usage_report.py
@@ -385,8 +385,8 @@ class UsageReportCounters:
logs_retention_30d_mb_in_period: int
logs_retention_90d_mb_in_period: int
# Byte-days of retention floored to whole MB-days (retention_byte_days // 1_000_000): ingested bytes
- # weighted by the full retention day count, so it scales to any retention day count. Zero for teams
- # on the default retention; they are covered by logs_mb_in_period alone. Report-only.
+ # weighted by the days of storage that start in the period, so it scales to any retention day count.
+ # Zero for teams on the default retention; they are covered by logs_mb_in_period alone. Report-only.
logs_retention_mb_days_in_period: int
# Per-SDK split of logs_records_in_period, which on its own has no SDK dimension. Keyed off the
# telemetry.sdk.name resource attribute each SDK sets on every record. See SDK_TELEMETRY_NAMES.
@@ -2600,12 +2600,13 @@ def get_teams_with_logs_retention_byte_days_in_period(
"""
Returns byte-days of log retention grouped by team: ingested bytes weighted by retention days.
- The consumer emits one `retention_byte_days` metric into `app_metrics2`
- (`retention_byte_days = bytes_ingested * retention_days`, summed per flush) only for teams on a
- non-default retention; default-retention teams emit nothing, so their storage is billed through
- `bytes_ingested` alone. Summed over the period it is the storage-duration of the logs kept beyond
- the default, not of all logs, and it scales to any retention day count. Each `(team_id, count)`
- tuple is ready for `convert_team_usage_rows_to_dict`.
+ The consumer emits `retention_byte_days` into `app_metrics2` only for teams on a non-default
+ retention; default-retention teams emit nothing, so their storage is billed through
+ `bytes_ingested` alone. Each flush writes one row per 30-day month of retention
+ (`bytes_ingested * 30`), dated at the start of that month, so future rows land in the period
+ their month starts in and each month of storage bills when it starts. Summed over the period it
+ is the storage-duration of the logs kept beyond the default, not of all logs. Each
+ `(team_id, count)` tuple is ready for `convert_team_usage_rows_to_dict`.
"""
with tags_context(product=Product.LOGS, feature=Feature.USAGE_REPORT):
return sync_execute(
@@ -2956,6 +2957,7 @@ def has_non_zero_usage(report: UsageReportCounters) -> bool:
or report.posthog_code_credits_used_in_period > 0
or report.task_sandbox_seconds_in_period > 0
or report.logs_bytes_in_period > 0
+ or report.logs_retention_mb_days_in_period > 0
or report.apm_tracing_bytes_in_period > 0
or report.metrics_records_in_period > 0
or report.workflow_emails_sent_in_period > 0
diff --git a/posthog/test/test_data_freshness.py b/posthog/test/test_data_freshness.py
index 03e0d0ac99f8..c21943a7ef9f 100644
--- a/posthog/test/test_data_freshness.py
+++ b/posthog/test/test_data_freshness.py
@@ -1,7 +1,7 @@
from datetime import UTC, datetime, timedelta
from typing import Optional
-from posthog.test.base import BaseTest
+from posthog.test.base import BaseTest, ClickhouseDestroyTablesMixin, ClickhouseTestMixin
from unittest.mock import patch
from django.test import SimpleTestCase
@@ -20,10 +20,12 @@ from posthog.data_freshness import (
)
from posthog.models.team.team import Team
from posthog.schema_enums import ProductKey
+from posthog.test.fixtures import create_app_metric2
from products.event_definitions.backend.models.event_definition import EventDefinition
from products.feature_flags.backend.data_freshness import DATA_SOURCES as FEATURE_FLAGS_DATA_SOURCES
from products.feature_flags.backend.models.feature_flag import FeatureFlag
+from products.logs.backend.data_freshness import DATA_SOURCES as LOGS_DATA_SOURCES
NOW = datetime(2026, 8, 3, 12, 0, tzinfo=UTC)
QUIET_BEFORE = NOW - timedelta(days=7)
@@ -193,3 +195,32 @@ class TestFeatureFlagsFreshness(BaseTest):
)
self.assertEqual(results[self.team.id].sources, expected_sources)
self.assertEqual(results[self.other_team.id].last_data_at, self.other_team_called_at)
+
+
+class TestLogsFreshness(ClickhouseTestMixin, ClickhouseDestroyTablesMixin, BaseTest):
+ def test_scheduled_retention_rows_do_not_make_a_project_live(self) -> None:
+ now = datetime.now(UTC).replace(minute=0, second=0, microsecond=0)
+ ingested_at = now - timedelta(days=2)
+ create_app_metric2(
+ team_id=self.team.pk,
+ app_source="logs",
+ timestamp=ingested_at,
+ metric_kind="usage",
+ metric_name="bytes_ingested",
+ )
+ # The consumer dates later retention months ahead, so they arrive in the window after ingestion stops.
+ create_app_metric2(
+ team_id=self.team.pk,
+ app_source="logs",
+ timestamp=now - timedelta(hours=1),
+ metric_kind="usage",
+ metric_name="retention_byte_days",
+ )
+
+ with patch("posthog.data_freshness.discover_data_sources", return_value=tuple(LOGS_DATA_SOURCES)):
+ results = {result.team_id: result for result in _compute([self.team])}
+
+ self.assertEqual(
+ results[self.team.id].sources,
+ [SourceFreshness(data_source=ProductKey.LOGS, last_data_at=ingested_at)],
+ )