:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(ingestion): set ingested_event when a team sends logs, traces or metrics (#120688)
Rafael Audibert9b94fb0d1446
diff --git a/nodejs/src/common/utils/team-manager.test.ts b/nodejs/src/common/utils/team-manager.test.tsindex 998b9f18b1b2..b71599f89934 100644--- a/nodejs/src/common/utils/team-manager.test.ts+++ b/nodejs/src/common/utils/team-manager.test.ts@@ -332,6 +332,13 @@ describe('TeamManager()', () => {)})+ it('flips the flag and captures once for a batch that repeats a team', async () => {+ await teamManager.setTeamsIngestedData([newTeam.id, newTeam.id, newTeam.id])++ expect(await readIngestedEvent(newTeam.id)).toBe(true)+ expect(mockCaptureTeamEvent).toHaveBeenCalledTimes(1)+ })+it('does nothing on a repeat call through a stale team object', async () => {await teamManager.setTeamIngestedEvent(newTeam, { $lib: 'web' })mockCaptureTeamEvent.mockClear()diff --git a/nodejs/src/common/utils/team-manager.ts b/nodejs/src/common/utils/team-manager.tsindex 69c4581b9249..ecafb0cfa953 100644--- a/nodejs/src/common/utils/team-manager.ts+++ b/nodejs/src/common/utils/team-manager.ts@@ -59,6 +59,22 @@ export class TeamManager {return team?.available_features.includes(feature) || false}+ // `ingested_event` means "this project received data", so logs, traces and metrics set it too.+ public async setTeamsIngestedData(teamIds: Iterable<number>): Promise<void> {+ await Promise.all(+ [...new Set(teamIds)].map(async (teamId) => {+ try {+ const team = await this.getTeam(teamId)+ if (team) {+ await this.setTeamIngestedEvent(team, {})+ }+ } catch (error) {+ logger.error('⚠️', 'Failed to set ingested_event', { teamId, error })+ }+ })+ )+ }+public async setTeamIngestedEvent(team: Team, properties: Properties): Promise<void> {if (!team.ingested_event) {// The cached team can be stale long after the first write, so this fires repeatedly fordiff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-consumer.test.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-consumer.test.tsindex 04cdb91be7a2..6e0f216b8d8f 100644--- a/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-consumer.test.ts+++ b/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-consumer.test.ts@@ -11,7 +11,7 @@ import { APP_METRICS_OUTPUT, AppMetricsOutput } from '~/common/outputs'import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'import { SingleIngestionOutput } from '~/common/outputs/single-ingestion-output'import { closeHub, createHub } from '~/common/utils/db/hub'-import { createTestTeamFixture } from '~/tests/helpers/sql'+import { createTeam, createTestTeamFixture, getTeam } from '~/tests/helpers/sql'import { Hub, Team } from '~/types'import { getDefaultMetricsIngestionConsumerConfig } from './config'@@ -88,6 +88,21 @@ describe('MetricsIngestionConsumer', () => {await closeHub(hub)})+ it('marks a team that receives its first metrics as having ingested data', async () => {+ const newTeamId = await createTeam(hub.postgres, team.organization_id, undefined, { ingested_event: false })+ const newTeam = await getTeam(hub.postgres, newTeamId)+ const message = createKafkaMessage({+ token: newTeam!.api_token,+ bytes_uncompressed: '100',+ record_count: '1',+ })++ const { backgroundTask } = await consumer.processKafkaBatch([message])+ await backgroundTask++ expect((await getTeam(hub.postgres, newTeamId))?.ingested_event).toBe(true)+ })+it('stamps the default retention-days header on messages produced to ClickHouse', async () => {const message = createKafkaMessage({token: team.api_token,diff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-consumer.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-consumer.tsindex c3820fccf464..90b3b2d24da5 100644--- a/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-consumer.ts+++ b/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-consumer.ts@@ -103,7 +103,11 @@ export class MetricsIngestionConsumer {return {backgroundTask: Promise.all([- this.produceValidMetricMessages(rateLimiterAllowedMessages),+ this.produceValidMetricMessages(rateLimiterAllowedMessages).then(() =>+ this.deps.teamManager.setTeamsIngestedData(+ rateLimiterAllowedMessages.map((message) => message.teamId)+ )+ ),this.emitUsageMetrics(usageStats),]),messages: rateLimiterAllowedMessages,diff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.test.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.test.tsindex e9e813c81204..369d2fa50ba4 100644--- a/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.test.ts+++ b/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.test.ts@@ -28,7 +28,7 @@ jest.mock('./services/metrics-rate-limiter.service', () => ({describe('MetricsPipelineConsumer', () => {const team = createTestTeam({ id: 1, api_token: 'token-a' })let outputs: ReturnType<typeof createMockIngestionOutputs>- let teamManager: { getTeam: jest.Mock; getTeamByToken: jest.Mock }+ let teamManager: { getTeam: jest.Mock; getTeamByToken: jest.Mock; setTeamsIngestedData: jest.Mock }let consumer: MetricsPipelineConsumerconst message: Message = {@@ -43,7 +43,11 @@ describe('MetricsPipelineConsumer', () => {beforeEach(() => {outputs = createMockIngestionOutputs()- teamManager = { getTeam: jest.fn(), getTeamByToken: jest.fn().mockResolvedValue(team) }+ teamManager = {+ getTeam: jest.fn(),+ getTeamByToken: jest.fn().mockResolvedValue(team),+ setTeamsIngestedData: jest.fn().mockResolvedValue(undefined),+ }consumer = new MetricsPipelineConsumer(getDefaultMetricsIngestionConsumerConfig(), {teamManager,quotaLimiting: { isTeamTokenQuotaLimited: jest.fn().mockResolvedValue(false) },@@ -63,11 +67,13 @@ describe('MetricsPipelineConsumer', () => {expect(settled).toBe(false)expect(outputs.queueMessages).not.toHaveBeenCalled()+ expect(teamManager.setTeamsIngestedData).not.toHaveBeenCalled()ack()await backgroundTaskexpect(outputs.produce).toHaveBeenCalledTimes(1)expect(outputs.queueMessages).toHaveBeenCalled()+ expect(teamManager.setTeamsIngestedData).toHaveBeenCalledWith([team.id])})it('runs on consumer-v2, which stores no offsets when the background task rejects', () => {diff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.tsindex 03f8611d9ac9..3a79028cc233 100644--- a/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.ts+++ b/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.ts@@ -59,6 +59,7 @@ export class MetricsPipelineConsumer {private promiseScheduler: FailureLatchingPromiseSchedulerprivate pipeline: MetricsIngestionPipelineprivate outputs: MetricsIngestionOutputs+ private teamManager: TeamManagerconstructor(config: MetricsIngestionConsumerConfig, deps: MetricsPipelineConsumerDeps) {// consumer-v1 stores a batch's offsets even when its background task rejects,@@ -68,6 +69,7 @@ export class MetricsPipelineConsumer {topic: config.METRICS_INGESTION_CONSUMER_CONSUME_TOPIC,})this.outputs = deps.outputs+ this.teamManager = deps.teamManagerthis.promiseScheduler = new FailureLatchingPromiseScheduler()this.pipeline = createMetricsIngestionPipeline({outputs: deps.outputs,@@ -116,6 +118,9 @@ export class MetricsPipelineConsumer {return {backgroundTask: instrumentFn('metricsIngestionConsumer.awaitScheduledWork', async () => {await this.promiseScheduler.waitForAllOrFail()+ await this.teamManager.setTeamsIngestedData(+ [...usage.entries()].filter(([, stats]) => stats.recordsAllowed > 0).map(([teamId]) => teamId)+ )await emitMetricsUsage(this.outputs, usage)}),}diff --git a/nodejs/src/logs/logs-ingestion-consumer.test.ts b/nodejs/src/logs/logs-ingestion-consumer.test.tsindex f15ca58ebe1f..632feccbe2f0 100644--- a/nodejs/src/logs/logs-ingestion-consumer.test.ts+++ b/nodejs/src/logs/logs-ingestion-consumer.test.ts@@ -514,6 +514,16 @@ describe('LogsIngestionConsumer', () => {})})+ it('marks a team that receives its first logs as having ingested data', async () => {+ const newTeamId = await createTeam(hub.postgres, team.organization_id, undefined, { ingested_event: false })+ const newTeam = await getTeam(hub.postgres, newTeamId)+ const messages = await createKafkaMessages([createLogMessage()], { token: newTeam!.api_token })++ await waitForBackgroundTasks(consumer.processKafkaBatch(messages))++ expect((await getTeam(hub.postgres, newTeamId))?.ingested_event).toBe(true)+ })+it('should produce messages with correct headers', async () => {const logData = createLogMessage({ level: 'error', message: 'Critical error' })const messages = await createKafkaMessages([logData], {diff --git a/nodejs/src/logs/logs-ingestion-consumer.ts b/nodejs/src/logs/logs-ingestion-consumer.tsindex b19c84dcde07..7c7613f6c7f1 100644--- a/nodejs/src/logs/logs-ingestion-consumer.ts+++ b/nodejs/src/logs/logs-ingestion-consumer.ts@@ -705,6 +705,9 @@ export class LogsIngestionConsumer {usageStats,transformationBatchBudget)+ await this.deps.teamManager.setTeamsIngestedData(+ rateLimiterAllowedMessages.map((message) => message.teamId)+ )await this.emitUsageMetrics(usageStats)// Best-effort flush of transformation app metrics + function logs; never block the data pathif (this.deps.logsTransformer) {