Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(ingestion): set ingested_event when a team sends logs, traces or metrics (#120688)

Rafael Audibert9b94fb0d1446
diff --git a/nodejs/src/common/utils/team-manager.test.ts b/nodejs/src/common/utils/team-manager.test.ts
index 998b9f18b1b2..b71599f89934 100644
--- a/nodejs/src/common/utils/team-manager.test.ts
+++ b/nodejs/src/common/utils/team-manager.test.ts
@@ -332,6 +332,13 @@ describe('TeamManager()', () => {
)
})
+ it('flips the flag and captures once for a batch that repeats a team', async () => {
+ await teamManager.setTeamsIngestedData([newTeam.id, newTeam.id, newTeam.id])
+
+ expect(await readIngestedEvent(newTeam.id)).toBe(true)
+ expect(mockCaptureTeamEvent).toHaveBeenCalledTimes(1)
+ })
+
it('does nothing on a repeat call through a stale team object', async () => {
await teamManager.setTeamIngestedEvent(newTeam, { $lib: 'web' })
mockCaptureTeamEvent.mockClear()
diff --git a/nodejs/src/common/utils/team-manager.ts b/nodejs/src/common/utils/team-manager.ts
index 69c4581b9249..ecafb0cfa953 100644
--- a/nodejs/src/common/utils/team-manager.ts
+++ b/nodejs/src/common/utils/team-manager.ts
@@ -59,6 +59,22 @@ export class TeamManager {
return team?.available_features.includes(feature) || false
}
+ // `ingested_event` means "this project received data", so logs, traces and metrics set it too.
+ public async setTeamsIngestedData(teamIds: Iterable<number>): Promise<void> {
+ await Promise.all(
+ [...new Set(teamIds)].map(async (teamId) => {
+ try {
+ const team = await this.getTeam(teamId)
+ if (team) {
+ await this.setTeamIngestedEvent(team, {})
+ }
+ } catch (error) {
+ logger.error('⚠️', 'Failed to set ingested_event', { teamId, error })
+ }
+ })
+ )
+ }
+
public async setTeamIngestedEvent(team: Team, properties: Properties): Promise<void> {
if (!team.ingested_event) {
// The cached team can be stale long after the first write, so this fires repeatedly for
diff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-consumer.test.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-consumer.test.ts
index 04cdb91be7a2..6e0f216b8d8f 100644
--- a/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-consumer.test.ts
+++ b/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-consumer.test.ts
@@ -11,7 +11,7 @@ import { APP_METRICS_OUTPUT, AppMetricsOutput } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { SingleIngestionOutput } from '~/common/outputs/single-ingestion-output'
import { closeHub, createHub } from '~/common/utils/db/hub'
-import { createTestTeamFixture } from '~/tests/helpers/sql'
+import { createTeam, createTestTeamFixture, getTeam } from '~/tests/helpers/sql'
import { Hub, Team } from '~/types'
import { getDefaultMetricsIngestionConsumerConfig } from './config'
@@ -88,6 +88,21 @@ describe('MetricsIngestionConsumer', () => {
await closeHub(hub)
})
+ it('marks a team that receives its first metrics as having ingested data', async () => {
+ const newTeamId = await createTeam(hub.postgres, team.organization_id, undefined, { ingested_event: false })
+ const newTeam = await getTeam(hub.postgres, newTeamId)
+ const message = createKafkaMessage({
+ token: newTeam!.api_token,
+ bytes_uncompressed: '100',
+ record_count: '1',
+ })
+
+ const { backgroundTask } = await consumer.processKafkaBatch([message])
+ await backgroundTask
+
+ expect((await getTeam(hub.postgres, newTeamId))?.ingested_event).toBe(true)
+ })
+
it('stamps the default retention-days header on messages produced to ClickHouse', async () => {
const message = createKafkaMessage({
token: team.api_token,
diff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-consumer.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-consumer.ts
index c3820fccf464..90b3b2d24da5 100644
--- a/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-consumer.ts
+++ b/nodejs/src/ingestion/pipelines/metrics/metrics-ingestion-consumer.ts
@@ -103,7 +103,11 @@ export class MetricsIngestionConsumer {
return {
backgroundTask: Promise.all([
- this.produceValidMetricMessages(rateLimiterAllowedMessages),
+ this.produceValidMetricMessages(rateLimiterAllowedMessages).then(() =>
+ this.deps.teamManager.setTeamsIngestedData(
+ rateLimiterAllowedMessages.map((message) => message.teamId)
+ )
+ ),
this.emitUsageMetrics(usageStats),
]),
messages: rateLimiterAllowedMessages,
diff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.test.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.test.ts
index e9e813c81204..369d2fa50ba4 100644
--- a/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.test.ts
+++ b/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.test.ts
@@ -28,7 +28,7 @@ jest.mock('./services/metrics-rate-limiter.service', () => ({
describe('MetricsPipelineConsumer', () => {
const team = createTestTeam({ id: 1, api_token: 'token-a' })
let outputs: ReturnType<typeof createMockIngestionOutputs>
- let teamManager: { getTeam: jest.Mock; getTeamByToken: jest.Mock }
+ let teamManager: { getTeam: jest.Mock; getTeamByToken: jest.Mock; setTeamsIngestedData: jest.Mock }
let consumer: MetricsPipelineConsumer
const message: Message = {
@@ -43,7 +43,11 @@ describe('MetricsPipelineConsumer', () => {
beforeEach(() => {
outputs = createMockIngestionOutputs()
- teamManager = { getTeam: jest.fn(), getTeamByToken: jest.fn().mockResolvedValue(team) }
+ teamManager = {
+ getTeam: jest.fn(),
+ getTeamByToken: jest.fn().mockResolvedValue(team),
+ setTeamsIngestedData: jest.fn().mockResolvedValue(undefined),
+ }
consumer = new MetricsPipelineConsumer(getDefaultMetricsIngestionConsumerConfig(), {
teamManager,
quotaLimiting: { isTeamTokenQuotaLimited: jest.fn().mockResolvedValue(false) },
@@ -63,11 +67,13 @@ describe('MetricsPipelineConsumer', () => {
expect(settled).toBe(false)
expect(outputs.queueMessages).not.toHaveBeenCalled()
+ expect(teamManager.setTeamsIngestedData).not.toHaveBeenCalled()
ack()
await backgroundTask
expect(outputs.produce).toHaveBeenCalledTimes(1)
expect(outputs.queueMessages).toHaveBeenCalled()
+ expect(teamManager.setTeamsIngestedData).toHaveBeenCalledWith([team.id])
})
it('runs on consumer-v2, which stores no offsets when the background task rejects', () => {
diff --git a/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.ts b/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.ts
index 03f8611d9ac9..3a79028cc233 100644
--- a/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.ts
+++ b/nodejs/src/ingestion/pipelines/metrics/metrics-pipeline-consumer.ts
@@ -59,6 +59,7 @@ export class MetricsPipelineConsumer {
private promiseScheduler: FailureLatchingPromiseScheduler
private pipeline: MetricsIngestionPipeline
private outputs: MetricsIngestionOutputs
+ private teamManager: TeamManager
constructor(config: MetricsIngestionConsumerConfig, deps: MetricsPipelineConsumerDeps) {
// consumer-v1 stores a batch's offsets even when its background task rejects,
@@ -68,6 +69,7 @@ export class MetricsPipelineConsumer {
topic: config.METRICS_INGESTION_CONSUMER_CONSUME_TOPIC,
})
this.outputs = deps.outputs
+ this.teamManager = deps.teamManager
this.promiseScheduler = new FailureLatchingPromiseScheduler()
this.pipeline = createMetricsIngestionPipeline({
outputs: deps.outputs,
@@ -116,6 +118,9 @@ export class MetricsPipelineConsumer {
return {
backgroundTask: instrumentFn('metricsIngestionConsumer.awaitScheduledWork', async () => {
await this.promiseScheduler.waitForAllOrFail()
+ await this.teamManager.setTeamsIngestedData(
+ [...usage.entries()].filter(([, stats]) => stats.recordsAllowed > 0).map(([teamId]) => teamId)
+ )
await emitMetricsUsage(this.outputs, usage)
}),
}
diff --git a/nodejs/src/logs/logs-ingestion-consumer.test.ts b/nodejs/src/logs/logs-ingestion-consumer.test.ts
index f15ca58ebe1f..632feccbe2f0 100644
--- a/nodejs/src/logs/logs-ingestion-consumer.test.ts
+++ b/nodejs/src/logs/logs-ingestion-consumer.test.ts
@@ -514,6 +514,16 @@ describe('LogsIngestionConsumer', () => {
})
})
+ it('marks a team that receives its first logs as having ingested data', async () => {
+ const newTeamId = await createTeam(hub.postgres, team.organization_id, undefined, { ingested_event: false })
+ const newTeam = await getTeam(hub.postgres, newTeamId)
+ const messages = await createKafkaMessages([createLogMessage()], { token: newTeam!.api_token })
+
+ await waitForBackgroundTasks(consumer.processKafkaBatch(messages))
+
+ expect((await getTeam(hub.postgres, newTeamId))?.ingested_event).toBe(true)
+ })
+
it('should produce messages with correct headers', async () => {
const logData = createLogMessage({ level: 'error', message: 'Critical error' })
const messages = await createKafkaMessages([logData], {
diff --git a/nodejs/src/logs/logs-ingestion-consumer.ts b/nodejs/src/logs/logs-ingestion-consumer.ts
index b19c84dcde07..7c7613f6c7f1 100644
--- a/nodejs/src/logs/logs-ingestion-consumer.ts
+++ b/nodejs/src/logs/logs-ingestion-consumer.ts
@@ -705,6 +705,9 @@ export class LogsIngestionConsumer {
usageStats,
transformationBatchBudget
)
+ await this.deps.teamManager.setTeamsIngestedData(
+ rateLimiterAllowedMessages.map((message) => message.teamId)
+ )
await this.emitUsageMetrics(usageStats)
// Best-effort flush of transformation app metrics + function logs; never block the data path
if (this.deps.logsTransformer) {