Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

fix(logs): send logs ingestion transformation logs with a loop guard (#130285)

Jon McCallum1e69a5cb7dce
diff --git a/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.test.ts b/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.test.ts
index e7d0b0aa7def..1b6a198fcb06 100644
--- a/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.test.ts
+++ b/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.test.ts
@@ -1,4 +1,5 @@
import { DateTime } from 'luxon'
+import { register } from 'prom-client'
import { APP_METRICS_OUTPUT, LOG_ENTRIES_OUTPUT } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
@@ -177,6 +178,24 @@ describe('HogFunctionMonitoringService', () => {
expect(fetchMock).toHaveBeenCalledTimes(2)
})
+ it('keeps logs about internal logs out of the logs product, writes them to log_entries, and counts them', async () => {
+ const fetchMock = jest.mocked(internalFetch).mockResolvedValue({ status: 200, dump: jest.fn() } as any)
+ const loopGuarded = async (): Promise<number> =>
+ (await register.getSingleMetric('logs_internal_producer_records_total')!.get()).values.find(
+ (v) => v.labels.outcome === 'loop_guard'
+ )?.value ?? 0
+ const before = await loopGuarded()
+ service = new HogFunctionMonitoringService(outputs, internalLogsProducer(), new Set(['hog_flow']))
+
+ service.queueLogs([log(2), log(2), log(3)], 'hog_flow', { aboutInternalLogs: true })
+ await service.flush()
+ await service.stop()
+
+ expect(produceMock).toHaveBeenCalledTimes(3)
+ expect(fetchMock).not.toHaveBeenCalled()
+ expect((await loopGuarded()) - before).toBe(2)
+ })
+
it('sends nothing to the logs product for a source not in CDP_INTERNAL_LOGS_SOURCES', async () => {
const fetchMock = jest.mocked(internalFetch).mockResolvedValue({ status: 200, dump: jest.fn() } as any)
service = new HogFunctionMonitoringService(outputs, internalLogsProducer(), new Set(['hog_function']))
diff --git a/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.ts b/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.ts
index 4e96a01c9771..b08099a4c2a2 100644
--- a/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.ts
+++ b/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.ts
@@ -198,7 +198,8 @@ export class HogFunctionMonitoringService {
metrics.forEach((metric) => this.queueAppMetric(metric, source))
}
- queueLogs(logEntries: LogEntry[], source: MetricLogSource) {
+ /** Set `aboutInternalLogs` for logs about a batch that came from the internal producer, so they never go back to it and loop. */
+ queueLogs(logEntries: LogEntry[], source: MetricLogSource, { aboutInternalLogs = false } = {}) {
const sourcedLogEntries = logEntries.map((logEntry) => ({
...logEntry,
log_source: source,
@@ -211,7 +212,12 @@ export class HogFunctionMonitoringService {
})
hogFunctionMonitoringPendingMessages.set(this.queuedLogMessages.length)
- if (this.internalLogsSources.has(source)) {
+ if (!this.internalLogsSources.has(source)) {
+ return
+ }
+ if (aboutInternalLogs) {
+ this.internalLogsProducer?.skipLooping(source, sourcedLogEntries)
+ } else {
this.internalLogsProducer?.enqueue(source, sourcedLogEntries)
}
}
diff --git a/nodejs/src/logs/framework/logs-ingestion-framework-consumer.test.ts b/nodejs/src/logs/framework/logs-ingestion-framework-consumer.test.ts
index b30566f275f7..03201d037e8e 100644
--- a/nodejs/src/logs/framework/logs-ingestion-framework-consumer.test.ts
+++ b/nodejs/src/logs/framework/logs-ingestion-framework-consumer.test.ts
@@ -14,6 +14,7 @@ import { createTeam, createTestTeamFixture, getTeam } from '~/tests/helpers/sql'
import { Hub, Team } from '~/types'
import { getDefaultTracesIngestionConsumerConfig } from '../config'
+import { INTERNAL_LOGS_HEADER, signInternalLogs } from '../internal-logs'
import { LogRecord, encodeLogRecords } from '../log-record-avro'
import { DEFAULT_LOGS_RETENTION_DAYS, LogsIngestionConsumer, logMessageDlqCounter } from '../logs-ingestion-consumer'
import { LOGS_DLQ_OUTPUT, LOGS_OUTPUT, LogsDlqOutput, LogsOutput } from '../outputs/outputs'
@@ -185,6 +186,38 @@ describe('LogsIngestionFrameworkConsumer', () => {
await closeHub(hub)
})
+ it('runs transformations on a batch whose internal logs header fails to verify, flagged as internal', async () => {
+ await consumer.stop()
+ const logsTransformer = {
+ startBatch: jest.fn(),
+ teamHasTransformations: jest.fn().mockResolvedValue(true),
+ transformRecords: jest.fn().mockResolvedValue({ recordsDropped: 0, recordsDroppedByFunctionId: new Map() }),
+ flush: jest.fn().mockResolvedValue(undefined),
+ }
+ consumer = withoutKafka(
+ new LogsIngestionFrameworkConsumer(
+ {
+ ...hub,
+ LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '*',
+ LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: '*',
+ LOGS_INTERNAL_INGESTION_SIGNING_KEYS: 'current-key',
+ },
+ { ...consumerDeps(), logsTransformer }
+ )
+ )
+ await consumer.start()
+
+ await processAndWait(
+ await createKafkaMessages(1, {
+ token: team.api_token,
+ [INTERNAL_LOGS_HEADER]: signInternalLogs('rotated-away-key', 'hog_function', team.id),
+ })
+ )
+
+ expect(logsTransformer.transformRecords).toHaveBeenCalledWith(team.id, expect.any(Array), undefined, true)
+ expect(producedTo(KAFKA_LOGS_CLICKHOUSE)).toHaveLength(1)
+ })
+
it('produces a valid message to the logs topic with the batch headers', async () => {
const messages = await createKafkaMessages(1, { token: team.api_token })
diff --git a/nodejs/src/logs/framework/logs-pipeline-context.ts b/nodejs/src/logs/framework/logs-pipeline-context.ts
index 74d19801ffc4..a126422226d3 100644
--- a/nodejs/src/logs/framework/logs-pipeline-context.ts
+++ b/nodejs/src/logs/framework/logs-pipeline-context.ts
@@ -117,11 +117,11 @@ export type LogsPipelineContext = {
/** Transformations write hog_function logs, which could come back here as internal logs and loop. */
export function appliesTeamLogRules(message: LogsIngestionMessage): boolean {
- return message.internalLogs?.teamRules ?? true
+ return message.verifiedAsInternalLogs?.teamRules ?? true
}
export function isUnbilledInternal(message: LogsIngestionMessage): boolean {
- return message.internalLogs !== undefined && !message.internalLogs.billed
+ return message.verifiedAsInternalLogs !== undefined && !message.verifiedAsInternalLogs.billed
}
/** A failed batch replays in full and the logs table has no dedupe, so each retry here avoids duplicates. */
diff --git a/nodejs/src/logs/framework/parse-logs-message-step.ts b/nodejs/src/logs/framework/parse-logs-message-step.ts
index edaaf048ec74..c06d580527dd 100644
--- a/nodejs/src/logs/framework/parse-logs-message-step.ts
+++ b/nodejs/src/logs/framework/parse-logs-message-step.ts
@@ -100,7 +100,12 @@ export function createParseLogsMessageStep<T extends LogsPipelineInput>(
recordCount,
minTimestampMicros: parseMinTimestampHeader(headers.min_timestamp),
backfillRequested: headers.backfill_days !== undefined,
- internalLogs: internalLogsVerifier.verify(headers[INTERNAL_LOGS_HEADER], team.id, headers.created_at),
+ sentAsInternalLogs: headers[INTERNAL_LOGS_HEADER] !== undefined,
+ verifiedAsInternalLogs: internalLogsVerifier.verify(
+ headers[INTERNAL_LOGS_HEADER],
+ team.id,
+ headers.created_at
+ ),
})
} catch (e) {
// A failed quarantine of bad size headers lands here, and a second DLQ failure fails the batch.
diff --git a/nodejs/src/logs/framework/process-log-message-step.ts b/nodejs/src/logs/framework/process-log-message-step.ts
index 840d9b447ba6..58ab57bdec29 100644
--- a/nodejs/src/logs/framework/process-log-message-step.ts
+++ b/nodejs/src/logs/framework/process-log-message-step.ts
@@ -254,8 +254,8 @@ async function processLogMessage(context: LogsPipelineContext, input: LogsProces
key: null,
headers: {
...withoutInternalLogsHeaders(parseKafkaHeaders(input.message.headers)),
- ...(input.internalLogs !== undefined
- ? { [INTERNAL_SOURCE_HEADER]: input.internalLogs.source }
+ ...(input.verifiedAsInternalLogs !== undefined
+ ? { [INTERNAL_SOURCE_HEADER]: input.verifiedAsInternalLogs.source }
: {}),
token: input.token,
team_id: input.teamId.toString(),
diff --git a/nodejs/src/logs/framework/resolve-log-buffer.ts b/nodejs/src/logs/framework/resolve-log-buffer.ts
index f786a8c61938..a5cf50cd3c66 100644
--- a/nodejs/src/logs/framework/resolve-log-buffer.ts
+++ b/nodejs/src/logs/framework/resolve-log-buffer.ts
@@ -78,7 +78,7 @@ async function buildRecordsTransform(
if (!(await transformer.teamHasTransformations(message.teamId))) {
return undefined
}
- return (records) => transformer.transformRecords(message.teamId, records, batchBudget)
+ return (records) => transformer.transformRecords(message.teamId, records, batchBudget, message.sentAsInternalLogs)
}
export async function resolveLogBuffer(
@@ -117,7 +117,7 @@ export async function resolveLogBuffer(
stages.push(makeTransformStage(recordsTransform))
}
let shortestRetentionDays =
- message.internalLogs !== undefined ? INTERNAL_LOGS_MIN_RETENTION_DAYS : defaultRetentionDays
+ message.verifiedAsInternalLogs !== undefined ? INTERNAL_LOGS_MIN_RETENTION_DAYS : defaultRetentionDays
if (useRetention && retentionRuleSet) {
stages.push(makeRetentionStage(retentionRuleSet, message.teamId, defaultRetentionDays))
shortestRetentionDays = Math.min(
diff --git a/nodejs/src/logs/internal-logs-producer.ts b/nodejs/src/logs/internal-logs-producer.ts
index 58dcc7e2ab93..7f80c5c346c4 100644
--- a/nodejs/src/logs/internal-logs-producer.ts
+++ b/nodejs/src/logs/internal-logs-producer.ts
@@ -96,6 +96,14 @@ export class InternalLogsProducer {
this.isTeamEnabled = buildIntegerMatcher(config.LOGS_INTERNAL_PRODUCER_TEAMS, true)
}
+ /** Counts the records `enqueue` would send, as `loop_guard`, so logs kept out of a loop show on the dashboard. */
+ skipLooping(source: string, entries: InternalLogEntry[]): void {
+ if (!this.enabled || !Object.hasOwn(INTERNAL_SOURCE_POLICIES, source)) {
+ return
+ }
+ countRecords(source, 'loop_guard', entries.filter((entry) => this.isTeamEnabled(entry.team_id)).length)
+ }
+
enqueue(source: string, entries: InternalLogEntry[]): void {
if (!this.enabled || !Object.hasOwn(INTERNAL_SOURCE_POLICIES, source)) {
return
diff --git a/nodejs/src/logs/internal-logs.test.ts b/nodejs/src/logs/internal-logs.test.ts
index 65cf99db89b5..3ab45c710805 100644
--- a/nodejs/src/logs/internal-logs.test.ts
+++ b/nodejs/src/logs/internal-logs.test.ts
@@ -1,3 +1,7 @@
+import { register } from 'prom-client'
+
+import { logger } from '~/common/utils/logger'
+
import { InternalLogsVerifier, signInternalLogs } from './internal-logs'
const KEY = 'internal-logs-test-key'
@@ -9,6 +13,8 @@ const SIGNED_AT = new Date('2026-10-27T12:00:00Z')
const CREATED_AT = '2026-10-27T12:01:00.123456789+00:00'
describe('InternalLogsVerifier', () => {
+ afterEach(() => jest.restoreAllMocks())
+
it.each([
{
name: 'accepts a batch signed with the newest key',
@@ -72,9 +78,20 @@ describe('InternalLogsVerifier', () => {
},
])(
'bills a batch with $name',
- ({ signature, keys = KEY, teams = String(TEAM_ID), createdAt = CREATED_AT as string | null }) => {
+ async ({ signature, keys = KEY, teams = String(TEAM_ID), createdAt = CREATED_AT as string | null }) => {
+ // Collected pod stdout comes back with no header, so a per-batch log line on a key mismatch loops.
+ const logSpies = (['info', 'warn', 'error'] as const).map((level) => jest.spyOn(logger, level))
+ const notInternal = async (): Promise<number> =>
+ (await register.getSingleMetric('logs_ingestion_internal_logs_total')!.get()).values
+ .filter((v) => v.labels.outcome === 'not_internal')
+ .reduce((sum, v) => sum + v.value, 0)
+ const before = await notInternal()
+
const verifier = new InternalLogsVerifier(keys, teams)
expect(verifier.verify(signature, TEAM_ID, createdAt ?? undefined)).toBeUndefined()
+
+ expect(await notInternal()).toBe(before + (signature === undefined ? 0 : 1))
+ logSpies.forEach((spy) => expect(spy).not.toHaveBeenCalled())
}
)
diff --git a/nodejs/src/logs/internal-logs.ts b/nodejs/src/logs/internal-logs.ts
index e9ecebc4737a..faeb94071d39 100644
--- a/nodejs/src/logs/internal-logs.ts
+++ b/nodejs/src/logs/internal-logs.ts
@@ -2,7 +2,6 @@ import { createHmac, timingSafeEqual } from 'crypto'
import { Counter } from 'prom-client'
import { buildIntegerMatcher } from '~/common/config/matchers'
-import { logger } from '~/common/utils/logger'
import type { ValueMatcher } from '~/types'
export const INTERNAL_LOGS_HEADER = 'posthog-internal'
@@ -115,10 +114,6 @@ export class InternalLogsVerifier {
if (!policy) {
logsInternalLogsCounter.inc({ outcome: 'not_internal', reason: reason ?? 'unknown_source' })
// The team is not a counter label, because a caller that rotates tokens would make its values unbounded.
- logger.warn('Internal logs header did not verify, batch billed as normal', {
- teamId,
- reason: reason ?? 'unknown_source',
- })
return undefined
}
logsInternalLogsCounter.inc({ outcome: 'internal', reason: '' })
diff --git a/nodejs/src/logs/logs-ingestion-consumer.test.ts b/nodejs/src/logs/logs-ingestion-consumer.test.ts
index 6654e2eb685d..af94b5e38301 100644
--- a/nodejs/src/logs/logs-ingestion-consumer.test.ts
+++ b/nodejs/src/logs/logs-ingestion-consumer.test.ts
@@ -2866,7 +2866,10 @@ describe('LogsIngestionConsumer', () => {
describe('hog log transformations', () => {
let transformerConsumer: LogsIngestionConsumer
- const buildTransformer = async (hog: string): Promise<LogsTransformerService> => {
+ const buildTransformer = async (
+ hog: string,
+ monitoring = { queueAppMetric: jest.fn(), queueLogs: jest.fn(), flush: jest.fn() }
+ ): Promise<LogsTransformerService> => {
const fn = {
id: '00000000-0000-0000-0000-000000000001',
team_id: team.id,
@@ -2883,7 +2886,6 @@ describe('LogsIngestionConsumer', () => {
getHogFunctionsForTeams: jest.fn().mockResolvedValue({ [team.id]: [fn] }),
getHogFunctionIdsForTeams: jest.fn().mockResolvedValue({ [team.id]: [fn.id] }),
}
- const monitoring = { queueAppMetric: jest.fn(), queueLogs: jest.fn(), flush: jest.fn() }
return new LogsTransformerService(manager as any, monitoring as any, {
siteUrl: 'http://localhost:8010',
hogTimeoutMs: 10,
@@ -2982,31 +2984,53 @@ describe('LogsIngestionConsumer', () => {
)
})
- it('does not run transformations when the team is not in the allowlist', async () => {
- const logsTransformer = await buildTransformer(`return null`)
+ const internalLogsConfig = {
+ LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: '*',
+ LOGS_INTERNAL_INGESTION_SIGNING_KEYS: 'internal-logs-test-key',
+ }
+
+ it('runs transformations on a batch whose internal logs header fails to verify, and keeps its logs out of the producer', async () => {
+ const monitoring = { queueAppMetric: jest.fn(), queueLogs: jest.fn(), flush: jest.fn() }
+ const logsTransformer = await buildTransformer(`throw Error('boom')`, monitoring)
transformerConsumer = await createLogsIngestionConsumer(
hub,
- { LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '' },
+ { LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '*', ...internalLogsConfig },
{ logsTransformer }
)
- const messages = await createKafkaMessages([createLogMessage()], { token: team.api_token })
- await waitForBackgroundTasks(transformerConsumer.processKafkaBatch(messages))
+ const [message] = await createKafkaMessages([createLogMessage()], { token: team.api_token })
+ message.headers!.push(
+ internalLogsHeaders(signInternalLogs('rotated-away-key', 'hog_function', team.id, new Date(Date.now())))
+ )
+ await waitForBackgroundTasks(transformerConsumer.processKafkaBatch([message]))
- const logsMessages = getProducedKafkaMessages().filter((m) => m.topic === 'clickhouse_logs_test')
- expect(logsMessages).toHaveLength(1)
+ expect(monitoring.queueLogs).toHaveBeenCalledWith(expect.any(Array), 'hog_function', {
+ aboutInternalLogs: true,
+ })
})
- it('does not run transformations when the killswitch is on', async () => {
+ it.each([
+ { name: 'the team is not in the allowlist', config: { LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '' } },
+ {
+ name: 'the killswitch is on',
+ config: { LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '*', LOGS_TRANSFORMATIONS_KILLSWITCH: true },
+ },
+ {
+ name: 'the internal logs header verifies',
+ config: { LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '*', ...internalLogsConfig },
+ signingKey: 'internal-logs-test-key',
+ },
+ ])('does not run transformations when $name', async ({ config, signingKey }) => {
const logsTransformer = await buildTransformer(`return null`)
- transformerConsumer = await createLogsIngestionConsumer(
- hub,
- { LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '*', LOGS_TRANSFORMATIONS_KILLSWITCH: true },
- { logsTransformer }
- )
+ transformerConsumer = await createLogsIngestionConsumer(hub, config, { logsTransformer })
- const messages = await createKafkaMessages([createLogMessage()], { token: team.api_token })
- await waitForBackgroundTasks(transformerConsumer.processKafkaBatch(messages))
+ const [message] = await createKafkaMessages([createLogMessage()], { token: team.api_token })
+ if (signingKey) {
+ message.headers!.push(
+ internalLogsHeaders(signInternalLogs(signingKey, 'hog_function', team.id, new Date(Date.now())))
+ )
+ }
+ await waitForBackgroundTasks(transformerConsumer.processKafkaBatch([message]))
const logsMessages = getProducedKafkaMessages().filter((m) => m.topic === 'clickhouse_logs_test')
expect(logsMessages).toHaveLength(1)
diff --git a/nodejs/src/logs/logs-ingestion-consumer.ts b/nodejs/src/logs/logs-ingestion-consumer.ts
index cd409df799c4..fd0372260c4e 100644
--- a/nodejs/src/logs/logs-ingestion-consumer.ts
+++ b/nodejs/src/logs/logs-ingestion-consumer.ts
@@ -515,11 +515,11 @@ export class LogsIngestionConsumer {
// Transformations write hog_function logs, which could come back here as internal logs and loop.
private appliesTeamLogRules(message: LogsIngestionMessage): boolean {
- return message.internalLogs?.teamRules ?? true
+ return message.verifiedAsInternalLogs?.teamRules ?? true
}
private isUnbilledInternal(message: LogsIngestionMessage): boolean {
- return message.internalLogs !== undefined && !message.internalLogs.billed
+ return message.verifiedAsInternalLogs !== undefined && !message.verifiedAsInternalLogs.billed
}
private isTransformationsEnabledForTeam(teamId: number): boolean {
@@ -558,7 +558,8 @@ export class LogsIngestionConsumer {
if (!(await transformer.teamHasTransformations(message.teamId))) {
return undefined
}
- return (records) => transformer.transformRecords(message.teamId, records, batchBudget)
+ return (records) =>
+ transformer.transformRecords(message.teamId, records, batchBudget, message.sentAsInternalLogs)
}
/**
@@ -629,7 +630,7 @@ export class LogsIngestionConsumer {
stages.push(makeTransformStage(recordsTransform))
}
let shortestRetentionDays =
- message.internalLogs !== undefined ? INTERNAL_LOGS_MIN_RETENTION_DAYS : defaultRetentionDays
+ message.verifiedAsInternalLogs !== undefined ? INTERNAL_LOGS_MIN_RETENTION_DAYS : defaultRetentionDays
if (useRetention && retentionRuleSet) {
stages.push(makeRetentionStage(retentionRuleSet, message.teamId, defaultRetentionDays))
shortestRetentionDays = Math.min(
@@ -1257,8 +1258,8 @@ export class LogsIngestionConsumer {
key: null,
headers: {
...withoutInternalLogsHeaders(parseKafkaHeaders(message.message.headers)),
- ...(message.internalLogs !== undefined
- ? { [INTERNAL_SOURCE_HEADER]: message.internalLogs.source }
+ ...(message.verifiedAsInternalLogs !== undefined
+ ? { [INTERNAL_SOURCE_HEADER]: message.verifiedAsInternalLogs.source }
: {}),
token: message.token,
team_id: message.teamId.toString(),
@@ -1592,7 +1593,8 @@ export class LogsIngestionConsumer {
recordCount,
minTimestampMicros: parseMinTimestampHeader(headers.min_timestamp),
backfillRequested: headers.backfill_days !== undefined,
- internalLogs: this.internalLogsVerifier.verify(
+ sentAsInternalLogs: headers[INTERNAL_LOGS_HEADER] !== undefined,
+ verifiedAsInternalLogs: this.internalLogsVerifier.verify(
headers[INTERNAL_LOGS_HEADER],
team.id,
headers.created_at
diff --git a/nodejs/src/logs/transformations/logs-transformer.service.ts b/nodejs/src/logs/transformations/logs-transformer.service.ts
index 6e0073246e61..d582b44aa5cb 100644
--- a/nodejs/src/logs/transformations/logs-transformer.service.ts
+++ b/nodejs/src/logs/transformations/logs-transformer.service.ts
@@ -163,7 +163,8 @@ export class LogsTransformerService {
public async transformRecords(
teamId: number,
records: LogRecord[],
- batchBudget?: TransformationBatchBudget
+ batchBudget?: TransformationBatchBudget,
+ sentAsInternalLogs = false
): Promise<{ recordsDropped: number; recordsDroppedByFunctionId: Map<string, number> }> {
const recordsDroppedByFunctionId = new Map<string, number>()
let recordsDropped = 0
@@ -270,7 +271,7 @@ export class LogsTransformerService {
}
transformationVmDurationHistogram.observe(messageVmMs / 1000)
- this.queueAggregates(teamId, aggregates)
+ this.queueAggregates(teamId, aggregates, sentAsInternalLogs)
return { recordsDropped, recordsDroppedByFunctionId }
}
@@ -495,7 +496,11 @@ export class LogsTransformerService {
return agg
}
- private queueAggregates(teamId: number, aggregates: Map<string, FunctionAggregates>): void {
+ private queueAggregates(
+ teamId: number,
+ aggregates: Map<string, FunctionAggregates>,
+ sentAsInternalLogs: boolean
+ ): void {
for (const [functionId, agg] of aggregates) {
if (agg.succeeded > 0) {
this.monitoring.queueAppMetric(
@@ -546,7 +551,7 @@ export class LogsTransformerService {
)
}
if (agg.errorLogs.length > 0) {
- this.monitoring.queueLogs(agg.errorLogs, 'hog_function')
+ this.monitoring.queueLogs(agg.errorLogs, 'hog_function', { aboutInternalLogs: sentAsInternalLogs })
}
}
}
diff --git a/nodejs/src/logs/types.ts b/nodejs/src/logs/types.ts
index 7af2c810b8bb..d6f46f93390e 100644
--- a/nodejs/src/logs/types.ts
+++ b/nodejs/src/logs/types.ts
@@ -14,5 +14,7 @@ export type LogsIngestionMessage = {
/** Earliest row timestamp in epoch microseconds, from the `min_timestamp` header; absent from older producers. */
minTimestampMicros?: number
backfillRequested?: boolean
- internalLogs?: InternalLogs
+ /** The posthog-internal header was present, whether or not it verified. */
+ sentAsInternalLogs: boolean
+ verifiedAsInternalLogs?: InternalLogs
}
diff --git a/nodejs/src/servers/ingestion-logs-server.ts b/nodejs/src/servers/ingestion-logs-server.ts
index 1709bdc5c8c2..1f0bcf93edeb 100644
--- a/nodejs/src/servers/ingestion-logs-server.ts
+++ b/nodejs/src/servers/ingestion-logs-server.ts
@@ -32,7 +32,10 @@ import { SamplingRulesCache } from '~/logs/sampling/sampling-rules-cache'
import { LogsTransformerService } from '~/logs/transformations/logs-transformer.service'
import { HogFunctionManagerService } from '../cdp/services/managers/hog-function-manager.service'
-import { HogFunctionMonitoringService } from '../cdp/services/monitoring/hog-function-monitoring.service'
+import {
+ TransformationsMonitoringConfig,
+ createTransformationsMonitoringService,
+} from '../cdp/services/monitoring/hog-function-monitoring.service'
import { EncryptedFields } from '../cdp/utils/encryption-utils'
import { CommonConfig } from '../common/config'
import { DatabaseConnectionConfig, KafkaBrokerConfig, RedisConnectionsConfig } from '../ingestion/config'
@@ -52,6 +55,7 @@ import { BaseServerConfig, CleanupResources, NodeServer, ServerLifecycle } from
*/
export type IngestionLogsServerConfig = BaseServerConfig &
LogsIngestionConsumerConfig &
+ TransformationsMonitoringConfig &
LogsIngestionOutputsConfig &
KafkaWarpstreamLogsProducerEnvConfig &
KafkaWarpstreamIngestionProducerEnvConfig &
@@ -145,7 +149,7 @@ export class IngestionLogsServer implements NodeServer {
this.pubsub,
new EncryptedFields(this.config.ENCRYPTION_SALT_KEYS)
)
- const hogFunctionMonitoring = new HogFunctionMonitoringService(outputs)
+ const hogFunctionMonitoring = createTransformationsMonitoringService(outputs, this.config, teamManager)
const logsTransformer = new LogsTransformerService(hogFunctionManager, hogFunctionMonitoring, {
siteUrl: this.config.SITE_URL,
@@ -176,7 +180,18 @@ export class IngestionLogsServer implements NodeServer {
usageBatch,
})
await consumer.start()
- return consumer.service
+ const service = consumer.service
+ return {
+ ...service,
+ // Drains here rather than in additionalCleanup, which runs after Postgres and the HTTP agents close.
+ onShutdown: async () => {
+ try {
+ await service.onShutdown()
+ } finally {
+ await hogFunctionMonitoring.stop()
+ }
+ },
+ }
})
const readyServices = await Promise.all(serviceLoaders.map((loader) => loader()))