:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
fix(logs): send logs ingestion transformation logs with a loop guard (#130285)
Jon McCallum1e69a5cb7dce
diff --git a/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.test.ts b/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.test.tsindex e7d0b0aa7def..1b6a198fcb06 100644--- a/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.test.ts+++ b/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.test.ts@@ -1,4 +1,5 @@import { DateTime } from 'luxon'+import { register } from 'prom-client'import { APP_METRICS_OUTPUT, LOG_ENTRIES_OUTPUT } from '~/common/outputs'import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'@@ -177,6 +178,24 @@ describe('HogFunctionMonitoringService', () => {expect(fetchMock).toHaveBeenCalledTimes(2)})+ it('keeps logs about internal logs out of the logs product, writes them to log_entries, and counts them', async () => {+ const fetchMock = jest.mocked(internalFetch).mockResolvedValue({ status: 200, dump: jest.fn() } as any)+ const loopGuarded = async (): Promise<number> =>+ (await register.getSingleMetric('logs_internal_producer_records_total')!.get()).values.find(+ (v) => v.labels.outcome === 'loop_guard'+ )?.value ?? 0+ const before = await loopGuarded()+ service = new HogFunctionMonitoringService(outputs, internalLogsProducer(), new Set(['hog_flow']))++ service.queueLogs([log(2), log(2), log(3)], 'hog_flow', { aboutInternalLogs: true })+ await service.flush()+ await service.stop()++ expect(produceMock).toHaveBeenCalledTimes(3)+ expect(fetchMock).not.toHaveBeenCalled()+ expect((await loopGuarded()) - before).toBe(2)+ })+it('sends nothing to the logs product for a source not in CDP_INTERNAL_LOGS_SOURCES', async () => {const fetchMock = jest.mocked(internalFetch).mockResolvedValue({ status: 200, dump: jest.fn() } as any)service = new HogFunctionMonitoringService(outputs, internalLogsProducer(), new Set(['hog_function']))diff --git a/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.ts b/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.tsindex 4e96a01c9771..b08099a4c2a2 100644--- a/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.ts+++ b/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.ts@@ -198,7 +198,8 @@ export class HogFunctionMonitoringService {metrics.forEach((metric) => this.queueAppMetric(metric, source))}- queueLogs(logEntries: LogEntry[], source: MetricLogSource) {+ /** Set `aboutInternalLogs` for logs about a batch that came from the internal producer, so they never go back to it and loop. */+ queueLogs(logEntries: LogEntry[], source: MetricLogSource, { aboutInternalLogs = false } = {}) {const sourcedLogEntries = logEntries.map((logEntry) => ({...logEntry,log_source: source,@@ -211,7 +212,12 @@ export class HogFunctionMonitoringService {})hogFunctionMonitoringPendingMessages.set(this.queuedLogMessages.length)- if (this.internalLogsSources.has(source)) {+ if (!this.internalLogsSources.has(source)) {+ return+ }+ if (aboutInternalLogs) {+ this.internalLogsProducer?.skipLooping(source, sourcedLogEntries)+ } else {this.internalLogsProducer?.enqueue(source, sourcedLogEntries)}}diff --git a/nodejs/src/logs/framework/logs-ingestion-framework-consumer.test.ts b/nodejs/src/logs/framework/logs-ingestion-framework-consumer.test.tsindex b30566f275f7..03201d037e8e 100644--- a/nodejs/src/logs/framework/logs-ingestion-framework-consumer.test.ts+++ b/nodejs/src/logs/framework/logs-ingestion-framework-consumer.test.ts@@ -14,6 +14,7 @@ import { createTeam, createTestTeamFixture, getTeam } from '~/tests/helpers/sql'import { Hub, Team } from '~/types'import { getDefaultTracesIngestionConsumerConfig } from '../config'+import { INTERNAL_LOGS_HEADER, signInternalLogs } from '../internal-logs'import { LogRecord, encodeLogRecords } from '../log-record-avro'import { DEFAULT_LOGS_RETENTION_DAYS, LogsIngestionConsumer, logMessageDlqCounter } from '../logs-ingestion-consumer'import { LOGS_DLQ_OUTPUT, LOGS_OUTPUT, LogsDlqOutput, LogsOutput } from '../outputs/outputs'@@ -185,6 +186,38 @@ describe('LogsIngestionFrameworkConsumer', () => {await closeHub(hub)})+ it('runs transformations on a batch whose internal logs header fails to verify, flagged as internal', async () => {+ await consumer.stop()+ const logsTransformer = {+ startBatch: jest.fn(),+ teamHasTransformations: jest.fn().mockResolvedValue(true),+ transformRecords: jest.fn().mockResolvedValue({ recordsDropped: 0, recordsDroppedByFunctionId: new Map() }),+ flush: jest.fn().mockResolvedValue(undefined),+ }+ consumer = withoutKafka(+ new LogsIngestionFrameworkConsumer(+ {+ ...hub,+ LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '*',+ LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: '*',+ LOGS_INTERNAL_INGESTION_SIGNING_KEYS: 'current-key',+ },+ { ...consumerDeps(), logsTransformer }+ )+ )+ await consumer.start()++ await processAndWait(+ await createKafkaMessages(1, {+ token: team.api_token,+ [INTERNAL_LOGS_HEADER]: signInternalLogs('rotated-away-key', 'hog_function', team.id),+ })+ )++ expect(logsTransformer.transformRecords).toHaveBeenCalledWith(team.id, expect.any(Array), undefined, true)+ expect(producedTo(KAFKA_LOGS_CLICKHOUSE)).toHaveLength(1)+ })+it('produces a valid message to the logs topic with the batch headers', async () => {const messages = await createKafkaMessages(1, { token: team.api_token })diff --git a/nodejs/src/logs/framework/logs-pipeline-context.ts b/nodejs/src/logs/framework/logs-pipeline-context.tsindex 74d19801ffc4..a126422226d3 100644--- a/nodejs/src/logs/framework/logs-pipeline-context.ts+++ b/nodejs/src/logs/framework/logs-pipeline-context.ts@@ -117,11 +117,11 @@ export type LogsPipelineContext = {/** Transformations write hog_function logs, which could come back here as internal logs and loop. */export function appliesTeamLogRules(message: LogsIngestionMessage): boolean {- return message.internalLogs?.teamRules ?? true+ return message.verifiedAsInternalLogs?.teamRules ?? true}export function isUnbilledInternal(message: LogsIngestionMessage): boolean {- return message.internalLogs !== undefined && !message.internalLogs.billed+ return message.verifiedAsInternalLogs !== undefined && !message.verifiedAsInternalLogs.billed}/** A failed batch replays in full and the logs table has no dedupe, so each retry here avoids duplicates. */diff --git a/nodejs/src/logs/framework/parse-logs-message-step.ts b/nodejs/src/logs/framework/parse-logs-message-step.tsindex edaaf048ec74..c06d580527dd 100644--- a/nodejs/src/logs/framework/parse-logs-message-step.ts+++ b/nodejs/src/logs/framework/parse-logs-message-step.ts@@ -100,7 +100,12 @@ export function createParseLogsMessageStep<T extends LogsPipelineInput>(recordCount,minTimestampMicros: parseMinTimestampHeader(headers.min_timestamp),backfillRequested: headers.backfill_days !== undefined,- internalLogs: internalLogsVerifier.verify(headers[INTERNAL_LOGS_HEADER], team.id, headers.created_at),+ sentAsInternalLogs: headers[INTERNAL_LOGS_HEADER] !== undefined,+ verifiedAsInternalLogs: internalLogsVerifier.verify(+ headers[INTERNAL_LOGS_HEADER],+ team.id,+ headers.created_at+ ),})} catch (e) {// A failed quarantine of bad size headers lands here, and a second DLQ failure fails the batch.diff --git a/nodejs/src/logs/framework/process-log-message-step.ts b/nodejs/src/logs/framework/process-log-message-step.tsindex 840d9b447ba6..58ab57bdec29 100644--- a/nodejs/src/logs/framework/process-log-message-step.ts+++ b/nodejs/src/logs/framework/process-log-message-step.ts@@ -254,8 +254,8 @@ async function processLogMessage(context: LogsPipelineContext, input: LogsProceskey: null,headers: {...withoutInternalLogsHeaders(parseKafkaHeaders(input.message.headers)),- ...(input.internalLogs !== undefined- ? { [INTERNAL_SOURCE_HEADER]: input.internalLogs.source }+ ...(input.verifiedAsInternalLogs !== undefined+ ? { [INTERNAL_SOURCE_HEADER]: input.verifiedAsInternalLogs.source }: {}),token: input.token,team_id: input.teamId.toString(),diff --git a/nodejs/src/logs/framework/resolve-log-buffer.ts b/nodejs/src/logs/framework/resolve-log-buffer.tsindex f786a8c61938..a5cf50cd3c66 100644--- a/nodejs/src/logs/framework/resolve-log-buffer.ts+++ b/nodejs/src/logs/framework/resolve-log-buffer.ts@@ -78,7 +78,7 @@ async function buildRecordsTransform(if (!(await transformer.teamHasTransformations(message.teamId))) {return undefined}- return (records) => transformer.transformRecords(message.teamId, records, batchBudget)+ return (records) => transformer.transformRecords(message.teamId, records, batchBudget, message.sentAsInternalLogs)}export async function resolveLogBuffer(@@ -117,7 +117,7 @@ export async function resolveLogBuffer(stages.push(makeTransformStage(recordsTransform))}let shortestRetentionDays =- message.internalLogs !== undefined ? INTERNAL_LOGS_MIN_RETENTION_DAYS : defaultRetentionDays+ message.verifiedAsInternalLogs !== undefined ? INTERNAL_LOGS_MIN_RETENTION_DAYS : defaultRetentionDaysif (useRetention && retentionRuleSet) {stages.push(makeRetentionStage(retentionRuleSet, message.teamId, defaultRetentionDays))shortestRetentionDays = Math.min(diff --git a/nodejs/src/logs/internal-logs-producer.ts b/nodejs/src/logs/internal-logs-producer.tsindex 58dcc7e2ab93..7f80c5c346c4 100644--- a/nodejs/src/logs/internal-logs-producer.ts+++ b/nodejs/src/logs/internal-logs-producer.ts@@ -96,6 +96,14 @@ export class InternalLogsProducer {this.isTeamEnabled = buildIntegerMatcher(config.LOGS_INTERNAL_PRODUCER_TEAMS, true)}+ /** Counts the records `enqueue` would send, as `loop_guard`, so logs kept out of a loop show on the dashboard. */+ skipLooping(source: string, entries: InternalLogEntry[]): void {+ if (!this.enabled || !Object.hasOwn(INTERNAL_SOURCE_POLICIES, source)) {+ return+ }+ countRecords(source, 'loop_guard', entries.filter((entry) => this.isTeamEnabled(entry.team_id)).length)+ }+enqueue(source: string, entries: InternalLogEntry[]): void {if (!this.enabled || !Object.hasOwn(INTERNAL_SOURCE_POLICIES, source)) {returndiff --git a/nodejs/src/logs/internal-logs.test.ts b/nodejs/src/logs/internal-logs.test.tsindex 65cf99db89b5..3ab45c710805 100644--- a/nodejs/src/logs/internal-logs.test.ts+++ b/nodejs/src/logs/internal-logs.test.ts@@ -1,3 +1,7 @@+import { register } from 'prom-client'++import { logger } from '~/common/utils/logger'+import { InternalLogsVerifier, signInternalLogs } from './internal-logs'const KEY = 'internal-logs-test-key'@@ -9,6 +13,8 @@ const SIGNED_AT = new Date('2026-10-27T12:00:00Z')const CREATED_AT = '2026-10-27T12:01:00.123456789+00:00'describe('InternalLogsVerifier', () => {+ afterEach(() => jest.restoreAllMocks())+it.each([{name: 'accepts a batch signed with the newest key',@@ -72,9 +78,20 @@ describe('InternalLogsVerifier', () => {},])('bills a batch with $name',- ({ signature, keys = KEY, teams = String(TEAM_ID), createdAt = CREATED_AT as string | null }) => {+ async ({ signature, keys = KEY, teams = String(TEAM_ID), createdAt = CREATED_AT as string | null }) => {+ // Collected pod stdout comes back with no header, so a per-batch log line on a key mismatch loops.+ const logSpies = (['info', 'warn', 'error'] as const).map((level) => jest.spyOn(logger, level))+ const notInternal = async (): Promise<number> =>+ (await register.getSingleMetric('logs_ingestion_internal_logs_total')!.get()).values+ .filter((v) => v.labels.outcome === 'not_internal')+ .reduce((sum, v) => sum + v.value, 0)+ const before = await notInternal()+const verifier = new InternalLogsVerifier(keys, teams)expect(verifier.verify(signature, TEAM_ID, createdAt ?? undefined)).toBeUndefined()++ expect(await notInternal()).toBe(before + (signature === undefined ? 0 : 1))+ logSpies.forEach((spy) => expect(spy).not.toHaveBeenCalled())})diff --git a/nodejs/src/logs/internal-logs.ts b/nodejs/src/logs/internal-logs.tsindex e9ecebc4737a..faeb94071d39 100644--- a/nodejs/src/logs/internal-logs.ts+++ b/nodejs/src/logs/internal-logs.ts@@ -2,7 +2,6 @@ import { createHmac, timingSafeEqual } from 'crypto'import { Counter } from 'prom-client'import { buildIntegerMatcher } from '~/common/config/matchers'-import { logger } from '~/common/utils/logger'import type { ValueMatcher } from '~/types'export const INTERNAL_LOGS_HEADER = 'posthog-internal'@@ -115,10 +114,6 @@ export class InternalLogsVerifier {if (!policy) {logsInternalLogsCounter.inc({ outcome: 'not_internal', reason: reason ?? 'unknown_source' })// The team is not a counter label, because a caller that rotates tokens would make its values unbounded.- logger.warn('Internal logs header did not verify, batch billed as normal', {- teamId,- reason: reason ?? 'unknown_source',- })return undefined}logsInternalLogsCounter.inc({ outcome: 'internal', reason: '' })diff --git a/nodejs/src/logs/logs-ingestion-consumer.test.ts b/nodejs/src/logs/logs-ingestion-consumer.test.tsindex 6654e2eb685d..af94b5e38301 100644--- a/nodejs/src/logs/logs-ingestion-consumer.test.ts+++ b/nodejs/src/logs/logs-ingestion-consumer.test.ts@@ -2866,7 +2866,10 @@ describe('LogsIngestionConsumer', () => {describe('hog log transformations', () => {let transformerConsumer: LogsIngestionConsumer- const buildTransformer = async (hog: string): Promise<LogsTransformerService> => {+ const buildTransformer = async (+ hog: string,+ monitoring = { queueAppMetric: jest.fn(), queueLogs: jest.fn(), flush: jest.fn() }+ ): Promise<LogsTransformerService> => {const fn = {id: '00000000-0000-0000-0000-000000000001',team_id: team.id,@@ -2883,7 +2886,6 @@ describe('LogsIngestionConsumer', () => {getHogFunctionsForTeams: jest.fn().mockResolvedValue({ [team.id]: [fn] }),getHogFunctionIdsForTeams: jest.fn().mockResolvedValue({ [team.id]: [fn.id] }),}- const monitoring = { queueAppMetric: jest.fn(), queueLogs: jest.fn(), flush: jest.fn() }return new LogsTransformerService(manager as any, monitoring as any, {siteUrl: 'http://localhost:8010',hogTimeoutMs: 10,@@ -2982,31 +2984,53 @@ describe('LogsIngestionConsumer', () => {)})- it('does not run transformations when the team is not in the allowlist', async () => {- const logsTransformer = await buildTransformer(`return null`)+ const internalLogsConfig = {+ LOGS_INTERNAL_INGESTION_ENABLED_TEAMS: '*',+ LOGS_INTERNAL_INGESTION_SIGNING_KEYS: 'internal-logs-test-key',+ }++ it('runs transformations on a batch whose internal logs header fails to verify, and keeps its logs out of the producer', async () => {+ const monitoring = { queueAppMetric: jest.fn(), queueLogs: jest.fn(), flush: jest.fn() }+ const logsTransformer = await buildTransformer(`throw Error('boom')`, monitoring)transformerConsumer = await createLogsIngestionConsumer(hub,- { LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '' },+ { LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '*', ...internalLogsConfig },{ logsTransformer })- const messages = await createKafkaMessages([createLogMessage()], { token: team.api_token })- await waitForBackgroundTasks(transformerConsumer.processKafkaBatch(messages))+ const [message] = await createKafkaMessages([createLogMessage()], { token: team.api_token })+ message.headers!.push(+ internalLogsHeaders(signInternalLogs('rotated-away-key', 'hog_function', team.id, new Date(Date.now())))+ )+ await waitForBackgroundTasks(transformerConsumer.processKafkaBatch([message]))- const logsMessages = getProducedKafkaMessages().filter((m) => m.topic === 'clickhouse_logs_test')- expect(logsMessages).toHaveLength(1)+ expect(monitoring.queueLogs).toHaveBeenCalledWith(expect.any(Array), 'hog_function', {+ aboutInternalLogs: true,+ })})- it('does not run transformations when the killswitch is on', async () => {+ it.each([+ { name: 'the team is not in the allowlist', config: { LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '' } },+ {+ name: 'the killswitch is on',+ config: { LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '*', LOGS_TRANSFORMATIONS_KILLSWITCH: true },+ },+ {+ name: 'the internal logs header verifies',+ config: { LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '*', ...internalLogsConfig },+ signingKey: 'internal-logs-test-key',+ },+ ])('does not run transformations when $name', async ({ config, signingKey }) => {const logsTransformer = await buildTransformer(`return null`)- transformerConsumer = await createLogsIngestionConsumer(- hub,- { LOGS_TRANSFORMATIONS_ENABLED_TEAMS: '*', LOGS_TRANSFORMATIONS_KILLSWITCH: true },- { logsTransformer }- )+ transformerConsumer = await createLogsIngestionConsumer(hub, config, { logsTransformer })- const messages = await createKafkaMessages([createLogMessage()], { token: team.api_token })- await waitForBackgroundTasks(transformerConsumer.processKafkaBatch(messages))+ const [message] = await createKafkaMessages([createLogMessage()], { token: team.api_token })+ if (signingKey) {+ message.headers!.push(+ internalLogsHeaders(signInternalLogs(signingKey, 'hog_function', team.id, new Date(Date.now())))+ )+ }+ await waitForBackgroundTasks(transformerConsumer.processKafkaBatch([message]))const logsMessages = getProducedKafkaMessages().filter((m) => m.topic === 'clickhouse_logs_test')expect(logsMessages).toHaveLength(1)diff --git a/nodejs/src/logs/logs-ingestion-consumer.ts b/nodejs/src/logs/logs-ingestion-consumer.tsindex cd409df799c4..fd0372260c4e 100644--- a/nodejs/src/logs/logs-ingestion-consumer.ts+++ b/nodejs/src/logs/logs-ingestion-consumer.ts@@ -515,11 +515,11 @@ export class LogsIngestionConsumer {// Transformations write hog_function logs, which could come back here as internal logs and loop.private appliesTeamLogRules(message: LogsIngestionMessage): boolean {- return message.internalLogs?.teamRules ?? true+ return message.verifiedAsInternalLogs?.teamRules ?? true}private isUnbilledInternal(message: LogsIngestionMessage): boolean {- return message.internalLogs !== undefined && !message.internalLogs.billed+ return message.verifiedAsInternalLogs !== undefined && !message.verifiedAsInternalLogs.billed}private isTransformationsEnabledForTeam(teamId: number): boolean {@@ -558,7 +558,8 @@ export class LogsIngestionConsumer {if (!(await transformer.teamHasTransformations(message.teamId))) {return undefined}- return (records) => transformer.transformRecords(message.teamId, records, batchBudget)+ return (records) =>+ transformer.transformRecords(message.teamId, records, batchBudget, message.sentAsInternalLogs)}/**@@ -629,7 +630,7 @@ export class LogsIngestionConsumer {stages.push(makeTransformStage(recordsTransform))}let shortestRetentionDays =- message.internalLogs !== undefined ? INTERNAL_LOGS_MIN_RETENTION_DAYS : defaultRetentionDays+ message.verifiedAsInternalLogs !== undefined ? INTERNAL_LOGS_MIN_RETENTION_DAYS : defaultRetentionDaysif (useRetention && retentionRuleSet) {stages.push(makeRetentionStage(retentionRuleSet, message.teamId, defaultRetentionDays))shortestRetentionDays = Math.min(@@ -1257,8 +1258,8 @@ export class LogsIngestionConsumer {key: null,headers: {...withoutInternalLogsHeaders(parseKafkaHeaders(message.message.headers)),- ...(message.internalLogs !== undefined- ? { [INTERNAL_SOURCE_HEADER]: message.internalLogs.source }+ ...(message.verifiedAsInternalLogs !== undefined+ ? { [INTERNAL_SOURCE_HEADER]: message.verifiedAsInternalLogs.source }: {}),token: message.token,team_id: message.teamId.toString(),@@ -1592,7 +1593,8 @@ export class LogsIngestionConsumer {recordCount,minTimestampMicros: parseMinTimestampHeader(headers.min_timestamp),backfillRequested: headers.backfill_days !== undefined,- internalLogs: this.internalLogsVerifier.verify(+ sentAsInternalLogs: headers[INTERNAL_LOGS_HEADER] !== undefined,+ verifiedAsInternalLogs: this.internalLogsVerifier.verify(headers[INTERNAL_LOGS_HEADER],team.id,headers.created_atdiff --git a/nodejs/src/logs/transformations/logs-transformer.service.ts b/nodejs/src/logs/transformations/logs-transformer.service.tsindex 6e0073246e61..d582b44aa5cb 100644--- a/nodejs/src/logs/transformations/logs-transformer.service.ts+++ b/nodejs/src/logs/transformations/logs-transformer.service.ts@@ -163,7 +163,8 @@ export class LogsTransformerService {public async transformRecords(teamId: number,records: LogRecord[],- batchBudget?: TransformationBatchBudget+ batchBudget?: TransformationBatchBudget,+ sentAsInternalLogs = false): Promise<{ recordsDropped: number; recordsDroppedByFunctionId: Map<string, number> }> {const recordsDroppedByFunctionId = new Map<string, number>()let recordsDropped = 0@@ -270,7 +271,7 @@ export class LogsTransformerService {}transformationVmDurationHistogram.observe(messageVmMs / 1000)- this.queueAggregates(teamId, aggregates)+ this.queueAggregates(teamId, aggregates, sentAsInternalLogs)return { recordsDropped, recordsDroppedByFunctionId }}@@ -495,7 +496,11 @@ export class LogsTransformerService {return agg}- private queueAggregates(teamId: number, aggregates: Map<string, FunctionAggregates>): void {+ private queueAggregates(+ teamId: number,+ aggregates: Map<string, FunctionAggregates>,+ sentAsInternalLogs: boolean+ ): void {for (const [functionId, agg] of aggregates) {if (agg.succeeded > 0) {this.monitoring.queueAppMetric(@@ -546,7 +551,7 @@ export class LogsTransformerService {)}if (agg.errorLogs.length > 0) {- this.monitoring.queueLogs(agg.errorLogs, 'hog_function')+ this.monitoring.queueLogs(agg.errorLogs, 'hog_function', { aboutInternalLogs: sentAsInternalLogs })}}}diff --git a/nodejs/src/logs/types.ts b/nodejs/src/logs/types.tsindex 7af2c810b8bb..d6f46f93390e 100644--- a/nodejs/src/logs/types.ts+++ b/nodejs/src/logs/types.ts@@ -14,5 +14,7 @@ export type LogsIngestionMessage = {/** Earliest row timestamp in epoch microseconds, from the `min_timestamp` header; absent from older producers. */minTimestampMicros?: numberbackfillRequested?: boolean- internalLogs?: InternalLogs+ /** The posthog-internal header was present, whether or not it verified. */+ sentAsInternalLogs: boolean+ verifiedAsInternalLogs?: InternalLogs}diff --git a/nodejs/src/servers/ingestion-logs-server.ts b/nodejs/src/servers/ingestion-logs-server.tsindex 1709bdc5c8c2..1f0bcf93edeb 100644--- a/nodejs/src/servers/ingestion-logs-server.ts+++ b/nodejs/src/servers/ingestion-logs-server.ts@@ -32,7 +32,10 @@ import { SamplingRulesCache } from '~/logs/sampling/sampling-rules-cache'import { LogsTransformerService } from '~/logs/transformations/logs-transformer.service'import { HogFunctionManagerService } from '../cdp/services/managers/hog-function-manager.service'-import { HogFunctionMonitoringService } from '../cdp/services/monitoring/hog-function-monitoring.service'+import {+ TransformationsMonitoringConfig,+ createTransformationsMonitoringService,+} from '../cdp/services/monitoring/hog-function-monitoring.service'import { EncryptedFields } from '../cdp/utils/encryption-utils'import { CommonConfig } from '../common/config'import { DatabaseConnectionConfig, KafkaBrokerConfig, RedisConnectionsConfig } from '../ingestion/config'@@ -52,6 +55,7 @@ import { BaseServerConfig, CleanupResources, NodeServer, ServerLifecycle } from*/export type IngestionLogsServerConfig = BaseServerConfig &LogsIngestionConsumerConfig &+ TransformationsMonitoringConfig &LogsIngestionOutputsConfig &KafkaWarpstreamLogsProducerEnvConfig &KafkaWarpstreamIngestionProducerEnvConfig &@@ -145,7 +149,7 @@ export class IngestionLogsServer implements NodeServer {this.pubsub,new EncryptedFields(this.config.ENCRYPTION_SALT_KEYS))- const hogFunctionMonitoring = new HogFunctionMonitoringService(outputs)+ const hogFunctionMonitoring = createTransformationsMonitoringService(outputs, this.config, teamManager)const logsTransformer = new LogsTransformerService(hogFunctionManager, hogFunctionMonitoring, {siteUrl: this.config.SITE_URL,@@ -176,7 +180,18 @@ export class IngestionLogsServer implements NodeServer {usageBatch,})await consumer.start()- return consumer.service+ const service = consumer.service+ return {+ ...service,+ // Drains here rather than in additionalCleanup, which runs after Postgres and the HTTP agents close.+ onShutdown: async () => {+ try {+ await service.onShutdown()+ } finally {+ await hogFunctionMonitoring.stop()+ }+ },+ }})const readyServices = await Promise.all(serviceLoaders.map((loader) => loader()))