:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(cdp): send hog flow logs to the logs product (#124635)
Jon McCallum60b46c923797
diff --git a/nodejs/src/cdp/cdp-services.ts b/nodejs/src/cdp/cdp-services.tsindex 72664c6604a1..c3bda80b7084 100644--- a/nodejs/src/cdp/cdp-services.ts+++ b/nodejs/src/cdp/cdp-services.ts@@ -9,6 +9,7 @@ import { getRedisHost } from '~/common/utils/db/redis'import { logger } from '~/common/utils/logger'import { PubSub } from '~/common/utils/pubsub'import { TeamManager } from '~/common/utils/team-manager'+import { InternalLogsProducer, InternalLogsProducerConfig } from '~/logs/internal-logs-producer'import type { CommonConfig } from '../common/config'import { InternalCaptureService } from '../common/services/internal-capture'@@ -165,6 +166,7 @@ export type CdpCoreServicesConfig = Pick<| 'CDP_WATCHER_OBSERVE_RESULTS_BUFFER_TIME_MS'| 'CDP_WATCHER_OBSERVE_RESULTS_BUFFER_MAX_RESULTS'| 'CDP_HOGFLOW_AWAITED_STEPS_ENABLED'+ | 'CDP_INTERNAL_LOGS_SOURCES'| 'SES_ACCESS_KEY_ID'| 'SES_SECRET_ACCESS_KEY'| 'SES_REGION'@@ -200,7 +202,8 @@ export type CdpCoreServicesConfig = Pick<| 'CDP_DLQ_BATCH_FAIL_RATIO'| 'CDP_EVENTS_DLQ_TOPIC'| 'CDP_EVENTS_DLQ_PRODUCER'- >+ > &+ InternalLogsProducerConfigexport interface CdpCoreServicesDeps {postgres: PostgresRouter@@ -571,7 +574,16 @@ export function createCdpCoreServices({ awaitedStepsEnabled: config.CDP_HOGFLOW_AWAITED_STEPS_ENABLED })- const hogFunctionMonitoringService = new HogFunctionMonitoringService(outputs)+ const internalLogsSources = new Set(+ config.CDP_INTERNAL_LOGS_SOURCES.split(',')+ .map((source) => source.trim())+ .filter(Boolean)+ )+ const hogFunctionMonitoringService = new HogFunctionMonitoringService(+ outputs,+ internalLogsSources.size > 0 ? new InternalLogsProducer(config, deps.teamManager) : undefined,+ internalLogsSources+ )const hogInvocationResultsService = new HogInvocationResultsService(outputs, config)const warehouseWebhooksService = new WarehouseWebhooksService(outputs)const capturedEventsService = new CapturedEventsService(deps.internalCaptureService, deps.teamManager)diff --git a/nodejs/src/cdp/config.ts b/nodejs/src/cdp/config.tsindex 6d798251fc7d..ac96bee83881 100644--- a/nodejs/src/cdp/config.ts+++ b/nodejs/src/cdp/config.ts@@ -67,6 +67,8 @@ export type CdpConfig = ClickhouseConfig & {CDP_CYCLOTRON_COMPRESS_KAFKA_DATA: boolean// Off until Django emits `$workflow_step_resume`, or parked steps never wake.CDP_HOGFLOW_AWAITED_STEPS_ENABLED: boolean+ // Comma-separated log sources, such as `hog_flow`, that also send their logs to the logs product. Empty sends none.+ CDP_INTERNAL_LOGS_SOURCES: string// Django produces cdp_internal_events to the cyclotron cluster; empty inherits the consumer default.CDP_INTERNAL_EVENTS_CONSUMER_METADATA_BROKER_LIST: stringCDP_REDIS_HOST: string@@ -319,6 +321,7 @@ export function getDefaultCdpConfig(): CdpConfig {CDP_CYCLOTRON_USE_BULK_COPY_JOB: isProdEnv() ? false : true,CDP_CYCLOTRON_COMPRESS_KAFKA_DATA: true,CDP_HOGFLOW_AWAITED_STEPS_ENABLED: isProdEnv() ? false : true,+ CDP_INTERNAL_LOGS_SOURCES: '',CDP_INTERNAL_EVENTS_CONSUMER_METADATA_BROKER_LIST: '',CDP_REDIS_HOST: '127.0.0.1',CDP_REDIS_PORT: 6379,diff --git a/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.test.ts b/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.test.tsindex 8b850fba40d2..44c5073a3609 100644--- a/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.test.ts+++ b/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.test.ts@@ -1,28 +1,36 @@+import { DateTime } from 'luxon'+import { APP_METRICS_OUTPUT, LOG_ENTRIES_OUTPUT } from '~/common/outputs'import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'import { parseJSON } from '~/common/utils/json-parse'+import { internalFetch } from '~/common/utils/request'+import { InternalLogsProducer } from '~/logs/internal-logs-producer'-import { MetricLogSource, MinimalAppMetric } from '../../types'+import { LogEntry, MetricLogSource, MinimalAppMetric } from '../../types'import { HogFunctionMonitoringService, MonitoringOutput } from './hog-function-monitoring.service'+jest.mock('~/common/utils/request', () => ({ internalFetch: jest.fn() }))+describe('HogFunctionMonitoringService', () => {let service: HogFunctionMonitoringServicelet queueMessagesMock: jest.Mock<Promise<void>, [any[]]>+ let outputs: IngestionOutputs<MonitoringOutput>+ let produceMock: jest.MockbeforeEach(() => {queueMessagesMock = jest.fn().mockResolvedValue(undefined)+ produceMock = jest.fn().mockResolvedValue(undefined)const fakeOutput = {- produce: jest.fn().mockResolvedValue(undefined),+ produce: produceMock,queueMessages: queueMessagesMock,checkHealth: jest.fn().mockResolvedValue(undefined),checkTopicExists: jest.fn().mockResolvedValue(undefined),} as any- service = new HogFunctionMonitoringService(- new IngestionOutputs<MonitoringOutput>({- [APP_METRICS_OUTPUT]: fakeOutput,- [LOG_ENTRIES_OUTPUT]: fakeOutput,- })- )+ outputs = new IngestionOutputs<MonitoringOutput>({+ [APP_METRICS_OUTPUT]: fakeOutput,+ [LOG_ENTRIES_OUTPUT]: fakeOutput,+ })+ service = new HogFunctionMonitoringService(outputs)})async function flushedRows(): Promise<Record<string, unknown>[]> {@@ -115,4 +123,69 @@ describe('HogFunctionMonitoringService', () => {expect.objectContaining({ app_source_id: 'flow-1/3', count: 2 }),])})++ const internalLogsProducer = (): InternalLogsProducer =>+ new InternalLogsProducer(+ {+ LOGS_INTERNAL_INGESTION_SIGNING_KEYS: 'key',+ LOGS_INTERNAL_PRODUCER_URL: 'http://capture-logs.test/i/v1/logs',+ LOGS_INTERNAL_PRODUCER_TEAMS: '2',+ LOGS_INTERNAL_PRODUCER_MAX_IN_FLIGHT: 20,+ LOGS_INTERNAL_PRODUCER_MAX_BUFFERED_BYTES: 1024 * 1024,+ LOGS_INTERNAL_PRODUCER_RETRY_TIMEOUT_MS: 1_000,+ },+ { getTeam: (teamId: number) => Promise.resolve({ id: teamId, api_token: `phc_${teamId}` } as any) }+ )++ const log = (teamId: number): LogEntry => ({+ team_id: teamId,+ log_source: 'hog_flow',+ log_source_id: 'flow-1',+ instance_id: 'run-1',+ timestamp: DateTime.fromISO('2026-10-27T12:00:00Z'),+ level: 'info',+ message: `team ${teamId}`,+ })++ it('sends accepted logs to the logs product with the log entry timestamps, and drains them on stop', async () => {+ const fetchMock = jest.mocked(internalFetch).mockResolvedValue({ status: 200, dump: jest.fn() } as any)+ service = new HogFunctionMonitoringService(+ outputs,+ internalLogsProducer(),+ new Set(['hog_flow', 'hog_function'])+ )+ service.queueLogs([log(2), log(2), log(3)], 'hog_flow')+ service.queueLogs([log(2)], 'hog_function')++ await service.flush()+ await new Promise((resolve) => setImmediate(resolve))++ expect(produceMock).toHaveBeenCalledTimes(4)+ expect(fetchMock).toHaveBeenCalledTimes(1)+ const [, options] = fetchMock.mock.calls[0]+ const body = parseJSON(options!.body as string) as any+ const records = body.resourceLogs[0].scopeLogs[0].logRecords+ expect(records.map((r: any) => r.body.stringValue)).toEqual(['team 2', 'team 2'])+ const entryMillis = produceMock.mock.calls+ .map(([message]) => parseJSON(message.value.toString()) as Record<string, unknown>)+ .filter((row) => row.team_id === 2 && row.log_source === 'hog_flow')+ .map((row) => DateTime.fromSQL(row.timestamp as string, { zone: 'utc' }).toMillis())+ expect(records.map((r: any) => Number(BigInt(r.timeUnixNano) / 1_000_000n))).toEqual(entryMillis)++ service.queueLogs([log(2)], 'hog_flow')+ await service.stop()+ expect(fetchMock).toHaveBeenCalledTimes(2)+ })++ it('sends nothing to the logs product for a source not in CDP_INTERNAL_LOGS_SOURCES', async () => {+ const fetchMock = jest.mocked(internalFetch).mockResolvedValue({ status: 200, dump: jest.fn() } as any)+ service = new HogFunctionMonitoringService(outputs, internalLogsProducer(), new Set(['hog_function']))++ service.queueLogs([log(2)], 'hog_flow')+ await service.flush()+ await service.stop()++ expect(produceMock).toHaveBeenCalledTimes(1)+ expect(fetchMock).not.toHaveBeenCalled()+ })})diff --git a/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.ts b/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.tsindex 8e58ec6ca1ae..07b4f53b505f 100644--- a/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.ts+++ b/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.ts@@ -6,6 +6,7 @@ import { AppMetricsAggregator } from '~/common/services/app-metrics-aggregator'import { safeClickhouseString } from '~/common/utils/db/utils'import { logger } from '~/common/utils/logger'import { captureException } from '~/common/utils/posthog'+import type { InternalLogsProducer } from '~/logs/internal-logs-producer'import {CyclotronJobInvocationHogFlow,@@ -69,19 +70,26 @@ export const isHogFlowResult = (return 'hogFlow' in result.invocation}+// Bounded so that shutdown still fits inside the pod's termination grace period.+const INTERNAL_LOGS_DRAIN_TIMEOUT_MS = 10_000+export class HogFunctionMonitoringService {queuedLogMessages: LogEntrySerialized[] = []private appMetricsAggregator: AppMetricsAggregator- constructor(private outputs: IngestionOutputs<MonitoringOutput>) {+ constructor(+ private outputs: IngestionOutputs<MonitoringOutput>,+ private internalLogsProducer?: InternalLogsProducer,+ private internalLogsSources: ReadonlySet<string> = new Set()+ ) {this.appMetricsAggregator = new AppMetricsAggregator(outputs)}- // Required by ResultSink. This sink holds nothing of its own: it writes through the shared- // Kafka outputs, which the server disconnects separately on shutdown.+ // Required by ResultSink. This sink writes through the shared Kafka outputs, which the server+ // disconnects separately on shutdown.async stop(): Promise<void> {- return Promise.resolve()+ await this.internalLogsProducer?.drain(INTERNAL_LOGS_DRAIN_TIMEOUT_MS)}async flush() {@@ -90,6 +98,14 @@ export class HogFunctionMonitoringService {this.queuedLogMessages = []hogFunctionMonitoringPendingMessages.set(0)+ // Only starts the sends. Delivery runs in the background, so a slow capture-logs cannot hold up the Kafka flush.+ // The catch makes sure a producer bug can never stop the log_entries write below.+ try {+ this.internalLogsProducer?.flush()+ } catch (error) {+ captureException(error)+ }+await Promise.all([this.appMetricsAggregator.flush().catch((error) => {// Best-effort — don't disrupt processing just for metrics.@@ -151,17 +167,21 @@ export class HogFunctionMonitoringService {}queueLogs(logEntries: LogEntry[], source: MetricLogSource) {- const logs = fixLogDeduplication(- logEntries.map((logEntry) => ({- ...logEntry,- log_source: source,- }))- )+ const sourcedLogEntries = logEntries.map((logEntry) => ({+ ...logEntry,+ log_source: source,+ }))+ // Sorts its input and moves equal timestamps apart in place, so `sourcedLogEntries` carries the same timestamps as `logs`.+ const logs = fixLogDeduplication(sourcedLogEntries)logs.forEach((logEntry) => {this.queuedLogMessages.push(logEntry)})hogFunctionMonitoringPendingMessages.set(this.queuedLogMessages.length)++ if (this.internalLogsSources.has(source)) {+ this.internalLogsProducer?.enqueue(source, sourcedLogEntries)+ }}queueInvocationResults(results: CyclotronJobInvocationResult[]): void {diff --git a/nodejs/src/cdp/utils.ts b/nodejs/src/cdp/utils.tsindex 78401817b4b8..3b0d9d422f2e 100644--- a/nodejs/src/cdp/utils.ts+++ b/nodejs/src/cdp/utils.ts@@ -275,6 +275,8 @@ export const fixLogDeduplication = (logs: LogEntry[]): LogEntrySerialized[] => {sortedLogs.forEach((logEntry) => {// TRICKY: The clickhouse table dedupes logs with the same timestamp - we need to ensure they are unique by simply plus-ing 1ms// if the timestamp is the same as the previous one+ // Changes the caller's entry in place, not only the copy below. HogFunctionMonitoringService relies on this to+ // send the same timestamps to the logs product, and its test fails by 1 ms if this stops.if (logEntry.timestamp <= previousTimestamp) {logEntry.timestamp = previousTimestamp.plus(1)}