Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

feat(cdp): send hog flow logs to the logs product (#124635)

Jon McCallum60b46c923797
diff --git a/nodejs/src/cdp/cdp-services.ts b/nodejs/src/cdp/cdp-services.ts
index 72664c6604a1..c3bda80b7084 100644
--- a/nodejs/src/cdp/cdp-services.ts
+++ b/nodejs/src/cdp/cdp-services.ts
@@ -9,6 +9,7 @@ import { getRedisHost } from '~/common/utils/db/redis'
import { logger } from '~/common/utils/logger'
import { PubSub } from '~/common/utils/pubsub'
import { TeamManager } from '~/common/utils/team-manager'
+import { InternalLogsProducer, InternalLogsProducerConfig } from '~/logs/internal-logs-producer'
import type { CommonConfig } from '../common/config'
import { InternalCaptureService } from '../common/services/internal-capture'
@@ -165,6 +166,7 @@ export type CdpCoreServicesConfig = Pick<
| 'CDP_WATCHER_OBSERVE_RESULTS_BUFFER_TIME_MS'
| 'CDP_WATCHER_OBSERVE_RESULTS_BUFFER_MAX_RESULTS'
| 'CDP_HOGFLOW_AWAITED_STEPS_ENABLED'
+ | 'CDP_INTERNAL_LOGS_SOURCES'
| 'SES_ACCESS_KEY_ID'
| 'SES_SECRET_ACCESS_KEY'
| 'SES_REGION'
@@ -200,7 +202,8 @@ export type CdpCoreServicesConfig = Pick<
| 'CDP_DLQ_BATCH_FAIL_RATIO'
| 'CDP_EVENTS_DLQ_TOPIC'
| 'CDP_EVENTS_DLQ_PRODUCER'
- >
+ > &
+ InternalLogsProducerConfig
export interface CdpCoreServicesDeps {
postgres: PostgresRouter
@@ -571,7 +574,16 @@ export function createCdpCoreServices(
{ awaitedStepsEnabled: config.CDP_HOGFLOW_AWAITED_STEPS_ENABLED }
)
- const hogFunctionMonitoringService = new HogFunctionMonitoringService(outputs)
+ const internalLogsSources = new Set(
+ config.CDP_INTERNAL_LOGS_SOURCES.split(',')
+ .map((source) => source.trim())
+ .filter(Boolean)
+ )
+ const hogFunctionMonitoringService = new HogFunctionMonitoringService(
+ outputs,
+ internalLogsSources.size > 0 ? new InternalLogsProducer(config, deps.teamManager) : undefined,
+ internalLogsSources
+ )
const hogInvocationResultsService = new HogInvocationResultsService(outputs, config)
const warehouseWebhooksService = new WarehouseWebhooksService(outputs)
const capturedEventsService = new CapturedEventsService(deps.internalCaptureService, deps.teamManager)
diff --git a/nodejs/src/cdp/config.ts b/nodejs/src/cdp/config.ts
index 6d798251fc7d..ac96bee83881 100644
--- a/nodejs/src/cdp/config.ts
+++ b/nodejs/src/cdp/config.ts
@@ -67,6 +67,8 @@ export type CdpConfig = ClickhouseConfig & {
CDP_CYCLOTRON_COMPRESS_KAFKA_DATA: boolean
// Off until Django emits `$workflow_step_resume`, or parked steps never wake.
CDP_HOGFLOW_AWAITED_STEPS_ENABLED: boolean
+ // Comma-separated log sources, such as `hog_flow`, that also send their logs to the logs product. Empty sends none.
+ CDP_INTERNAL_LOGS_SOURCES: string
// Django produces cdp_internal_events to the cyclotron cluster; empty inherits the consumer default.
CDP_INTERNAL_EVENTS_CONSUMER_METADATA_BROKER_LIST: string
CDP_REDIS_HOST: string
@@ -319,6 +321,7 @@ export function getDefaultCdpConfig(): CdpConfig {
CDP_CYCLOTRON_USE_BULK_COPY_JOB: isProdEnv() ? false : true,
CDP_CYCLOTRON_COMPRESS_KAFKA_DATA: true,
CDP_HOGFLOW_AWAITED_STEPS_ENABLED: isProdEnv() ? false : true,
+ CDP_INTERNAL_LOGS_SOURCES: '',
CDP_INTERNAL_EVENTS_CONSUMER_METADATA_BROKER_LIST: '',
CDP_REDIS_HOST: '127.0.0.1',
CDP_REDIS_PORT: 6379,
diff --git a/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.test.ts b/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.test.ts
index 8b850fba40d2..44c5073a3609 100644
--- a/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.test.ts
+++ b/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.test.ts
@@ -1,28 +1,36 @@
+import { DateTime } from 'luxon'
+
import { APP_METRICS_OUTPUT, LOG_ENTRIES_OUTPUT } from '~/common/outputs'
import { IngestionOutputs } from '~/common/outputs/ingestion-outputs'
import { parseJSON } from '~/common/utils/json-parse'
+import { internalFetch } from '~/common/utils/request'
+import { InternalLogsProducer } from '~/logs/internal-logs-producer'
-import { MetricLogSource, MinimalAppMetric } from '../../types'
+import { LogEntry, MetricLogSource, MinimalAppMetric } from '../../types'
import { HogFunctionMonitoringService, MonitoringOutput } from './hog-function-monitoring.service'
+jest.mock('~/common/utils/request', () => ({ internalFetch: jest.fn() }))
+
describe('HogFunctionMonitoringService', () => {
let service: HogFunctionMonitoringService
let queueMessagesMock: jest.Mock<Promise<void>, [any[]]>
+ let outputs: IngestionOutputs<MonitoringOutput>
+ let produceMock: jest.Mock
beforeEach(() => {
queueMessagesMock = jest.fn().mockResolvedValue(undefined)
+ produceMock = jest.fn().mockResolvedValue(undefined)
const fakeOutput = {
- produce: jest.fn().mockResolvedValue(undefined),
+ produce: produceMock,
queueMessages: queueMessagesMock,
checkHealth: jest.fn().mockResolvedValue(undefined),
checkTopicExists: jest.fn().mockResolvedValue(undefined),
} as any
- service = new HogFunctionMonitoringService(
- new IngestionOutputs<MonitoringOutput>({
- [APP_METRICS_OUTPUT]: fakeOutput,
- [LOG_ENTRIES_OUTPUT]: fakeOutput,
- })
- )
+ outputs = new IngestionOutputs<MonitoringOutput>({
+ [APP_METRICS_OUTPUT]: fakeOutput,
+ [LOG_ENTRIES_OUTPUT]: fakeOutput,
+ })
+ service = new HogFunctionMonitoringService(outputs)
})
async function flushedRows(): Promise<Record<string, unknown>[]> {
@@ -115,4 +123,69 @@ describe('HogFunctionMonitoringService', () => {
expect.objectContaining({ app_source_id: 'flow-1/3', count: 2 }),
])
})
+
+ const internalLogsProducer = (): InternalLogsProducer =>
+ new InternalLogsProducer(
+ {
+ LOGS_INTERNAL_INGESTION_SIGNING_KEYS: 'key',
+ LOGS_INTERNAL_PRODUCER_URL: 'http://capture-logs.test/i/v1/logs',
+ LOGS_INTERNAL_PRODUCER_TEAMS: '2',
+ LOGS_INTERNAL_PRODUCER_MAX_IN_FLIGHT: 20,
+ LOGS_INTERNAL_PRODUCER_MAX_BUFFERED_BYTES: 1024 * 1024,
+ LOGS_INTERNAL_PRODUCER_RETRY_TIMEOUT_MS: 1_000,
+ },
+ { getTeam: (teamId: number) => Promise.resolve({ id: teamId, api_token: `phc_${teamId}` } as any) }
+ )
+
+ const log = (teamId: number): LogEntry => ({
+ team_id: teamId,
+ log_source: 'hog_flow',
+ log_source_id: 'flow-1',
+ instance_id: 'run-1',
+ timestamp: DateTime.fromISO('2026-10-27T12:00:00Z'),
+ level: 'info',
+ message: `team ${teamId}`,
+ })
+
+ it('sends accepted logs to the logs product with the log entry timestamps, and drains them on stop', async () => {
+ const fetchMock = jest.mocked(internalFetch).mockResolvedValue({ status: 200, dump: jest.fn() } as any)
+ service = new HogFunctionMonitoringService(
+ outputs,
+ internalLogsProducer(),
+ new Set(['hog_flow', 'hog_function'])
+ )
+ service.queueLogs([log(2), log(2), log(3)], 'hog_flow')
+ service.queueLogs([log(2)], 'hog_function')
+
+ await service.flush()
+ await new Promise((resolve) => setImmediate(resolve))
+
+ expect(produceMock).toHaveBeenCalledTimes(4)
+ expect(fetchMock).toHaveBeenCalledTimes(1)
+ const [, options] = fetchMock.mock.calls[0]
+ const body = parseJSON(options!.body as string) as any
+ const records = body.resourceLogs[0].scopeLogs[0].logRecords
+ expect(records.map((r: any) => r.body.stringValue)).toEqual(['team 2', 'team 2'])
+ const entryMillis = produceMock.mock.calls
+ .map(([message]) => parseJSON(message.value.toString()) as Record<string, unknown>)
+ .filter((row) => row.team_id === 2 && row.log_source === 'hog_flow')
+ .map((row) => DateTime.fromSQL(row.timestamp as string, { zone: 'utc' }).toMillis())
+ expect(records.map((r: any) => Number(BigInt(r.timeUnixNano) / 1_000_000n))).toEqual(entryMillis)
+
+ service.queueLogs([log(2)], 'hog_flow')
+ await service.stop()
+ expect(fetchMock).toHaveBeenCalledTimes(2)
+ })
+
+ it('sends nothing to the logs product for a source not in CDP_INTERNAL_LOGS_SOURCES', async () => {
+ const fetchMock = jest.mocked(internalFetch).mockResolvedValue({ status: 200, dump: jest.fn() } as any)
+ service = new HogFunctionMonitoringService(outputs, internalLogsProducer(), new Set(['hog_function']))
+
+ service.queueLogs([log(2)], 'hog_flow')
+ await service.flush()
+ await service.stop()
+
+ expect(produceMock).toHaveBeenCalledTimes(1)
+ expect(fetchMock).not.toHaveBeenCalled()
+ })
})
diff --git a/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.ts b/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.ts
index 8e58ec6ca1ae..07b4f53b505f 100644
--- a/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.ts
+++ b/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.ts
@@ -6,6 +6,7 @@ import { AppMetricsAggregator } from '~/common/services/app-metrics-aggregator'
import { safeClickhouseString } from '~/common/utils/db/utils'
import { logger } from '~/common/utils/logger'
import { captureException } from '~/common/utils/posthog'
+import type { InternalLogsProducer } from '~/logs/internal-logs-producer'
import {
CyclotronJobInvocationHogFlow,
@@ -69,19 +70,26 @@ export const isHogFlowResult = (
return 'hogFlow' in result.invocation
}
+// Bounded so that shutdown still fits inside the pod's termination grace period.
+const INTERNAL_LOGS_DRAIN_TIMEOUT_MS = 10_000
+
export class HogFunctionMonitoringService {
queuedLogMessages: LogEntrySerialized[] = []
private appMetricsAggregator: AppMetricsAggregator
- constructor(private outputs: IngestionOutputs<MonitoringOutput>) {
+ constructor(
+ private outputs: IngestionOutputs<MonitoringOutput>,
+ private internalLogsProducer?: InternalLogsProducer,
+ private internalLogsSources: ReadonlySet<string> = new Set()
+ ) {
this.appMetricsAggregator = new AppMetricsAggregator(outputs)
}
- // Required by ResultSink. This sink holds nothing of its own: it writes through the shared
- // Kafka outputs, which the server disconnects separately on shutdown.
+ // Required by ResultSink. This sink writes through the shared Kafka outputs, which the server
+ // disconnects separately on shutdown.
async stop(): Promise<void> {
- return Promise.resolve()
+ await this.internalLogsProducer?.drain(INTERNAL_LOGS_DRAIN_TIMEOUT_MS)
}
async flush() {
@@ -90,6 +98,14 @@ export class HogFunctionMonitoringService {
this.queuedLogMessages = []
hogFunctionMonitoringPendingMessages.set(0)
+ // Only starts the sends. Delivery runs in the background, so a slow capture-logs cannot hold up the Kafka flush.
+ // The catch makes sure a producer bug can never stop the log_entries write below.
+ try {
+ this.internalLogsProducer?.flush()
+ } catch (error) {
+ captureException(error)
+ }
+
await Promise.all([
this.appMetricsAggregator.flush().catch((error) => {
// Best-effort — don't disrupt processing just for metrics.
@@ -151,17 +167,21 @@ export class HogFunctionMonitoringService {
}
queueLogs(logEntries: LogEntry[], source: MetricLogSource) {
- const logs = fixLogDeduplication(
- logEntries.map((logEntry) => ({
- ...logEntry,
- log_source: source,
- }))
- )
+ const sourcedLogEntries = logEntries.map((logEntry) => ({
+ ...logEntry,
+ log_source: source,
+ }))
+ // Sorts its input and moves equal timestamps apart in place, so `sourcedLogEntries` carries the same timestamps as `logs`.
+ const logs = fixLogDeduplication(sourcedLogEntries)
logs.forEach((logEntry) => {
this.queuedLogMessages.push(logEntry)
})
hogFunctionMonitoringPendingMessages.set(this.queuedLogMessages.length)
+
+ if (this.internalLogsSources.has(source)) {
+ this.internalLogsProducer?.enqueue(source, sourcedLogEntries)
+ }
}
queueInvocationResults(results: CyclotronJobInvocationResult[]): void {
diff --git a/nodejs/src/cdp/utils.ts b/nodejs/src/cdp/utils.ts
index 78401817b4b8..3b0d9d422f2e 100644
--- a/nodejs/src/cdp/utils.ts
+++ b/nodejs/src/cdp/utils.ts
@@ -275,6 +275,8 @@ export const fixLogDeduplication = (logs: LogEntry[]): LogEntrySerialized[] => {
sortedLogs.forEach((logEntry) => {
// TRICKY: The clickhouse table dedupes logs with the same timestamp - we need to ensure they are unique by simply plus-ing 1ms
// if the timestamp is the same as the previous one
+ // Changes the caller's entry in place, not only the copy below. HogFunctionMonitoringService relies on this to
+ // send the same timestamps to the logs product, and its test fails by 1 ms if this stops.
if (logEntry.timestamp <= previousTimestamp) {
logEntry.timestamp = previousTimestamp.plus(1)
}