:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.
Python ab-testingai-analyticsanalyticscdpdata-warehouseexperimentsfeature-flagsjavascriptproduct-analyticspythonreactsession-replaysurveystypescriptweb-analytics
feat(logs): send hog transformation logs to the logs product behind a switch (#129305)
Jon McCalluma668ad7f9ec6
diff --git a/nodejs/src/cdp/cdp-services.ts b/nodejs/src/cdp/cdp-services.tsindex a58385794e57..5d0c0b9e1f5e 100644--- a/nodejs/src/cdp/cdp-services.ts+++ b/nodejs/src/cdp/cdp-services.ts@@ -48,7 +48,10 @@ import { MessageAssetsService } from './services/messaging/message-assets.servicimport { PushNotificationService } from './services/messaging/push-notification.service'import { RecipientPreferencesService } from './services/messaging/recipient-preferences.service'import { RecipientTokensService } from './services/messaging/recipient-tokens.service'-import { HogFunctionMonitoringService } from './services/monitoring/hog-function-monitoring.service'+import {+ HogFunctionMonitoringService,+ parseInternalLogsSources,+} from './services/monitoring/hog-function-monitoring.service'import { HogInvocationResultsService } from './services/monitoring/hog-invocation-results.service'import { HogWatcherService } from './services/monitoring/hog-watcher.service'import { WorkflowsActivationReporter } from './services/monitoring/workflows-activation-reporter'@@ -592,11 +595,7 @@ export function createCdpCoreServices({ awaitedStepsEnabled: config.CDP_HOGFLOW_AWAITED_STEPS_ENABLED })- const internalLogsSources = new Set(- config.CDP_INTERNAL_LOGS_SOURCES.split(',')- .map((source) => source.trim())- .filter(Boolean)- )+ const internalLogsSources = parseInternalLogsSources(config.CDP_INTERNAL_LOGS_SOURCES)const hogFunctionMonitoringService = new HogFunctionMonitoringService(outputs,internalLogsSources.size > 0 ? new InternalLogsProducer(config, deps.teamManager) : undefined,diff --git a/nodejs/src/cdp/hog-transformations/hog-transformer.service.test.ts b/nodejs/src/cdp/hog-transformations/hog-transformer.service.test.tsindex 3e82d615931d..80a25bb13cd9 100644--- a/nodejs/src/cdp/hog-transformations/hog-transformer.service.test.ts+++ b/nodejs/src/cdp/hog-transformations/hog-transformer.service.test.ts@@ -81,6 +81,13 @@ describe('HogTransformer', () => {// Deliberately not the default, so the assertion below proves the value is plumbed// through rather than hardcoded in the factory.TRANSFORMATIONS_HOG_TIMEOUT_MS: 123,+ CDP_INTERNAL_LOGS_SOURCES: '',+ LOGS_INTERNAL_INGESTION_SIGNING_KEYS: '',+ LOGS_INTERNAL_PRODUCER_URL: '',+ LOGS_INTERNAL_PRODUCER_TEAMS: '',+ LOGS_INTERNAL_PRODUCER_MAX_IN_FLIGHT: 20,+ LOGS_INTERNAL_PRODUCER_MAX_BUFFERED_BYTES: 1024 * 1024,+ LOGS_INTERNAL_PRODUCER_RETRY_TIMEOUT_MS: 1_000,}const transformer = createHogTransformerService(config, {...hub,diff --git a/nodejs/src/cdp/hog-transformations/hog-transformer.service.ts b/nodejs/src/cdp/hog-transformations/hog-transformer.service.tsindex b7f9c5f35cab..64071bda86c0 100644--- a/nodejs/src/cdp/hog-transformations/hog-transformer.service.ts+++ b/nodejs/src/cdp/hog-transformations/hog-transformer.service.ts@@ -7,6 +7,7 @@ import { PostgresRouter } from '~/common/utils/db/postgres'import { GeoIPService, GeoIp } from '~/common/utils/geoip'import { logger } from '~/common/utils/logger'import { PubSub } from '~/common/utils/pubsub'+import type { TeamManager } from '~/common/utils/team-manager'import { PluginEvent, Properties } from '~/plugin-scaffold'import {@@ -22,7 +23,12 @@ import { HogInputsService } from '../services/hog-inputs.service'import { LegacyPluginExecutorService } from '../services/legacy-plugin-executor.service'import { HogFunctionManagerService } from '../services/managers/hog-function-manager.service'import { IntegrationManagerService } from '../services/managers/integration-manager.service'-import { HogFunctionMonitoringService, MonitoringOutput } from '../services/monitoring/hog-function-monitoring.service'+import {+ HogFunctionMonitoringService,+ MonitoringOutput,+ TransformationsMonitoringConfig,+ createTransformationsMonitoringService,+} from '../services/monitoring/hog-function-monitoring.service'import { EncryptedFields } from '../utils/encryption-utils'import { HogConfigurationError } from '../utils/hog-error-classification'import { convertToHogFunctionFilterGlobal, filterFunctionInstrumented } from '../utils/hog-function-filtering'@@ -134,6 +140,7 @@ export class HogTransformerService implements HogTransformer {public async stop(): Promise<void> {await this.processInvocationResults()+ await this.hogFunctionMonitoringService.stop()}public async processInvocationResults(): Promise<void> {@@ -417,7 +424,8 @@ export type HogTransformerServiceConfig = Pick<| 'CDP_HOG_RUST_VM_BATCH_EXECUTION_ENABLED'| 'MMDB_FILE_LOCATION'| 'TRANSFORMATIONS_HOG_TIMEOUT_MS'->+> &+ TransformationsMonitoringConfigexport interface HogTransformerServiceDeps {geoipService: GeoIPService@@ -426,6 +434,7 @@ export interface HogTransformerServiceDeps {encryptedFields: EncryptedFieldsintegrationManager: IntegrationManagerServicemonitoringOutputs: IngestionOutputs<MonitoringOutput>+ teamManager: Pick<TeamManager, 'getTeam'>}/**@@ -444,7 +453,11 @@ export function createHogTransformerService(hogInputsService)const pluginExecutor = new LegacyPluginExecutorService(deps.postgres, deps.geoipService)- const hogFunctionMonitoringService = new HogFunctionMonitoringService(deps.monitoringOutputs)+ const hogFunctionMonitoringService = createTransformationsMonitoringService(+ deps.monitoringOutputs,+ config,+ deps.teamManager+ )return new HogTransformerService(hogFunctionManager,hogExecutor,diff --git a/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.test.ts b/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.test.tsindex c6299fef4cec..e7d0b0aa7def 100644--- a/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.test.ts+++ b/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.test.ts@@ -7,7 +7,11 @@ import { internalFetch } from '~/common/utils/request'import { InternalLogsProducer } from '~/logs/internal-logs-producer'import { LogEntry, MetricLogSource, MinimalAppMetric } from '../../types'-import { HogFunctionMonitoringService, MonitoringOutput } from './hog-function-monitoring.service'+import {+ HogFunctionMonitoringService,+ MonitoringOutput,+ createTransformationsMonitoringService,+} from './hog-function-monitoring.service'jest.mock('~/common/utils/request', () => ({ internalFetch: jest.fn() }))@@ -124,18 +128,18 @@ describe('HogFunctionMonitoringService', () => {])})- const internalLogsProducer = (): InternalLogsProducer =>- new InternalLogsProducer(- {- LOGS_INTERNAL_INGESTION_SIGNING_KEYS: 'key',- LOGS_INTERNAL_PRODUCER_URL: 'http://capture-logs.test/i/v1/logs',- LOGS_INTERNAL_PRODUCER_TEAMS: '2',- LOGS_INTERNAL_PRODUCER_MAX_IN_FLIGHT: 20,- LOGS_INTERNAL_PRODUCER_MAX_BUFFERED_BYTES: 1024 * 1024,- LOGS_INTERNAL_PRODUCER_RETRY_TIMEOUT_MS: 1_000,- },- { getTeam: (teamId: number) => Promise.resolve({ id: teamId, api_token: `phc_${teamId}` } as any) }- )+ const producerConfig = {+ LOGS_INTERNAL_INGESTION_SIGNING_KEYS: 'key',+ LOGS_INTERNAL_PRODUCER_URL: 'http://capture-logs.test/i/v1/logs',+ LOGS_INTERNAL_PRODUCER_TEAMS: '2',+ LOGS_INTERNAL_PRODUCER_MAX_IN_FLIGHT: 20,+ LOGS_INTERNAL_PRODUCER_MAX_BUFFERED_BYTES: 1024 * 1024,+ LOGS_INTERNAL_PRODUCER_RETRY_TIMEOUT_MS: 1_000,+ }+ const teamManager = {+ getTeam: (teamId: number) => Promise.resolve({ id: teamId, api_token: `phc_${teamId}` } as any),+ }+ const internalLogsProducer = (): InternalLogsProducer => new InternalLogsProducer(producerConfig, teamManager)const log = (teamId: number): LogEntry => ({team_id: teamId,@@ -184,4 +188,23 @@ describe('HogFunctionMonitoringService', () => {expect(produceMock).toHaveBeenCalledTimes(1)expect(fetchMock).not.toHaveBeenCalled()})++ it.each([+ ['hog_flow,hog_function', 0],+ ['hog_flow,hog_function_transformations', 1],+ ])('sends transformation logs only when the transformations switch is on (%s)', async (sources, sends) => {+ const fetchMock = jest.mocked(internalFetch).mockResolvedValue({ status: 200, dump: jest.fn() } as any)+ service = createTransformationsMonitoringService(+ outputs,+ { ...producerConfig, CDP_INTERNAL_LOGS_SOURCES: sources },+ teamManager+ )++ service.queueLogs([{ ...log(2), log_source: 'hog_function' }], 'hog_function')+ await service.flush()+ await service.stop()++ expect(produceMock).toHaveBeenCalledTimes(1)+ expect(fetchMock).toHaveBeenCalledTimes(sends)+ })})diff --git a/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.ts b/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.tsindex 07b4f53b505f..4e96a01c9771 100644--- a/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.ts+++ b/nodejs/src/cdp/services/monitoring/hog-function-monitoring.service.ts@@ -6,8 +6,10 @@ import { AppMetricsAggregator } from '~/common/services/app-metrics-aggregator'import { safeClickhouseString } from '~/common/utils/db/utils'import { logger } from '~/common/utils/logger'import { captureException } from '~/common/utils/posthog'-import type { InternalLogsProducer } from '~/logs/internal-logs-producer'+import type { TeamManager } from '~/common/utils/team-manager'+import { InternalLogsProducer, InternalLogsProducerConfig } from '~/logs/internal-logs-producer'+import type { CdpConfig } from '../../config'import {CyclotronJobInvocationHogFlow,CyclotronJobInvocationHogFunction,@@ -73,6 +75,36 @@ export const isHogFlowResult = (// Bounded so that shutdown still fits inside the pod's termination grace period.const INTERNAL_LOGS_DRAIN_TIMEOUT_MS = 10_000+// A switch token of its own, so ingestion turns on apart from the CDP workers that `hog_function` covers.+const HOG_FUNCTION_TRANSFORMATIONS_SWITCH = 'hog_function_transformations'++export function parseInternalLogsSources(value: string): Set<string> {+ return new Set(+ value+ .split(',')+ .map((source) => source.trim())+ .filter(Boolean)+ )+}++export type TransformationsMonitoringConfig = Pick<CdpConfig, 'CDP_INTERNAL_LOGS_SOURCES'> & InternalLogsProducerConfig++/** Monitoring for hog transformations. Their logs are stored under the `hog_function` source. */+export function createTransformationsMonitoringService(+ outputs: IngestionOutputs<MonitoringOutput>,+ config: TransformationsMonitoringConfig,+ teamManager: Pick<TeamManager, 'getTeam'>+): HogFunctionMonitoringService {+ const enabled = parseInternalLogsSources(config.CDP_INTERNAL_LOGS_SOURCES).has(HOG_FUNCTION_TRANSFORMATIONS_SWITCH)+ return enabled+ ? new HogFunctionMonitoringService(+ outputs,+ new InternalLogsProducer(config, teamManager),+ new Set(['hog_function'])+ )+ : new HogFunctionMonitoringService(outputs)+}+export class HogFunctionMonitoringService {queuedLogMessages: LogEntrySerialized[] = []diff --git a/nodejs/src/servers/error-tracking-server.ts b/nodejs/src/servers/error-tracking-server.tsindex 9548849bc5be..322151882ca9 100644--- a/nodejs/src/servers/error-tracking-server.ts+++ b/nodejs/src/servers/error-tracking-server.ts@@ -202,6 +202,7 @@ export class ErrorTrackingServer implements NodeServer {encryptedFields,integrationManager,monitoringOutputs: outputs,+ teamManager: sharedServices.container.teamManager,}const errorTrackingSharedScope = extend(sharedServicesScope, 'errortracking-shared', (_container, builder) =>diff --git a/nodejs/src/servers/ingestion-api-server.ts b/nodejs/src/servers/ingestion-api-server.tsindex ccdbe623f36d..b5bdd32f85bd 100644--- a/nodejs/src/servers/ingestion-api-server.ts+++ b/nodejs/src/servers/ingestion-api-server.ts@@ -297,6 +297,7 @@ export class IngestionApiServer implements NodeServer {encryptedFields,integrationManager,monitoringOutputs: ingestionOutputs,+ teamManager,}this.hogTransformer = createHogTransformerService(this.config, hogTransformerDeps)await this.hogTransformer.start()diff --git a/nodejs/src/servers/ingestion-general-server.ts b/nodejs/src/servers/ingestion-general-server.tsindex e80bb178dffe..587c9013926f 100644--- a/nodejs/src/servers/ingestion-general-server.ts+++ b/nodejs/src/servers/ingestion-general-server.ts@@ -271,6 +271,7 @@ export class IngestionGeneralServer implements NodeServer {encryptedFields,integrationManager,monitoringOutputs: ingestionOutputs,+ teamManager,}const ingestionDeps: IngestionConsumerDeps = {diff --git a/nodejs/tests/ingestion/ingestion-e2e.test.ts b/nodejs/tests/ingestion/ingestion-e2e.test.tsindex 32e2764c3fe4..efae8072f152 100644--- a/nodejs/tests/ingestion/ingestion-e2e.test.ts+++ b/nodejs/tests/ingestion/ingestion-e2e.test.ts@@ -78,6 +78,7 @@ describe.each([encryptedFields: infra.encryptedFields,integrationManager: infra.integrationManager,monitoringOutputs: createTestMonitoringOutputs(kafkaProducer),+ teamManager: infra.teamManager,}),})jest.spyOn(infra.groupRepository, 'fetchGroup')diff --git a/nodejs/tests/ingestion/person-updates-e2e.test.ts b/nodejs/tests/ingestion/person-updates-e2e.test.tsindex bad28af6c76e..d4c27db31a61 100644--- a/nodejs/tests/ingestion/person-updates-e2e.test.ts+++ b/nodejs/tests/ingestion/person-updates-e2e.test.ts@@ -87,6 +87,7 @@ describe.each(FLAG_COMBINATIONS)('Person Updates E2E ($#)', (config) => {encryptedFields: infra.encryptedFields,integrationManager: infra.integrationManager,monitoringOutputs: createTestMonitoringOutputs(kafkaProducer),+ teamManager: infra.teamManager,}),outputs,clickhouseGroupRepository: new ClickhouseGroupRepository(outputs),