Skip to content

:hedgehog: PostHog is the leading platform for building self-driving products. Our developer tools – AI observability, analytics, session replay, flags, experiments, error tracking, logs, and more – capture all the context agents need to diagnose problems, uncover opportunities, and ship fixes. Steer it all from Slack, web, desktop, or the MCP.

8.4 KB
Raw Blame
import { GroupReadRepository } from '~/common/groups/repositories/group-repository.interface'
import { RedisV2 } from '~/common/redis/redis-v2'
import { QuotaLimiting } from '~/common/services/quota-limiting.service'
import { GeoIPService } from '~/common/utils/geoip'
import { logger } from '~/common/utils/logger'

import type { CommonConfig } from '../../common/config'
import { HealthCheckResult, PluginServerService, Team, TeamId } from '../../types'
import {
    CdpCoreServicesConfig,
    CdpCoreServicesDeps,
    CdpOutputs,
    CdpValkeyShadowPools,
    createCdpCoreServices,
} from '../cdp-services'
import type { CdpConfig } from '../config'
import { HogFlow } from '../schema/hogflow'
import { HogExecutorAsyncService } from '../services/hog-executor-async.service'
import { HogInputsService } from '../services/hog-inputs.service'
import { HogFlowExecutorService } from '../services/hogflows/hogflow-executor.service'
import { HogFlowFunctionsService } from '../services/hogflows/hogflow-functions.service'
import { HogFlowManagerService } from '../services/hogflows/hogflow-manager.service'
import { InvocationResultsService } from '../services/invocation-results.service'
import { LegacyPluginExecutorService } from '../services/legacy-plugin-executor.service'
import { GroupsManagerService } from '../services/managers/groups-manager.service'
import { HogFunctionManagerService } from '../services/managers/hog-function-manager.service'
import { HogFunctionTemplateManagerService } from '../services/managers/hog-function-template-manager.service'
import { PersonsManagerService } from '../services/managers/persons-manager.service'
import { RecipientsManagerService } from '../services/managers/recipients-manager.service'
import { EmailAttachmentsService } from '../services/messaging/email-attachments.service'
import { EmailService } from '../services/messaging/email.service'
import { RecipientPreferencesService } from '../services/messaging/recipient-preferences.service'
import { HogFunctionMonitoringService } from '../services/monitoring/hog-function-monitoring.service'
import { HogMaskerService } from '../services/monitoring/hog-masker.service'
import { HogWatcherService } from '../services/monitoring/hog-watcher.service'
import { WorkflowsActivationReporter } from '../services/monitoring/workflows-activation-reporter'
import { NativeDestinationExecutorService } from '../services/native-destination-executor.service'
import { SegmentDestinationExecutorService } from '../services/segment-destination-executor.service'
import { CdpUsageReporterService } from '../services/usage/cdp-usage-reporter.service'
import { HogFunctionType, HogFunctionTypeType } from '../types'

export type TeamWork = { team: Team; hogFunctions: HogFunctionType[]; hogFlows: HogFlow[] }

export type CdpConsumerBaseConfig = CdpCoreServicesConfig &
    Pick<CommonConfig, 'KAFKA_CLIENT_RACK'> &
    Pick<CdpConfig, 'CDP_OVERFLOW_QUEUE_ENABLED'>

export interface CdpConsumerBaseDeps extends CdpCoreServicesDeps {
    geoipService: GeoIPService
    groupRepository: GroupReadRepository
    quotaLimiting: QuotaLimiting
}

export interface TeamIDWithConfig {
    teamId: TeamId | null
    consoleLogIngestionEnabled: boolean
}

export abstract class CdpConsumerBase<TConfig extends CdpConsumerBaseConfig = CdpConsumerBaseConfig> {
    redis: RedisV2
    valkeyShadow: CdpValkeyShadowPools
    isStopping = false

    hogExecutorAsync: HogExecutorAsyncService
    hogInputsService: HogInputsService
    hogFlowExecutor: HogFlowExecutorService
    hogMasker: HogMaskerService
    hogWatcher: HogWatcherService
    hogWatcherMirror: HogWatcherService

    groupsManager: GroupsManagerService
    hogFlowManager: HogFlowManagerService
    hogFunctionManager: HogFunctionManagerService
    hogFunctionTemplateManager: HogFunctionTemplateManagerService
    hogFlowFunctionsService: HogFlowFunctionsService
    personsManager: PersonsManagerService
    recipientsManager: RecipientsManagerService

    emailService: EmailService
    emailAttachmentsService: EmailAttachmentsService
    hogFunctionMonitoringService: HogFunctionMonitoringService
    workflowsActivationReporter: WorkflowsActivationReporter
    cdpUsageReporter: CdpUsageReporterService
    invocationResultsService: InvocationResultsService
    nativeDestinationExecutorService: NativeDestinationExecutorService
    pluginDestinationExecutorService: LegacyPluginExecutorService
    recipientPreferencesService: RecipientPreferencesService
    segmentDestinationExecutorService: SegmentDestinationExecutorService

    protected outputs: CdpOutputs
    protected abstract name: string

    constructor(
        protected config: TConfig,
        protected deps: CdpConsumerBaseDeps
    ) {
        const services = createCdpCoreServices(config, deps)

        this.redis = services.redis
        this.valkeyShadow = services.valkeyShadow
        this.hogFunctionManager = services.hogFunctionManager
        this.hogFlowManager = services.hogFlowManager
        this.hogWatcher = services.hogWatcher
        this.hogWatcherMirror = services.hogWatcherMirror
        this.hogExecutorAsync = services.hogExecutorAsync
        this.hogInputsService = services.hogInputsService
        this.hogFunctionTemplateManager = services.hogFunctionTemplateManager
        this.hogFlowFunctionsService = services.hogFlowFunctionsService
        this.personsManager = services.personsManager
        this.recipientsManager = services.recipientsManager
        this.recipientPreferencesService = services.recipientPreferencesService
        this.hogFlowExecutor = services.hogFlowExecutor
        this.emailService = services.emailService
        this.emailAttachmentsService = services.emailAttachmentsService
        this.hogFunctionMonitoringService = services.hogFunctionMonitoringService
        this.workflowsActivationReporter = services.workflowsActivationReporter
        this.cdpUsageReporter = services.cdpUsageReporter
        this.invocationResultsService = services.invocationResultsService
        this.nativeDestinationExecutorService = services.nativeDestinationExecutorService
        this.segmentDestinationExecutorService = services.segmentDestinationExecutorService
        this.outputs = services.outputs

        // Base-only services
        this.hogMasker = new HogMaskerService(services.redis, services.valkeyShadow.writer)
        this.groupsManager = new GroupsManagerService(deps.teamManager, deps.groupRepository)
        this.pluginDestinationExecutorService = new LegacyPluginExecutorService(deps.postgres, deps.geoipService)
    }

    public get service(): PluginServerService {
        return {
            id: this.name,
            onShutdown: async () => await this.stop(),
            healthcheck: () => this.isHealthy(),
        }
    }

    /** The team, or null when it has no hog function of these types and no hog flow. */
    protected async loadTeamWithHogFunctionsOrFlows(
        teamId: TeamId,
        hogTypes: HogFunctionTypeType[]
    ): Promise<Team | null> {
        return (await this.loadTeamWork(teamId, hogTypes))?.team ?? null
    }

    protected async loadTeamWork(teamId: TeamId, hogTypes: HogFunctionTypeType[]): Promise<TeamWork | null> {
        const [hogFunctions, hogFlows, team] = await Promise.all([
            this.hogFunctionManager.getHogFunctionsForTeam(teamId, hogTypes),
            this.hogFlowManager.getHogFlowsForTeam(teamId),
            this.deps.teamManager.getTeam(teamId),
        ])
        return team && (hogFunctions.length || hogFlows.length) ? { team, hogFunctions, hogFlows } : null
    }

    public async start(): Promise<void> {
        // Outputs are resolved in the constructor via `createCdpCoreServices` — no
        // per-consumer producer lifecycle. The outer server owns producer shutdown
        // through `cdpProducerRegistry.disconnectAll()`.
    }

    public async stop(): Promise<void> {
        logger.info('🔁', `${this.name} - stopping`)
        this.isStopping = true
        // Billing records live only in memory until they are sent, so a graceful stop drains them.
        await this.cdpUsageReporter.shutdown()
        // Release the per-consumer conversion-watcher pool created in createCdpCoreServices, so
        // graceful shutdown frees its cyclotron connections like the other cyclotron pools do.
        await this.invocationResultsService.stop()
        await this.emailAttachmentsService.stop()
        logger.info('👍', `${this.name} - stopped!`)
    }

    public abstract isHealthy(): HealthCheckResult
}