Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

31.3 KB
Raw Blame
import { createServer } from 'http'
import os from 'os'
import { ActivepiecesError, isNil, spreadIfDefined, tryCatch, tryCatchSync } from '@activepieces/core-utils'
import { ACTION_RUN_CACHE_ACTIVE_WINDOW_MS, ACTION_RUN_CACHE_FIRST_SWEEP_DELAY_MS, ACTION_RUN_CACHE_SWEEP_INTERVAL_MS, actionRunCache, cacheUtils, createResolver, createSandboxRuntime, markDevPiecesRebuilt, Runtime } from '@activepieces/sandbox'
import { aiCostReporter, aiObservability, createLogger, modelCatalog, modelTierCatalog, systemUsage, wideEvent } from '@activepieces/server-utils'
import { ApEdition, ApiToWorkerContract, ConsumeJobRequest, createNotifyServer, createRpcClient, EngineResponseStatus, ExecutionMode, JobData, LONG_RUNNING_RPC_METHODS, SandboxInformation, WebsocketClientEvent, WebsocketServerEvent, WorkerJobType, WorkerMachineHealthcheckRequest, WorkerProps, WorkerSettingsResponse, WorkerToApiContract } from '@activepieces/shared'
import { nanoid } from 'nanoid'
import { io, Socket } from 'socket.io-client'
import { createApiToWorkerHandlers } from './api-notify-service'
import { getApiUrl, system, WorkerSystemProp } from './config/configs'
import { logger } from './config/logger'
import { workerSettings } from './config/worker-settings'
import { getHandler } from './execute/job-registry'
import { JobContext, JobResult, JobResultKind } from './execute/types'
import { sandboxConfig } from './runtime/sandbox-config'
import { VERSION_MISMATCH_POLL_PAUSE_MS, versionChecker } from './utils/version-checker'


let socket: Socket | null = null
let polling = false
let connectionGeneration = 0

const workerId = `worker-${nanoid()}`

const workerHostname = os.hostname()

let healthServerInstance: ReturnType<typeof createServer> | null = null

let shouldStartHealthServer = false

let runtime: Runtime | null = null

// Jobs executing across all poll loops. stop() waits for these to finish + report before tearing
// down, so a deploy doesn't orphan them in the app's BullMQ `active` list. Abrupt death (OOM/SIGKILL)
// still falls to the stalled-scan.
let inFlightJobs = 0

const DRAIN_TIMEOUT_MS = 25_000

// Sandbox memory/state is UI-only telemetry (the workers page). Computing it needs a full
// process-table scan (si.processes()), so it must NOT run on the poll hot path — sampling it on
// every poll pegged worker CPU and collapsed throughput (~7x, #13497). Sample it on a slow
// interval instead and have buildMachineInfo() read the cached snapshot.
let cachedSandboxInfo: SandboxInformation[] = []
let sandboxInfoInterval: NodeJS.Timeout | null = null
const SANDBOX_INFO_REFRESH_MS = 15_000
const SERVER_PING_TIMEOUT_MS = 5_000
const MACHINE_INFO_TIMEOUT_MS = 15_000
const POLL_LIVENESS_TIMEOUT_MS = 600_000
const POLL_WATCHDOG_INTERVAL_MS = 30_000
const RPC_TIMEOUT_MS = 60_000
const LONG_RUNNING_RPC_MARGIN_MS = 120_000
const FALLBACK_FLOW_TIMEOUT_SECONDS = 600

let pollLoopLiveness: PollLoopLiveness[] = []
let pollWatchdogInterval: NodeJS.Timeout | null = null

let cacheSweepInterval: NodeJS.Timeout | null = null
let cacheSweepFirstRunTimeout: NodeJS.Timeout | null = null

export const worker = {
    async start({ apiUrl, socketUrl, workerToken, withHealthServer = false }: WorkerStartParams): Promise<void> {
        versionChecker.assertReleaseReadable()
        modelCatalog.load().catch(() => undefined)
        await modelTierCatalog.warmUp()
        const workerGroupId = system.get(WorkerSystemProp.WORKER_GROUP_ID)
        const projectWorker = system.getBoolean(WorkerSystemProp.PROJECT_WORKER) ?? true
        const workerGroupScope = system.get(WorkerSystemProp.WORKER_GROUP_SCOPE)
        socket = io(socketUrl.url, {
            auth: { token: workerToken, workerId, workerGroupId, projectWorker, workerGroupScope },
            path: socketUrl.path,
            transports: ['websocket'],
            reconnection: true,
        })

        const apiClient = createRpcClient<WorkerToApiContract>(socket, rpcTimeoutMsFor)
        aiCostReporter.install({
            log: logger,
            report: (request) => apiClient.reportAiUsage(request),
            pageWebhookUrl: () => tryCatchSync(() => workerSettings.getSettings()).data?.PAGE_ONCALL_WEBHOOK,
        })

        socket.on('connect', async () => {
            logger.info('Connected to API server via Socket.IO')
            resetPollLoopLiveness({ loopCount: 0 })
            await fetchAndStoreSettings(socket!)
            void startPollingWorkers(apiClient).catch((err) => {
                logger.error({ error: err }, 'Polling workers crashed unexpectedly')
            })
        })

        socket.on('disconnect', (reason) => {
            connectionGeneration++
            polling = false
            logger.warn({ reason }, 'Disconnected from API server')
            // 'io client disconnect' is our own stop() — it already drained and tore down the runtime.
            // For any other reason the socket dropped while a job may still be running locally; the app
            // reclaims that job on disconnect, so kill the runtime now or the original keeps executing
            // to completion and double-runs the requeued copy. (The reconnect path recreates it.)
            // Also close the health server so orchestrators stop routing/keeping traffic on a worker
            // that can't consume jobs; the reconnect path brings it back up after prewarm completes.
            if (reason !== 'io client disconnect') {
                abortInFlightRuntime()
                stopHealthServer()
            }
            // Socket.IO does NOT auto-reconnect when the server initiates the disconnect
            // (reason 'io server disconnect' — e.g. the API process restarts/hot-reloads).
            // Without a manual reconnect the worker stays dead and silently stops consuming
            // jobs. Every other non-client reason already triggers built-in reconnection.
            if (reason === 'io server disconnect') {
                socket?.connect()
            }
        })

        socket.on('connect_error', (error) => {
            logger.error({ error: error.message }, 'Socket.IO connection error')
        })

        socket.on(WebsocketClientEvent.REFRESH_PIECE, () => {
            logger.info('Dev pieces rebuilt, sandboxes will be replaced before their next job')
            markDevPiecesRebuilt()
        })

        createNotifyServer<ApiToWorkerContract>(socket, createApiToWorkerHandlers({
            getRuntime: () => runtime,
            apiClient,
            getInternalApiUrl: getApiUrl,
            log: logger,
        }), logger)

        shouldStartHealthServer = withHealthServer
        startSandboxInfoSampling()
        startPollWatchdog()
        startCacheSweeper()
        logger.info({ apiUrl, socketUrl }, 'Worker started, polling for jobs...')
    },

    async stop(): Promise<void> {
        polling = false
        pollLoopLiveness = []
        stopPollWatchdog()
        stopSandboxInfoSampling()
        stopCacheSweeper()
        await drainInFlightJobs()
        if (runtime) {
            await runtime.shutdown(logger)
            runtime = null
        }
        socket?.disconnect()
        socket = null
        stopHealthServer()
        logger.info('Worker stopped')
    },

    stopPolling(): void {
        polling = false
    },
}

async function startPollingWorkers(apiClient: WorkerToApiContract): Promise<void> {
    if (polling) return
    polling = true

    const generation = connectionGeneration

    // The destination is one box per worker (concurrency 1), scaling out with replicas (ADR 0003).
    // Transitional compatibility mode (ADR 0004): honor AP_WORKER_CONCURRENCY=N by running N poll
    // loops over N in-process boxes, each routed by its workerIndex. The default (5, main's historical
    // value) is registered in configs.ts to preserve old behavior, so system.get always returns it; the
    // '1' below is an unreachable belt-and-suspenders fallback. At N>1 an OOM takes down all N in-flight
    // jobs, so the operator must size the container accordingly.
    const rawConcurrency = Number(system.get(WorkerSystemProp.WORKER_CONCURRENCY) ?? '1')
    const concurrency = Number.isInteger(rawConcurrency) && rawConcurrency > 0 ? rawConcurrency : 1
    if (!Number.isInteger(rawConcurrency) || rawConcurrency < 1) {
        logger.warn({ rawConcurrency }, 'Invalid AP_WORKER_CONCURRENCY value, falling back to 1')
    }
    // Bring up a fresh runtime on every (re)connect — a prior connection's in-flight job is killed
    // along with its box (usually already done by the disconnect handler), so it fails fast and is
    // retried instead of lingering on a reused box. Reusing the box made the next generation's poll
    // loop run a second operation on the same single-operation engine child (acquire() hands back a
    // busy sandbox) and let the lingering job's lock lapse during the reconnect → BullMQ "stalled";
    // a deploy disconnects every worker at once, so reuse turned that into mass stalls.
    abortInFlightRuntime()

    const createdRuntime = createSandboxRuntime({
        concurrency,
        basePath: sandboxConfig.getCacheBasePath(),
        getSettings: () => sandboxConfig.getSandboxSettings(),
    })
    runtime = createdRuntime

    // Opt-in via AP_PREWARM_CACHE_ON_STARTUP: the warm-up resolves and compiles every enabled flow,
    // so its startup memory/CPU spike grows with flow count and can OOM small workers on large instances.
    // When enabled it is awaited so the health server only comes up once the cache is warm and
    // orchestrators don't route/keep traffic on a cold worker; when disabled it is skipped entirely.
    if (system.getBoolean(WorkerSystemProp.PREWARM_CACHE_ON_STARTUP) ?? false) {
        while (polling && connectionGeneration === generation && !versionChecker.connectedAppVersionIsCompatible(logger)) {
            await sleep(VERSION_MISMATCH_POLL_PAUSE_MS)
        }
        if (polling && connectionGeneration === generation) {
            const { error: prewarmError } = await tryCatch(() => createdRuntime.prewarm({
                log: logger,
                apiClient,
                internalApiUrl: getApiUrl(),
            }))
            if (prewarmError) {
                logger.error({ error: prewarmError }, 'Prewarm failed, continuing without a warm cache')
            }
        }
    }

    // The generation check keeps a disconnect that landed mid-prewarm from re-opening the health
    // server for a connection that no longer exists — the reconnect's own run brings it back up.
    const stillCurrentConnection = polling && connectionGeneration === generation
    if (shouldStartHealthServer && stillCurrentConnection && isNil(healthServerInstance)) {
        healthServerInstance = startHealthServer()
    }

    logger.info({ concurrency }, 'Starting poll loops')

    resetPollLoopLiveness({ loopCount: concurrency })
    await Promise.all(Array.from({ length: concurrency }, (_, workerIndex) =>
        pollAndExecute(apiClient, createdRuntime, workerIndex, generation),
    ))
}

async function pollAndExecute(apiClient: WorkerToApiContract, runtime: Runtime, workerIndex: number, generation: number): Promise<void> {
    const workerLog = logger.child({ workerIndex })
    workerLog.info('Polling worker started')

    while (polling && connectionGeneration === generation) {
        markPollLoopIteration({ workerIndex, busy: false })
        if (!versionChecker.connectedAppVersionIsCompatible(workerLog)) {
            await sleep(VERSION_MISMATCH_POLL_PAUSE_MS)
            continue
        }

        const { data: machineInfo, error: machineError } = await tryCatch(buildMachineInfo)
        if (machineError) {
            workerLog.error({ error: machineError }, 'Failed to build machine info')
            await sleep(20000)
            continue
        }

        const { data: job, error: pollError } = await tryCatch(() => apiClient.poll(machineInfo))
        if (pollError) {
            workerLog.error({ error: pollError }, 'Poll failed')
            await sleep(25000)
            continue
        }

        if (!job) {
            workerLog.debug('Poll returned null, re-polling')
            continue
        }

        workerLog.debug({ job: { id: job.jobId, type: job.jobData.jobType } }, 'Job received from poll')

        // Counted for the duration of execute + completeJob so a graceful shutdown can wait for
        // the report to land before tearing down the socket (otherwise the job is orphaned in active).
        inFlightJobs++
        markPollLoopIteration({ workerIndex, busy: true })
        try {
            const lockExtensionInterval = setInterval(() => {
                void tryCatch(() => apiClient.extendLock({ jobId: job.jobId, token: job.token, queueName: job.queueName })).then(({ error }) => {
                    if (error) {
                        workerLog.warn({ error, job: { id: job.jobId } }, 'Failed to extend lock')
                    }
                })
            }, 30_000)

            const { data: result, error: execError } = await tryCatch(() =>
                executeJob(apiClient, job, runtime, workerIndex),
            )

            const { error: completeError } = await tryCatch(() =>
                apiClient.completeJob({
                    jobId: job.jobId,
                    token: job.token,
                    queueName: job.queueName,
                    status: execError
                        ? EngineResponseStatus.INTERNAL_ERROR
                        : result.status,
                    errorMessage: buildErrorMessage(execError ?? undefined, result ?? undefined),
                    logs: extractLogs(execError ?? undefined, result ?? undefined),
                    response: result?.kind === JobResultKind.SYNCHRONOUS ? result.response : undefined,
                }),
            )

            clearInterval(lockExtensionInterval)

            if (completeError) {
                workerLog.error({ error: completeError, job: { id: job.jobId } }, 'Failed to complete job')
            }
        }
        finally {
            inFlightJobs--
            markPollLoopIteration({ workerIndex, busy: false })
        }
    }
    workerLog.warn({ polling, generation, connectionGeneration }, 'Poll loop exited — this worker consumes no jobs until it starts again')
}

async function drainInFlightJobs(): Promise<void> {
    const deadline = Date.now() + DRAIN_TIMEOUT_MS
    while (inFlightJobs > 0 && Date.now() < deadline) {
        await sleep(100)
    }
    if (inFlightJobs > 0) {
        logger.warn({ inFlightJobs }, 'Drain timeout reached with jobs still in flight; leaving them for the stalled-scan to reclaim')
    }
}

// Kill the current runtime (and the job running on it) without awaiting — awaiting the shutdown
// while a job is in-flight deadlocks (the await never resolves, polling never restarts). Nulling
// `runtime` lets the next (re)connect create a fresh box.
function abortInFlightRuntime(): void {
    if (isNil(runtime)) {
        return
    }
    const oldRuntime = runtime
    runtime = null
    void tryCatch(() => oldRuntime.shutdown(logger)).then(({ error }) => {
        if (error) {
            logger.error({ error }, 'Failed to shut down runtime')
        }
    })
}

function flowTimeoutMs(): number {
    const { data: settings } = tryCatchSync(() => workerSettings.getSettings())
    return (settings?.FLOW_TIMEOUT_SECONDS ?? FALLBACK_FLOW_TIMEOUT_SECONDS) * 1000
}

function rpcTimeoutMsFor(method: string): number {
    if (!LONG_RUNNING_RPC_METHODS.includes(method)) {
        return RPC_TIMEOUT_MS
    }
    const { data: settings } = tryCatchSync(() => workerSettings.getSettings())
    if (isNil(settings)) {
        logger.warn({ rpc: { method } }, 'Worker settings have not arrived, timing a long-running RPC by the default flow timeout')
    }
    return flowTimeoutMs() + LONG_RUNNING_RPC_MARGIN_MS
}

async function executeJob(apiClient: WorkerToApiContract, job: ConsumeJobRequest, runtime: Runtime, workerIndex: number): Promise<JobResult> {
    const rawData = job.jobData
    const jobData = JobData.parse(rawData)
    // Chat jobs reuse `runId` as the per-message chat run id (NOT a flow run);
    // map it to the `run`/`conversation` groups so chat logs correlate correctly.
    const isAgentJob = jobData.jobType === WorkerJobType.EXECUTE_AGENT_RUN
    const jobLogger = createLogger({
        event: 'job.execute',
        job: { id: job.jobId, type: jobData.jobType },
        ...spreadIfDefined('requestId', 'requestId' in jobData ? jobData.requestId : 'httpRequestId' in jobData ? jobData.httpRequestId : undefined),
        ...spreadIfDefined('project', 'projectId' in jobData && jobData.projectId != null ? { id: jobData.projectId } : undefined),
        ...spreadIfDefined('platform', 'platformId' in jobData ? { id: jobData.platformId } : undefined),
        ...spreadIfDefined('flow', 'flowId' in jobData ? { id: jobData.flowId } : undefined),
        ...spreadIfDefined('flowRun', !isAgentJob && 'runId' in jobData ? { id: jobData.runId } : undefined),
        ...spreadIfDefined('run', isAgentJob && 'runId' in jobData ? { id: jobData.runId } : undefined),
        ...spreadIfDefined('conversation', 'conversationId' in jobData ? { id: jobData.conversationId } : undefined),
        ...spreadIfDefined('flowVersion', 'flowVersionId' in jobData ? { id: jobData.flowVersionId } : undefined),
        ...spreadIfDefined('waitpoint', 'parentWaitpointId' in jobData && jobData.parentWaitpointId != null ? { id: jobData.parentWaitpointId } : undefined),
    })
    return wideEvent.run({
        logger: jobLogger,
        fn: async () => {
            const log = logger.child({ job: { id: job.jobId, type: jobData.jobType } })
            const apiUrl = getApiUrl()
            const { PUBLIC_URL: publicUrl } = await workerSettings.waitForSettings()
            log.debug({ apiUrl, publicUrl }, 'Worker settings resolved')
            const publicApiUrl = ensurePublicApiUrl(publicUrl)
            // The engine forks in-process inside this worker, so it reaches the app over the worker's own
            // app URL (cluster-internal when co-located with the app, the public URL for a standalone worker).
            const internalApiUrl = apiUrl
            const ctx: JobContext = {
                apiClient,
                runtime,
                resolver: createResolver({
                    apiClient,
                    basePath: sandboxConfig.getCacheBasePath(),
                    getSettings: () => sandboxConfig.getSandboxSettings(),
                    log,
                }),
                workerIndex,
                jobId: job.jobId,
                lastAttempt: job.lastAttempt ?? true,
                engineToken: job.engineToken,
                internalApiUrl,
                publicApiUrl,
                log,
            }
            try {
                const handler = await getHandler(jobData.jobType)
                log.debug({ handlerType: handler.jobType }, 'Executing job with handler')
                const { data: result, error } = await tryCatch(() => handler.execute(ctx, jobData))
                if (error) {
                    log.error({ error }, 'Job execution failed')
                    wideEvent.error(error)
                    wideEvent.set({ outcome: 'failed' })
                    throw error
                }
                log.debug('Job completed')
                wideEvent.set({ outcome: 'success' })
                return result
            }
            finally {
                jobLogger.emit()
            }
        },
    })
}

export function ensurePublicApiUrl(publicUrl: string): string {
    if (publicUrl.endsWith('/api/')) return publicUrl
    if (publicUrl.endsWith('/api')) return publicUrl + '/'
    if (publicUrl.endsWith('/')) return publicUrl + 'api/'
    return publicUrl + '/api/'
}

async function fetchAndStoreSettings(sock: Socket): Promise<void> {
    const { data: request, error } = await tryCatch(buildMachineInfo)
    if (error) {
        logger.error({ error }, 'Failed to build machine info for settings fetch')
        return
    }
    return new Promise<void>((resolve) => {
        sock.emit(WebsocketServerEvent.FETCH_WORKER_SETTINGS, request, (response: WorkerSettingsResponse) => {
            const localExecutionMode = system.get(WorkerSystemProp.EXECUTION_MODE)
            if (!isNil(localExecutionMode)) {
                response.EXECUTION_MODE = localExecutionMode
            }
            const localSandboxMemoryLimit = system.get(WorkerSystemProp.SANDBOX_MEMORY_LIMIT)
            if (!isNil(localSandboxMemoryLimit) && localSandboxMemoryLimit !== '') {
                if (!/^[1-9]\d*$/.test(localSandboxMemoryLimit)) {
                    throw new Error(`AP_SANDBOX_MEMORY_LIMIT must be a positive integer in KB. Got: ${localSandboxMemoryLimit}`)
                }
                response.SANDBOX_MEMORY_LIMIT = localSandboxMemoryLimit
            }
            const workerGroupId = system.get(WorkerSystemProp.WORKER_GROUP_ID)
            if (!isNil(workerGroupId)) {
                if (response.EDITION === ApEdition.CLOUD) {
                    const processSandboxedModes: string[] = [ExecutionMode.SANDBOX_PROCESS, ExecutionMode.SANDBOX_CODE_AND_PROCESS]
                    if (!processSandboxedModes.includes(response.EXECUTION_MODE)) {
                        throw new Error(`Worker group "${workerGroupId}" requires AP_EXECUTION_MODE to be one of: ${processSandboxedModes.join(', ')}. Got: ${response.EXECUTION_MODE}`)
                    }
                }
                const reuseSandbox = system.get(WorkerSystemProp.REUSE_SANDBOX)
                if (isNil(reuseSandbox)) {
                    throw new Error(`Worker group "${workerGroupId}" requires AP_REUSE_SANDBOX to be set (true or false)`)
                }
            }
            workerSettings.set(response)
            aiObservability.init({ enabled: response.EDITION === ApEdition.CLOUD })
            logger.info({ environment: response.ENVIRONMENT, executionMode: response.EXECUTION_MODE, sandboxMemoryLimitKb: response.SANDBOX_MEMORY_LIMIT }, 'Worker settings loaded')
            resolve()
        })
    })
}

function getWorkerProps(): WorkerProps {
    try {
        const settings = sandboxConfig.getSandboxSettings()
        return {
            EXECUTION_MODE: settings.EXECUTION_MODE,
            WORKER_CONCURRENCY: system.get(WorkerSystemProp.WORKER_CONCURRENCY)!,
            SANDBOX_MEMORY_LIMIT: settings.SANDBOX_MEMORY_LIMIT,
            REUSE_SANDBOX: system.get(WorkerSystemProp.REUSE_SANDBOX) ?? 'false',
            version: versionChecker.workerVersion,
        }
    }
    catch {
        return {}
    }
}

async function buildMachineInfo(): Promise<WorkerMachineHealthcheckRequest> {
    return withTimeout({
        promise: collectMachineInfo(),
        timeoutMs: MACHINE_INFO_TIMEOUT_MS,
        message: 'Timed out collecting machine info',
    })
}

async function collectMachineInfo(): Promise<WorkerMachineHealthcheckRequest> {
    const memInfo = await systemUsage.getContainerMemoryUsage()
    const diskInfo = await systemUsage.getDiskInfo()
    const cpuCores = await systemUsage.getCpuCores()
    const serverPingMs = await probeServerPing()
    return {
        workerId,
        cpuUsagePercentage: systemUsage.getCpuUsage(),
        diskInfo,
        workerProps: getWorkerProps(),
        ramUsagePercentage: memInfo.ramUsage,
        totalAvailableRamInBytes: memInfo.totalRamInBytes,
        totalCpuCores: cpuCores,
        ip: workerHostname,
        serverPingMs,
        sandboxes: cachedSandboxInfo,
    }
}

async function probeServerPing(): Promise<number | undefined> {
    const startedAt = Date.now()
    const probe = tryCatch(() => fetch(`${getApiUrl()}v1/health`, { signal: AbortSignal.timeout(SERVER_PING_TIMEOUT_MS) }))
        .then(({ data: response, error }) => {
            void response?.body?.cancel()
            return error ? undefined : Date.now() - startedAt
        })
    return Promise.race([probe, sleep(SERVER_PING_TIMEOUT_MS).then(() => undefined)])
}

function withTimeout<T>({ promise, timeoutMs, message }: { promise: Promise<T>, timeoutMs: number, message: string }): Promise<T> {
    return new Promise<T>((resolve, reject) => {
        const timer = setTimeout(() => reject(new Error(message)), timeoutMs)
        promise.then(resolve, reject).finally(() => clearTimeout(timer))
    })
}

function startSandboxInfoSampling(): void {
    if (!isNil(sandboxInfoInterval)) {
        return
    }
    void refreshSandboxInfo()
    sandboxInfoInterval = setInterval(() => void refreshSandboxInfo(), SANDBOX_INFO_REFRESH_MS)
    sandboxInfoInterval.unref()
}

function stopSandboxInfoSampling(): void {
    if (!isNil(sandboxInfoInterval)) {
        clearInterval(sandboxInfoInterval)
        sandboxInfoInterval = null
    }
    cachedSandboxInfo = []
}

async function refreshSandboxInfo(): Promise<void> {
    const { data, error } = await tryCatch(buildSandboxInfo)
    if (error) {
        logger.warn({ error }, 'Failed to refresh sandbox info')
        return
    }
    cachedSandboxInfo = data
}

async function buildSandboxInfo(): Promise<SandboxInformation[]> {
    const activeExecutors = runtime?.getActiveExecutors() ?? []
    if (activeExecutors.length === 0) {
        return []
    }
    const memoryByPid = await systemUsage.getProcessTreeMemoryBytesByPids(activeExecutors.map((executor) => executor.pid))
    return activeExecutors.map((executor) => ({
        sandboxId: executor.sandboxId,
        boxId: executor.boxId,
        busy: executor.busy,
        memoryUsageBytes: memoryByPid.get(executor.pid) ?? 0,
    }))
}

function buildErrorMessage(execError: Error | undefined, result: JobResult | undefined): string | undefined {
    if (execError) {
        return execError.message
    }
    const isFailure = result?.kind === JobResultKind.SYNCHRONOUS && result.status !== EngineResponseStatus.OK
    if (!isFailure) {
        return undefined
    }
    return result.errorMessage
}

function extractLogs(execError: Error | undefined, result: JobResult | undefined): string | undefined {
    if (execError instanceof ActivepiecesError) {
        const params = execError.error.params as Record<string, unknown>
        const parts: string[] = []
        if (params?.['standardOutput']) parts.push(`stdout:\n${params['standardOutput']}`)
        if (params?.['standardError']) parts.push(`stderr:\n${params['standardError']}`)
        return parts.length > 0 ? parts.join('\n') : undefined
    }
    if (result && 'logs' in result) {
        return result.logs
    }
    return undefined
}

function sleep(ms: number): Promise<void> {
    return new Promise((resolve) => setTimeout(resolve, ms))
}

function resetPollLoopLiveness({ loopCount }: { loopCount: number }): void {
    const startedAt = Date.now()
    pollLoopLiveness = Array.from({ length: loopCount }, () => ({ iteratedAt: startedAt, busy: false }))
}

function markPollLoopIteration({ workerIndex, busy }: { workerIndex: number, busy: boolean }): void {
    const liveness = pollLoopLiveness[workerIndex]
    if (isNil(liveness)) {
        return
    }
    liveness.iteratedAt = Date.now()
    liveness.busy = busy
}

function findStalledPollLoop(): StalledPollLoop | undefined {
    const connected = socket?.connected ?? false
    if (!connected) {
        return undefined
    }
    const now = Date.now()
    const workerIndex = findStalledLoopIndex({ loops: pollLoopLiveness, now, timeoutMs: POLL_LIVENESS_TIMEOUT_MS })
    if (workerIndex === -1) {
        return undefined
    }
    return { workerIndex, stalledForMs: now - pollLoopLiveness[workerIndex].iteratedAt }
}

function startCacheSweeper(): void {
    if (!isNil(cacheSweepInterval) || !isNil(cacheSweepFirstRunTimeout)) {
        return
    }
    cacheSweepFirstRunTimeout = setTimeout(() => {
        cacheSweepFirstRunTimeout = null
        void sweepActionRunCache()
        cacheSweepInterval = setInterval(() => void sweepActionRunCache(), ACTION_RUN_CACHE_SWEEP_INTERVAL_MS)
        cacheSweepInterval.unref()
    }, ACTION_RUN_CACHE_FIRST_SWEEP_DELAY_MS)
    cacheSweepFirstRunTimeout.unref()
}

function stopCacheSweeper(): void {
    if (!isNil(cacheSweepFirstRunTimeout)) {
        clearTimeout(cacheSweepFirstRunTimeout)
        cacheSweepFirstRunTimeout = null
    }
    if (!isNil(cacheSweepInterval)) {
        clearInterval(cacheSweepInterval)
        cacheSweepInterval = null
    }
}

async function sweepActionRunCache(): Promise<void> {
    const { error } = await tryCatch(() => actionRunCache.sweep({
        basePath: sandboxConfig.getCacheBasePath(),
        log: logger,
        activeWindowMs: Math.max(ACTION_RUN_CACHE_ACTIVE_WINDOW_MS, flowTimeoutMs()),
    }))
    if (error) {
        logger.warn({ error }, 'Action-run code cache sweep failed')
    }
    // Reclaims the directories left by earlier LATEST_CACHE_VERSION values. It rides the periodic
    // sweep rather than startup because it can only run once the rollout grace period has passed,
    // which is long after a worker boots. Canary workers never reclaim: they run ahead of prod for
    // days, and deleting the previous version's cache there would make a rollback start cold.
    if (system.getBoolean(WorkerSystemProp.IS_CANARY_WORKER) ?? false) {
        return
    }
    await cacheUtils(sandboxConfig.getCacheBasePath()).deleteStaleCache(logger)
}

function startPollWatchdog(): void {
    if (!isNil(pollWatchdogInterval)) {
        return
    }
    pollWatchdogInterval = setInterval(() => {
        const stalled = findStalledPollLoop()
        if (isNil(stalled)) {
            return
        }
        logger.error({ ...stalled, loopCount: pollLoopLiveness.length }, 'Poll loop stalled — exiting so the process manager restarts a worker that can consume jobs')
        process.exit(1)
    }, POLL_WATCHDOG_INTERVAL_MS)
    pollWatchdogInterval.unref()
}

function stopPollWatchdog(): void {
    if (!isNil(pollWatchdogInterval)) {
        clearInterval(pollWatchdogInterval)
        pollWatchdogInterval = null
    }
}


function startHealthServer(): ReturnType<typeof createServer> {
    const port = Number(process.env[WorkerSystemProp.PORT] ?? system.get(WorkerSystemProp.PORT))
    const healthPaths = new Set(['/worker/health', '/v1/health', '/api/v1/health'])
    const server = createServer((req, res) => {
        if (req.method === 'GET' && req.url && healthPaths.has(req.url)) {
            const stalled = !isNil(findStalledPollLoop())
            res.writeHead(stalled ? 503 : 200, { 'Content-Type': 'application/json' })
            res.end(JSON.stringify({ status: stalled ? 'poll_loop_stalled' : 'ok' }))
        }
        else {
            res.writeHead(404)
            res.end()
        }
    })
    server.listen(port, () => {
        logger.info({ port }, 'Health server listening')
    })
    return server
}

// closeAllConnections drops kubelet keep-alive sockets too, so the next probe is refused
// immediately instead of riding an already-open connection until it happens to close.
function stopHealthServer(): void {
    if (isNil(healthServerInstance)) {
        return
    }
    healthServerInstance.closeAllConnections()
    healthServerInstance.close()
    healthServerInstance = null
    logger.info('Health server stopped, probes will fail until reconnect and prewarm complete')
}

type WorkerStartParams = {
    apiUrl: string
    socketUrl: { url: string, path: string }
    workerToken: string
    withHealthServer?: boolean
}

export function findStalledLoopIndex({ loops, now, timeoutMs }: { loops: PollLoopLiveness[], now: number, timeoutMs: number }): number {
    return loops.findIndex((loop) => !loop.busy && now - loop.iteratedAt > timeoutMs)
}

export type PollLoopLiveness = {
    iteratedAt: number
    busy: boolean
}

type StalledPollLoop = {
    workerIndex: number
    stalledForMs: number
}