AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents
TypeScript ai-agentai-agent-toolsai-agentsai-agents-frameworkmcpmcp-servermcp-toolsmcpsn8n-alternativeno-code-automationworkflowworkflow-automationworkflows
fix(worker): read sandbox memory with pidtree + pidusage (#16562)
Abdul736d3557ab33
diff --git a/bun.lock b/bun.lockindex 8b114d44682..3d205c95937 100644--- a/bun.lock+++ b/bun.lock@@ -11137,6 +11137,8 @@"dayjs": "1.11.9","evlog": "2.22.4","fastify": "5.12.5",+ "pidtree": "0.6.0",+ "pidusage": "4.0.1","protobufjs": "7.6.6","request-filtering-agent": "3.2.1","systeminformation": "5.31.7",@@ -11146,6 +11148,7 @@"devDependencies": {"@ai-sdk/devtools": "1.0.9","@types/node": "24.11.0",+ "@types/pidusage": "2.0.5","vitest": "3.2.6",},},@@ -14789,6 +14792,8 @@"@types/phoenix": ["@types/[email protected]", "", {}, "sha512-oN9ive//QSBkf19rfDv45M7eZPi0eEXylht2OLEXicu5b4KoQ1OzXIw+xDSGWxSxe1JmepRR/ZH283vsu518/Q=="],+ "@types/pidusage": ["@types/[email protected]", "", {}, "sha512-MIiyZI4/MK9UGUXWt0jJcCZhVw7YdhBuTOuqP/BjuLDLZ2PmmViMIQgZiWxtaMicQfAz/kMrZ5T7PKxFSkTeUA=="],+"@types/qrcode": ["@types/[email protected]", "", { "dependencies": { "@types/node": "*" } }, "sha512-te7NQcV2BOvdj2b1hCAHzAoMNuj65kNBMz0KBaxM6c3VGBOhU0dURQKOtH8CFNI/dsKkwlv32p26qYQTWoB5bw=="],"@types/qs": ["@types/[email protected]", "", {}, "sha512-eOunJqu0K1923aExK6y8p6fsihYEn/BYuQ4g0CxAAgFc4b/ZLN4CrsRZ55srTdqoiLzU2B2evC+apEIxprEzkQ=="],@@ -17217,6 +17222,8 @@"pidtree": ["[email protected]", "", { "bin": { "pidtree": "bin/pidtree.js" } }, "sha512-eG2dWTVw5bzqGRztnHExczNxt5VGsE6OwTeCG3fdUf9KBsZzO3R5OIIIzWR+iZA0NtZ+RDVdaoE2dK1cn6jH4g=="],+ "pidusage": ["[email protected]", "", { "dependencies": { "safe-buffer": "^5.2.1" } }, "sha512-yCH2dtLHfEBnzlHUJymR/Z1nN2ePG3m392Mv8TFlTP1B0xkpMQNHAnfkY0n2tAi6ceKO6YWhxYfZ96V4vVkh/g=="],+"pify": ["[email protected]", "", {}, "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog=="],"pinkie": ["[email protected]", "", {}, "sha512-MnUuEycAemtSaeFSjXKW/aroV7akBbY+Sv+RkyqFjgAe73F+MR0TBWKBRDkmfWq/HiFmdavfZ1G7h4SPZXaCSg=="],diff --git a/packages/server/utils/package.json b/packages/server/utils/package.jsonindex 3ed0fd5a016..8ead804ad81 100644--- a/packages/server/utils/package.json+++ b/packages/server/utils/package.json@@ -29,6 +29,8 @@"check-disk-space": "3.4.0","dayjs": "1.11.9","fastify": "5.12.5",+ "pidtree": "0.6.0",+ "pidusage": "4.0.1","request-filtering-agent": "3.2.1","systeminformation": "5.31.7","tslib": "2.6.2",@@ -39,6 +41,7 @@"devDependencies": {"@ai-sdk/devtools": "1.0.9","@types/node": "24.11.0",+ "@types/pidusage": "2.0.5","vitest": "3.2.6"}}diff --git a/packages/server/utils/src/system-usage.ts b/packages/server/utils/src/system-usage.tsindex 76d1ace6518..616e3e543f3 100644--- a/packages/server/utils/src/system-usage.ts+++ b/packages/server/utils/src/system-usage.ts@@ -3,6 +3,8 @@ import os from 'os'import { tryCatch } from '@activepieces/core-utils';import { MachineInformation } from '@activepieces/shared';import checkDiskSpace from 'check-disk-space'+import pidtree from 'pidtree'+import pidusage from 'pidusage'import si from 'systeminformation'import { fileSystemUtils } from './file-system-utils'@@ -205,30 +207,14 @@ export const systemUsage = {},async getProcessTreeMemoryBytesByPids(pids: number[]): Promise<Map<number, number>> {- const result = new Map<number, number>()if (pids.length === 0) {- return result+ return new Map()}- // si.processes() walks the whole process table — do it once for all pids, never per-pid.- const { data, error } = await tryCatch(() => si.processes())- if (error || !data) {- for (const pid of pids) {- result.set(pid, 0)- }- return result- }- const childrenByParent = new Map<number, number[]>()- const rssBytesByPid = new Map<number, number>()- for (const proc of data.list) {- rssBytesByPid.set(proc.pid, proc.memRss * 1024)- const siblings = childrenByParent.get(proc.parentPid) ?? []- siblings.push(proc.pid)- childrenByParent.set(proc.parentPid, siblings)- }- for (const pid of pids) {- result.set(pid, sumProcessTreeRssBytes({ rootPid: pid, rssBytesByPid, childrenByParent }))+ const { data, error } = await tryCatch(() => readProcessTreeMemoryBytes(pids))+ if (error) {+ return new Map(pids.map((pid) => [pid, 0]))}- return result+ return data},}@@ -236,16 +222,26 @@ type CpuCounters = { steal: number, total: number }type CfsCounters = { throttled: number, periods: number }export type CpuPressure = { cpuStealPercentage?: number, cpuThrottledPercentage?: number }-function sumProcessTreeRssBytes({ rootPid, rssBytesByPid, childrenByParent }: { rootPid: number, rssBytesByPid: Map<number, number>, childrenByParent: Map<number, number[]> }): number {- let total = 0- const queue = [rootPid]- const visited = new Set<number>()- while (queue.length > 0) {- const current = queue.shift()!- if (visited.has(current)) continue- visited.add(current)- total += rssBytesByPid.get(current) ?? 0- queue.push(...(childrenByParent.get(current) ?? []))+async function readProcessTreeMemoryBytes(rootPids: number[]): Promise<Map<number, number>> {+ const processTable = await pidtree(-1, { advanced: true })+ const childrenByParent = new Map<number, number[]>()+ for (const { ppid, pid } of processTable) {+ const children = childrenByParent.get(ppid) ?? []+ children.push(pid)+ childrenByParent.set(ppid, children)+ }+ const processTrees = rootPids.map((rootPid) => ({ rootPid, selfAndDescendantPids: collectSelfAndDescendants({ rootPid, childrenByParent }) }))+ const statsByPid = await pidusage([...new Set(processTrees.flatMap(({ selfAndDescendantPids }) => selfAndDescendantPids))])+ return new Map(processTrees.map(({ rootPid, selfAndDescendantPids }) => [+ rootPid,+ selfAndDescendantPids.reduce((total, pid) => total + (statsByPid[pid]?.memory ?? 0), 0),+ ]))+}++function collectSelfAndDescendants({ rootPid, childrenByParent }: { rootPid: number, childrenByParent: Map<number, number[]> }): number[] {+ const pids = [rootPid]+ for (let i = 0; i < pids.length; i++) {+ pids.push(...(childrenByParent.get(pids[i]) ?? []).filter((child) => !pids.includes(child)))}- return total+ return pids}diff --git a/packages/server/utils/test/system-usage.test.ts b/packages/server/utils/test/system-usage.test.tsindex 74875874a0e..26234e8faca 100644--- a/packages/server/utils/test/system-usage.test.ts+++ b/packages/server/utils/test/system-usage.test.ts@@ -18,23 +18,33 @@ vi.mock('../src/file-system-utils', () => ({vi.mock('systeminformation', () => ({default: {mem: vi.fn(),- processes: vi.fn(),},}))+vi.mock('pidtree', () => ({+ default: vi.fn(),+}))++vi.mock('pidusage', () => ({+ default: vi.fn(),+}))+vi.mock('check-disk-space', () => ({default: vi.fn(),}))import fs from 'fs'import checkDiskSpace from 'check-disk-space'+import pidtree from 'pidtree'+import pidusage from 'pidusage'import si from 'systeminformation'import { fileSystemUtils } from '../src/file-system-utils'const mockFileExists = vi.mocked(fileSystemUtils.fileExists)const mockReadFile = vi.mocked(fs.promises.readFile)const mockMem = vi.mocked(si.mem)-const mockProcesses = vi.mocked(si.processes)+const mockPidtree = vi.mocked(pidtree)+const mockPidusage = vi.mocked(pidusage)const mockCheckDiskSpace = vi.mocked(checkDiskSpace)function mockCgroupFile(path: string, content: string) {@@ -275,40 +285,68 @@ describe('getCpuUsage', () => {})describe('getProcessTreeMemoryBytesByPids', () => {- function mockProcessList(list: { pid: number, parentPid: number, memRss: number }[]) {- mockProcesses.mockResolvedValue({ list } as never)+ function mockProcessTable(list: { pid: number, ppid: number, memory: number }[]) {+ mockPidtree.mockResolvedValue(list.map(({ pid, ppid }) => ({ pid, ppid })) as never)+ mockPidusage.mockImplementation((async (pids: number[]) => Object.fromEntries(+ list.filter(({ pid }) => pids.includes(pid)).map(({ pid, memory }) => [pid, { memory }]),+ )) as never)}it('returns an empty map without scanning when given no pids', async () => {const result = await systemUsage.getProcessTreeMemoryBytesByPids([])expect(result.size).toBe(0)- expect(mockProcesses).not.toHaveBeenCalled()+ expect(mockPidtree).not.toHaveBeenCalled()})- it('sums the rss of each pid and its descendants (memRss is KiB)', async () => {- // 100 -> 200 -> 300, and an unrelated 999. memRss is in KiB, summed as bytes.- mockProcessList([- { pid: 100, parentPid: 1, memRss: 10 },- { pid: 200, parentPid: 100, memRss: 20 },- { pid: 300, parentPid: 200, memRss: 30 },- { pid: 999, parentPid: 1, memRss: 50 },+ it('sums the memory of each pid and its descendants', async () => {+ mockProcessTable([+ { pid: 100, ppid: 1, memory: 10 },+ { pid: 200, ppid: 100, memory: 20 },+ { pid: 300, ppid: 200, memory: 30 },+ { pid: 999, ppid: 1, memory: 50 },])const result = await systemUsage.getProcessTreeMemoryBytesByPids([100, 999])- expect(result.get(100)).toBe((10 + 20 + 30) * 1024)- expect(result.get(999)).toBe(50 * 1024)+ expect(result.get(100)).toBe(10 + 20 + 30)+ expect(result.get(999)).toBe(50)})- it('scans the process table only once for many pids', async () => {- mockProcessList([- { pid: 100, parentPid: 1, memRss: 10 },- { pid: 200, parentPid: 1, memRss: 20 },+ it('scans the process table and reads usage only once for many pids', async () => {+ mockProcessTable([+ { pid: 100, ppid: 1, memory: 10 },+ { pid: 200, ppid: 1, memory: 20 },])await systemUsage.getProcessTreeMemoryBytesByPids([100, 200])- expect(mockProcesses).toHaveBeenCalledTimes(1)+ expect(mockPidtree).toHaveBeenCalledTimes(1)+ expect(mockPidusage).toHaveBeenCalledTimes(1)+ })++ it('counts a pid that exited between the scan and the read as 0', async () => {+ mockPidtree.mockResolvedValue([{ pid: 100, ppid: 1 }, { pid: 200, ppid: 100 }] as never)+ mockPidusage.mockResolvedValue({ 100: { memory: 10 } } as never)+ const result = await systemUsage.getProcessTreeMemoryBytesByPids([100])+ expect(result.get(100)).toBe(10)+ })++ it('counts a pid that exited before its tree was read as 0 without affecting the others', async () => {+ mockProcessTable([+ { pid: 200, ppid: 1, memory: 20 },+ ])+ const result = await systemUsage.getProcessTreeMemoryBytesByPids([100, 200])+ expect(result.get(100)).toBe(0)+ expect(result.get(200)).toBe(20)+ })++ it('maps every pid to 0 when scanning the process table fails', async () => {+ mockPidtree.mockRejectedValue(new Error('ps failed'))+ const result = await systemUsage.getProcessTreeMemoryBytesByPids([100, 200])+ expect(result.get(100)).toBe(0)+ expect(result.get(200)).toBe(0)+ expect(mockPidusage).not.toHaveBeenCalled()})- it('maps every pid to 0 when the scan fails', async () => {- mockProcesses.mockRejectedValue(new Error('scan failed'))+ it('maps every pid to 0 when reading usage fails', async () => {+ mockPidtree.mockResolvedValue([{ pid: 100, ppid: 1 }] as never)+ mockPidusage.mockRejectedValue(new Error('read failed'))const result = await systemUsage.getProcessTreeMemoryBytesByPids([100, 200])expect(result.get(100)).toBe(0)expect(result.get(200)).toBe(0)diff --git a/packages/server/worker/src/lib/main.ts b/packages/server/worker/src/lib/main.tsindex 30d593c7025..b79c61e95ee 100644--- a/packages/server/worker/src/lib/main.ts+++ b/packages/server/worker/src/lib/main.ts@@ -1,4 +1,4 @@-import { aiDevtools } from '@activepieces/server-utils'+import { aiDevtools, evlogSetup } from '@activepieces/server-utils'import { ApEnvironment } from '@activepieces/shared'import { getApiUrl, getSocketUrl, system, WorkerSystemProp } from './config/configs'import { logger } from './config/logger'@@ -6,6 +6,22 @@ import { workerSystemSnapshot } from './utils/system-snapshot'import { worker } from './worker'const workerToken = system.getOrThrow(WorkerSystemProp.WORKER_TOKEN)+const LOG_FLUSH_TIMEOUT_MS = 3_000++process.on('uncaughtException', (error) => {+ logger.error({ error }, 'Uncaught exception in worker')+ flushLogsThenExit()+})+process.on('unhandledRejection', (reason) => {+ logger.error({ error: reason }, 'Unhandled rejection in worker')+ flushLogsThenExit()+})++function flushLogsThenExit(): void {+ worker.stopPolling()+ setTimeout(() => process.exit(1), LOG_FLUSH_TIMEOUT_MS)+ evlogSetup.flush().catch(() => undefined).finally(() => process.exit(1))+}async function main(): Promise<void> {await aiDevtools.register({ enabled: system.get(WorkerSystemProp.ENVIRONMENT) === ApEnvironment.DEVELOPMENT })diff --git a/packages/server/worker/src/lib/worker.ts b/packages/server/worker/src/lib/worker.tsindex 56d0c9e0b90..1c8762548d3 100644--- a/packages/server/worker/src/lib/worker.ts+++ b/packages/server/worker/src/lib/worker.ts@@ -150,6 +150,10 @@ export const worker = {stopHealthServer()logger.info('Worker stopped')},++ stopPolling(): void {+ polling = false+ },}async function startPollingWorkers(apiClient: WorkerToApiContract): Promise<void> {