Skip to content

AI Agents & MCPs & AI Workflow Automation • (~400 MCP servers for AI agents) • AI Automation / AI Agent with MCPs • AI Workflows & AI Agents • MCPs for AI Agents

fix(worker): read sandbox memory with pidtree + pidusage (#16562)

Abdul736d3557ab33
diff --git a/bun.lock b/bun.lock
index 8b114d44682..3d205c95937 100644
--- a/bun.lock
+++ b/bun.lock
@@ -11137,6 +11137,8 @@
"dayjs": "1.11.9",
"evlog": "2.22.4",
"fastify": "5.12.5",
+ "pidtree": "0.6.0",
+ "pidusage": "4.0.1",
"protobufjs": "7.6.6",
"request-filtering-agent": "3.2.1",
"systeminformation": "5.31.7",
@@ -11146,6 +11148,7 @@
"devDependencies": {
"@ai-sdk/devtools": "1.0.9",
"@types/node": "24.11.0",
+ "@types/pidusage": "2.0.5",
"vitest": "3.2.6",
},
},
@@ -14789,6 +14792,8 @@
"@types/phoenix": ["@types/[email protected]", "", {}, "sha512-oN9ive//QSBkf19rfDv45M7eZPi0eEXylht2OLEXicu5b4KoQ1OzXIw+xDSGWxSxe1JmepRR/ZH283vsu518/Q=="],
+ "@types/pidusage": ["@types/[email protected]", "", {}, "sha512-MIiyZI4/MK9UGUXWt0jJcCZhVw7YdhBuTOuqP/BjuLDLZ2PmmViMIQgZiWxtaMicQfAz/kMrZ5T7PKxFSkTeUA=="],
+
"@types/qrcode": ["@types/[email protected]", "", { "dependencies": { "@types/node": "*" } }, "sha512-te7NQcV2BOvdj2b1hCAHzAoMNuj65kNBMz0KBaxM6c3VGBOhU0dURQKOtH8CFNI/dsKkwlv32p26qYQTWoB5bw=="],
"@types/qs": ["@types/[email protected]", "", {}, "sha512-eOunJqu0K1923aExK6y8p6fsihYEn/BYuQ4g0CxAAgFc4b/ZLN4CrsRZ55srTdqoiLzU2B2evC+apEIxprEzkQ=="],
@@ -17217,6 +17222,8 @@
"pidtree": ["[email protected]", "", { "bin": { "pidtree": "bin/pidtree.js" } }, "sha512-eG2dWTVw5bzqGRztnHExczNxt5VGsE6OwTeCG3fdUf9KBsZzO3R5OIIIzWR+iZA0NtZ+RDVdaoE2dK1cn6jH4g=="],
+ "pidusage": ["[email protected]", "", { "dependencies": { "safe-buffer": "^5.2.1" } }, "sha512-yCH2dtLHfEBnzlHUJymR/Z1nN2ePG3m392Mv8TFlTP1B0xkpMQNHAnfkY0n2tAi6ceKO6YWhxYfZ96V4vVkh/g=="],
+
"pify": ["[email protected]", "", {}, "sha512-udgsAY+fTnvv7kI7aaxbqwWNb0AHiB0qBO89PZKPkoTmGOgdbrHDKD+0B2X4uTfJ/FT1R09r9gTsjUjNJotuog=="],
"pinkie": ["[email protected]", "", {}, "sha512-MnUuEycAemtSaeFSjXKW/aroV7akBbY+Sv+RkyqFjgAe73F+MR0TBWKBRDkmfWq/HiFmdavfZ1G7h4SPZXaCSg=="],
diff --git a/packages/server/utils/package.json b/packages/server/utils/package.json
index 3ed0fd5a016..8ead804ad81 100644
--- a/packages/server/utils/package.json
+++ b/packages/server/utils/package.json
@@ -29,6 +29,8 @@
"check-disk-space": "3.4.0",
"dayjs": "1.11.9",
"fastify": "5.12.5",
+ "pidtree": "0.6.0",
+ "pidusage": "4.0.1",
"request-filtering-agent": "3.2.1",
"systeminformation": "5.31.7",
"tslib": "2.6.2",
@@ -39,6 +41,7 @@
"devDependencies": {
"@ai-sdk/devtools": "1.0.9",
"@types/node": "24.11.0",
+ "@types/pidusage": "2.0.5",
"vitest": "3.2.6"
}
}
diff --git a/packages/server/utils/src/system-usage.ts b/packages/server/utils/src/system-usage.ts
index 76d1ace6518..616e3e543f3 100644
--- a/packages/server/utils/src/system-usage.ts
+++ b/packages/server/utils/src/system-usage.ts
@@ -3,6 +3,8 @@ import os from 'os'
import { tryCatch } from '@activepieces/core-utils';
import { MachineInformation } from '@activepieces/shared';
import checkDiskSpace from 'check-disk-space'
+import pidtree from 'pidtree'
+import pidusage from 'pidusage'
import si from 'systeminformation'
import { fileSystemUtils } from './file-system-utils'
@@ -205,30 +207,14 @@ export const systemUsage = {
},
async getProcessTreeMemoryBytesByPids(pids: number[]): Promise<Map<number, number>> {
- const result = new Map<number, number>()
if (pids.length === 0) {
- return result
+ return new Map()
}
- // si.processes() walks the whole process table — do it once for all pids, never per-pid.
- const { data, error } = await tryCatch(() => si.processes())
- if (error || !data) {
- for (const pid of pids) {
- result.set(pid, 0)
- }
- return result
- }
- const childrenByParent = new Map<number, number[]>()
- const rssBytesByPid = new Map<number, number>()
- for (const proc of data.list) {
- rssBytesByPid.set(proc.pid, proc.memRss * 1024)
- const siblings = childrenByParent.get(proc.parentPid) ?? []
- siblings.push(proc.pid)
- childrenByParent.set(proc.parentPid, siblings)
- }
- for (const pid of pids) {
- result.set(pid, sumProcessTreeRssBytes({ rootPid: pid, rssBytesByPid, childrenByParent }))
+ const { data, error } = await tryCatch(() => readProcessTreeMemoryBytes(pids))
+ if (error) {
+ return new Map(pids.map((pid) => [pid, 0]))
}
- return result
+ return data
},
}
@@ -236,16 +222,26 @@ type CpuCounters = { steal: number, total: number }
type CfsCounters = { throttled: number, periods: number }
export type CpuPressure = { cpuStealPercentage?: number, cpuThrottledPercentage?: number }
-function sumProcessTreeRssBytes({ rootPid, rssBytesByPid, childrenByParent }: { rootPid: number, rssBytesByPid: Map<number, number>, childrenByParent: Map<number, number[]> }): number {
- let total = 0
- const queue = [rootPid]
- const visited = new Set<number>()
- while (queue.length > 0) {
- const current = queue.shift()!
- if (visited.has(current)) continue
- visited.add(current)
- total += rssBytesByPid.get(current) ?? 0
- queue.push(...(childrenByParent.get(current) ?? []))
+async function readProcessTreeMemoryBytes(rootPids: number[]): Promise<Map<number, number>> {
+ const processTable = await pidtree(-1, { advanced: true })
+ const childrenByParent = new Map<number, number[]>()
+ for (const { ppid, pid } of processTable) {
+ const children = childrenByParent.get(ppid) ?? []
+ children.push(pid)
+ childrenByParent.set(ppid, children)
+ }
+ const processTrees = rootPids.map((rootPid) => ({ rootPid, selfAndDescendantPids: collectSelfAndDescendants({ rootPid, childrenByParent }) }))
+ const statsByPid = await pidusage([...new Set(processTrees.flatMap(({ selfAndDescendantPids }) => selfAndDescendantPids))])
+ return new Map(processTrees.map(({ rootPid, selfAndDescendantPids }) => [
+ rootPid,
+ selfAndDescendantPids.reduce((total, pid) => total + (statsByPid[pid]?.memory ?? 0), 0),
+ ]))
+}
+
+function collectSelfAndDescendants({ rootPid, childrenByParent }: { rootPid: number, childrenByParent: Map<number, number[]> }): number[] {
+ const pids = [rootPid]
+ for (let i = 0; i < pids.length; i++) {
+ pids.push(...(childrenByParent.get(pids[i]) ?? []).filter((child) => !pids.includes(child)))
}
- return total
+ return pids
}
diff --git a/packages/server/utils/test/system-usage.test.ts b/packages/server/utils/test/system-usage.test.ts
index 74875874a0e..26234e8faca 100644
--- a/packages/server/utils/test/system-usage.test.ts
+++ b/packages/server/utils/test/system-usage.test.ts
@@ -18,23 +18,33 @@ vi.mock('../src/file-system-utils', () => ({
vi.mock('systeminformation', () => ({
default: {
mem: vi.fn(),
- processes: vi.fn(),
},
}))
+vi.mock('pidtree', () => ({
+ default: vi.fn(),
+}))
+
+vi.mock('pidusage', () => ({
+ default: vi.fn(),
+}))
+
vi.mock('check-disk-space', () => ({
default: vi.fn(),
}))
import fs from 'fs'
import checkDiskSpace from 'check-disk-space'
+import pidtree from 'pidtree'
+import pidusage from 'pidusage'
import si from 'systeminformation'
import { fileSystemUtils } from '../src/file-system-utils'
const mockFileExists = vi.mocked(fileSystemUtils.fileExists)
const mockReadFile = vi.mocked(fs.promises.readFile)
const mockMem = vi.mocked(si.mem)
-const mockProcesses = vi.mocked(si.processes)
+const mockPidtree = vi.mocked(pidtree)
+const mockPidusage = vi.mocked(pidusage)
const mockCheckDiskSpace = vi.mocked(checkDiskSpace)
function mockCgroupFile(path: string, content: string) {
@@ -275,40 +285,68 @@ describe('getCpuUsage', () => {
})
describe('getProcessTreeMemoryBytesByPids', () => {
- function mockProcessList(list: { pid: number, parentPid: number, memRss: number }[]) {
- mockProcesses.mockResolvedValue({ list } as never)
+ function mockProcessTable(list: { pid: number, ppid: number, memory: number }[]) {
+ mockPidtree.mockResolvedValue(list.map(({ pid, ppid }) => ({ pid, ppid })) as never)
+ mockPidusage.mockImplementation((async (pids: number[]) => Object.fromEntries(
+ list.filter(({ pid }) => pids.includes(pid)).map(({ pid, memory }) => [pid, { memory }]),
+ )) as never)
}
it('returns an empty map without scanning when given no pids', async () => {
const result = await systemUsage.getProcessTreeMemoryBytesByPids([])
expect(result.size).toBe(0)
- expect(mockProcesses).not.toHaveBeenCalled()
+ expect(mockPidtree).not.toHaveBeenCalled()
})
- it('sums the rss of each pid and its descendants (memRss is KiB)', async () => {
- // 100 -> 200 -> 300, and an unrelated 999. memRss is in KiB, summed as bytes.
- mockProcessList([
- { pid: 100, parentPid: 1, memRss: 10 },
- { pid: 200, parentPid: 100, memRss: 20 },
- { pid: 300, parentPid: 200, memRss: 30 },
- { pid: 999, parentPid: 1, memRss: 50 },
+ it('sums the memory of each pid and its descendants', async () => {
+ mockProcessTable([
+ { pid: 100, ppid: 1, memory: 10 },
+ { pid: 200, ppid: 100, memory: 20 },
+ { pid: 300, ppid: 200, memory: 30 },
+ { pid: 999, ppid: 1, memory: 50 },
])
const result = await systemUsage.getProcessTreeMemoryBytesByPids([100, 999])
- expect(result.get(100)).toBe((10 + 20 + 30) * 1024)
- expect(result.get(999)).toBe(50 * 1024)
+ expect(result.get(100)).toBe(10 + 20 + 30)
+ expect(result.get(999)).toBe(50)
})
- it('scans the process table only once for many pids', async () => {
- mockProcessList([
- { pid: 100, parentPid: 1, memRss: 10 },
- { pid: 200, parentPid: 1, memRss: 20 },
+ it('scans the process table and reads usage only once for many pids', async () => {
+ mockProcessTable([
+ { pid: 100, ppid: 1, memory: 10 },
+ { pid: 200, ppid: 1, memory: 20 },
])
await systemUsage.getProcessTreeMemoryBytesByPids([100, 200])
- expect(mockProcesses).toHaveBeenCalledTimes(1)
+ expect(mockPidtree).toHaveBeenCalledTimes(1)
+ expect(mockPidusage).toHaveBeenCalledTimes(1)
+ })
+
+ it('counts a pid that exited between the scan and the read as 0', async () => {
+ mockPidtree.mockResolvedValue([{ pid: 100, ppid: 1 }, { pid: 200, ppid: 100 }] as never)
+ mockPidusage.mockResolvedValue({ 100: { memory: 10 } } as never)
+ const result = await systemUsage.getProcessTreeMemoryBytesByPids([100])
+ expect(result.get(100)).toBe(10)
+ })
+
+ it('counts a pid that exited before its tree was read as 0 without affecting the others', async () => {
+ mockProcessTable([
+ { pid: 200, ppid: 1, memory: 20 },
+ ])
+ const result = await systemUsage.getProcessTreeMemoryBytesByPids([100, 200])
+ expect(result.get(100)).toBe(0)
+ expect(result.get(200)).toBe(20)
+ })
+
+ it('maps every pid to 0 when scanning the process table fails', async () => {
+ mockPidtree.mockRejectedValue(new Error('ps failed'))
+ const result = await systemUsage.getProcessTreeMemoryBytesByPids([100, 200])
+ expect(result.get(100)).toBe(0)
+ expect(result.get(200)).toBe(0)
+ expect(mockPidusage).not.toHaveBeenCalled()
})
- it('maps every pid to 0 when the scan fails', async () => {
- mockProcesses.mockRejectedValue(new Error('scan failed'))
+ it('maps every pid to 0 when reading usage fails', async () => {
+ mockPidtree.mockResolvedValue([{ pid: 100, ppid: 1 }] as never)
+ mockPidusage.mockRejectedValue(new Error('read failed'))
const result = await systemUsage.getProcessTreeMemoryBytesByPids([100, 200])
expect(result.get(100)).toBe(0)
expect(result.get(200)).toBe(0)
diff --git a/packages/server/worker/src/lib/main.ts b/packages/server/worker/src/lib/main.ts
index 30d593c7025..b79c61e95ee 100644
--- a/packages/server/worker/src/lib/main.ts
+++ b/packages/server/worker/src/lib/main.ts
@@ -1,4 +1,4 @@
-import { aiDevtools } from '@activepieces/server-utils'
+import { aiDevtools, evlogSetup } from '@activepieces/server-utils'
import { ApEnvironment } from '@activepieces/shared'
import { getApiUrl, getSocketUrl, system, WorkerSystemProp } from './config/configs'
import { logger } from './config/logger'
@@ -6,6 +6,22 @@ import { workerSystemSnapshot } from './utils/system-snapshot'
import { worker } from './worker'
const workerToken = system.getOrThrow(WorkerSystemProp.WORKER_TOKEN)
+const LOG_FLUSH_TIMEOUT_MS = 3_000
+
+process.on('uncaughtException', (error) => {
+ logger.error({ error }, 'Uncaught exception in worker')
+ flushLogsThenExit()
+})
+process.on('unhandledRejection', (reason) => {
+ logger.error({ error: reason }, 'Unhandled rejection in worker')
+ flushLogsThenExit()
+})
+
+function flushLogsThenExit(): void {
+ worker.stopPolling()
+ setTimeout(() => process.exit(1), LOG_FLUSH_TIMEOUT_MS)
+ evlogSetup.flush().catch(() => undefined).finally(() => process.exit(1))
+}
async function main(): Promise<void> {
await aiDevtools.register({ enabled: system.get(WorkerSystemProp.ENVIRONMENT) === ApEnvironment.DEVELOPMENT })
diff --git a/packages/server/worker/src/lib/worker.ts b/packages/server/worker/src/lib/worker.ts
index 56d0c9e0b90..1c8762548d3 100644
--- a/packages/server/worker/src/lib/worker.ts
+++ b/packages/server/worker/src/lib/worker.ts
@@ -150,6 +150,10 @@ export const worker = {
stopHealthServer()
logger.info('Worker stopped')
},
+
+ stopPolling(): void {
+ polling = false
+ },
}
async function startPollingWorkers(apiClient: WorkerToApiContract): Promise<void> {