🎥 Make videos programmatically with React
`@remotion/renderer`: Fix available memory estimates on macOS and Linux
Jonny Burger26318a9d8246
diff --git a/packages/renderer/src/memory/from-macos-sysctl.ts b/packages/renderer/src/memory/from-macos-sysctl.tsnew file mode 100644index 0000000000..6027bff740--- /dev/null+++ b/packages/renderer/src/memory/from-macos-sysctl.ts@@ -0,0 +1,67 @@+import {execFileSync} from 'node:child_process';+import {platform, totalmem} from 'node:os';+import type {LogLevel} from '../log-level';+import {Log} from '../logger';++export const getAvailableMemoryFromMacos = (+ logLevel: LogLevel,+): number | null => {+ if (platform() !== 'darwin') {+ return null;+ }++ try {+ const output = execFileSync(+ '/usr/sbin/sysctl',+ [+ '-n',+ 'vm.page_free_count',+ 'vm.page_pageable_external_count',+ 'vm.page_purgeable_count',+ // hw.pagesize can differ from the kernel page size under Rosetta.+ 'vm.pagesize',+ 'kern.memorystatus_vm_pressure_level',+ ],+ {encoding: 'utf-8', timeout: 1000, stdio: ['ignore', 'pipe', 'pipe']},+ );+ const values = output.trim().split(/\s+/);+ if (values.length !== 5 || values.some((value) => !/^\d+$/.test(value))) {+ throw new Error('Unexpected macOS memory information from sysctl');+ }++ const numbers = values.map(Number);+ const [free, fileBacked, purgeable, pageSize, pressureLevel] = numbers;+ if (+ numbers.some((value) => !Number.isSafeInteger(value)) ||+ pageSize === 0 ||+ ![1, 2, 4].includes(pressureLevel)+ ) {+ throw new Error('Invalid macOS memory information from sysctl');+ }++ // The sysctl exposes dispatch pressure flags: normal=1, warning=2, critical=4.+ if (pressureLevel === 4) {+ Log.verbose(+ {indent: false, logLevel},+ 'macOS memory pressure is critical. Using os.freemem().',+ );+ return null;+ }++ // XNU's fully reclaimable pages exclude anonymous and compressed memory:+ // https://github.com/apple-oss-distributions/xnu/blob/main/doc/vm/memorystatus_notify.md+ const availableMemory = (free + fileBacked + purgeable) * pageSize;+ if (!Number.isSafeInteger(availableMemory)) {+ throw new Error('Invalid macOS available memory size');+ }++ return Math.min(availableMemory, totalmem());+ } catch (err) {+ Log.verbose(+ {indent: false, logLevel},+ 'Could not read reclaimable macOS memory. Using os.freemem().',+ err,+ );+ return null;+ }+};diff --git a/packages/renderer/src/memory/from-proc-meminfo.ts b/packages/renderer/src/memory/from-proc-meminfo.tsindex 941c029e30..7dfb39d4a8 100644--- a/packages/renderer/src/memory/from-proc-meminfo.ts+++ b/packages/renderer/src/memory/from-proc-meminfo.ts@@ -2,7 +2,7 @@ import {existsSync, readFileSync} from 'node:fs';import type {LogLevel} from '../log-level';import {Log} from '../logger';-export const getFreeMemoryFromProcMeminfo = (+export const getAvailableMemoryFromProcMeminfo = (logLevel: LogLevel,): number | null => {if (!existsSync('/proc/meminfo')) {@@ -11,7 +11,7 @@ export const getFreeMemoryFromProcMeminfo = (try {const data = readFileSync('/proc/meminfo', 'utf-8');- // Split the file by lines and find the line with MemFree+ // MemAvailable includes memory that can be reclaimed without swapping.const lines = data.split('\n');const memAvailableLine = lines.find((line) =>line.startsWith('MemAvailable'),@@ -30,6 +30,9 @@ export const getFreeMemoryFromProcMeminfo = (const value = parseInt(matches[1], 10);const unit = matches[2].toLowerCase();+ if (!Number.isSafeInteger(value)) {+ throw new Error('Invalid MemAvailable value');+ }// Convert the value to bytes based on its unitswitch (unit) {diff --git a/packages/renderer/src/memory/get-available-memory.ts b/packages/renderer/src/memory/get-available-memory.tsindex e2ff770d4e..55f7c2329d 100644--- a/packages/renderer/src/memory/get-available-memory.ts+++ b/packages/renderer/src/memory/get-available-memory.ts@@ -1,47 +1,46 @@-import {freemem} from 'node:os';+import {freemem, platform} from 'node:os';import type {LogLevel} from '../log-level';import {Log} from '../logger';import {getAvailableMemoryFromCgroup} from './from-docker-cgroup';import {getMaxLambdaMemory} from './from-lambda-env';-import {getFreeMemoryFromProcMeminfo} from './from-proc-meminfo';+import {getAvailableMemoryFromMacos} from './from-macos-sysctl';+import {getAvailableMemoryFromProcMeminfo} from './from-proc-meminfo';export const getAvailableMemory = (logLevel: LogLevel) => {- // If we are in Lambda, the most reliable way is to read the environment variable- // This is also pretty much free- const maxMemory = getMaxLambdaMemory();-- if (maxMemory !== null) {- const nodeMemory = freemem();- return Math.min(nodeMemory, maxMemory);+ let hostMemory = freemem();+ let source = 'os.freemem()';+ if (platform() === 'linux') {+ const procInfo = getAvailableMemoryFromProcMeminfo(logLevel);+ if (procInfo !== null) {+ hostMemory = procInfo;+ source = '/proc/meminfo MemAvailable';+ }+ } else if (platform() === 'darwin') {+ const macosMemory = getAvailableMemoryFromMacos(logLevel);+ if (macosMemory !== null) {+ hostMemory = macosMemory;+ source = 'macOS sysctl (free, file-backed and purgeable pages)';+ }}- // If cgroup memory data is available, we are in Docker, and we should respect it- const cgroupMemory = getAvailableMemoryFromCgroup();- if (cgroupMemory !== null) {+ let availableMemory = hostMemory;+ const cgroupMemory =+ platform() === 'linux' ? getAvailableMemoryFromCgroup() : null;+ if (cgroupMemory !== null && Number.isFinite(cgroupMemory)) {// There are 2 Docker memory configurations:// 1. --memory=[num]// 2. Global Docker memory limit// If cgroup limit is higher than global memory, the global memory limit still applies- const nodeMemory = freemem();- const _procInfo = getFreeMemoryFromProcMeminfo(logLevel);-- if (cgroupMemory > nodeMemory * 1.25 && Number.isFinite(cgroupMemory)) {+ if (cgroupMemory > hostMemory * 1.25) {Log.warn({indent: false, logLevel}, 'Detected differing memory amounts:');Log.warn({indent: false, logLevel},`Memory reported by CGroup: ${(cgroupMemory / 1024 / 1024).toFixed(2)} MB`,);- if (_procInfo !== null) {- Log.warn(- {indent: false, logLevel},- `Memory reported by /proc/meminfo: ${(_procInfo / 1024 / 1024).toFixed(2)} MB`,- );- }-Log.warn({indent: false, logLevel},- `Memory reported by Node: ${(nodeMemory / 1024 / 1024).toFixed(2)} MB`,+ `Memory reported by ${source}: ${(hostMemory / 1024 / 1024).toFixed(2)} MB`,);Log.warn({indent: false, logLevel},@@ -53,14 +52,19 @@ export const getAvailableMemory = (logLevel: LogLevel) => {);}- return Math.min(nodeMemory, cgroupMemory);+ availableMemory = Math.min(availableMemory, Math.max(0, cgroupMemory));+ source += ', bounded by remaining cgroup memory';}- const procInfo = getFreeMemoryFromProcMeminfo(logLevel);-- if (procInfo !== null) {- return Math.min(freemem(), procInfo);+ const maxLambdaMemory = getMaxLambdaMemory();+ if (maxLambdaMemory !== null && Number.isFinite(maxLambdaMemory)) {+ availableMemory = Math.min(availableMemory, Math.max(0, maxLambdaMemory));+ source += ', bounded by Lambda memory';}- return freemem();+ Log.verbose(+ {indent: false, logLevel},+ `Available memory: ${(availableMemory / 1024 / 1024).toFixed(2)} MiB (${source})`,+ );+ return availableMemory;};