Skip to content

🎥 Make videos programmatically with React

`@remotion/renderer`: Fix available memory estimates on macOS and Linux

Jonny Burger26318a9d8246
diff --git a/packages/renderer/src/memory/from-macos-sysctl.ts b/packages/renderer/src/memory/from-macos-sysctl.ts
new file mode 100644
index 0000000000..6027bff740
--- /dev/null
+++ b/packages/renderer/src/memory/from-macos-sysctl.ts
@@ -0,0 +1,67 @@
+import {execFileSync} from 'node:child_process';
+import {platform, totalmem} from 'node:os';
+import type {LogLevel} from '../log-level';
+import {Log} from '../logger';
+
+export const getAvailableMemoryFromMacos = (
+ logLevel: LogLevel,
+): number | null => {
+ if (platform() !== 'darwin') {
+ return null;
+ }
+
+ try {
+ const output = execFileSync(
+ '/usr/sbin/sysctl',
+ [
+ '-n',
+ 'vm.page_free_count',
+ 'vm.page_pageable_external_count',
+ 'vm.page_purgeable_count',
+ // hw.pagesize can differ from the kernel page size under Rosetta.
+ 'vm.pagesize',
+ 'kern.memorystatus_vm_pressure_level',
+ ],
+ {encoding: 'utf-8', timeout: 1000, stdio: ['ignore', 'pipe', 'pipe']},
+ );
+ const values = output.trim().split(/\s+/);
+ if (values.length !== 5 || values.some((value) => !/^\d+$/.test(value))) {
+ throw new Error('Unexpected macOS memory information from sysctl');
+ }
+
+ const numbers = values.map(Number);
+ const [free, fileBacked, purgeable, pageSize, pressureLevel] = numbers;
+ if (
+ numbers.some((value) => !Number.isSafeInteger(value)) ||
+ pageSize === 0 ||
+ ![1, 2, 4].includes(pressureLevel)
+ ) {
+ throw new Error('Invalid macOS memory information from sysctl');
+ }
+
+ // The sysctl exposes dispatch pressure flags: normal=1, warning=2, critical=4.
+ if (pressureLevel === 4) {
+ Log.verbose(
+ {indent: false, logLevel},
+ 'macOS memory pressure is critical. Using os.freemem().',
+ );
+ return null;
+ }
+
+ // XNU's fully reclaimable pages exclude anonymous and compressed memory:
+ // https://github.com/apple-oss-distributions/xnu/blob/main/doc/vm/memorystatus_notify.md
+ const availableMemory = (free + fileBacked + purgeable) * pageSize;
+ if (!Number.isSafeInteger(availableMemory)) {
+ throw new Error('Invalid macOS available memory size');
+ }
+
+ return Math.min(availableMemory, totalmem());
+ } catch (err) {
+ Log.verbose(
+ {indent: false, logLevel},
+ 'Could not read reclaimable macOS memory. Using os.freemem().',
+ err,
+ );
+ return null;
+ }
+};
diff --git a/packages/renderer/src/memory/from-proc-meminfo.ts b/packages/renderer/src/memory/from-proc-meminfo.ts
index 941c029e30..7dfb39d4a8 100644
--- a/packages/renderer/src/memory/from-proc-meminfo.ts
+++ b/packages/renderer/src/memory/from-proc-meminfo.ts
@@ -2,7 +2,7 @@ import {existsSync, readFileSync} from 'node:fs';
import type {LogLevel} from '../log-level';
import {Log} from '../logger';
-export const getFreeMemoryFromProcMeminfo = (
+export const getAvailableMemoryFromProcMeminfo = (
logLevel: LogLevel,
): number | null => {
if (!existsSync('/proc/meminfo')) {
@@ -11,7 +11,7 @@ export const getFreeMemoryFromProcMeminfo = (
try {
const data = readFileSync('/proc/meminfo', 'utf-8');
- // Split the file by lines and find the line with MemFree
+ // MemAvailable includes memory that can be reclaimed without swapping.
const lines = data.split('\n');
const memAvailableLine = lines.find((line) =>
line.startsWith('MemAvailable'),
@@ -30,6 +30,9 @@ export const getFreeMemoryFromProcMeminfo = (
const value = parseInt(matches[1], 10);
const unit = matches[2].toLowerCase();
+ if (!Number.isSafeInteger(value)) {
+ throw new Error('Invalid MemAvailable value');
+ }
// Convert the value to bytes based on its unit
switch (unit) {
diff --git a/packages/renderer/src/memory/get-available-memory.ts b/packages/renderer/src/memory/get-available-memory.ts
index e2ff770d4e..55f7c2329d 100644
--- a/packages/renderer/src/memory/get-available-memory.ts
+++ b/packages/renderer/src/memory/get-available-memory.ts
@@ -1,47 +1,46 @@
-import {freemem} from 'node:os';
+import {freemem, platform} from 'node:os';
import type {LogLevel} from '../log-level';
import {Log} from '../logger';
import {getAvailableMemoryFromCgroup} from './from-docker-cgroup';
import {getMaxLambdaMemory} from './from-lambda-env';
-import {getFreeMemoryFromProcMeminfo} from './from-proc-meminfo';
+import {getAvailableMemoryFromMacos} from './from-macos-sysctl';
+import {getAvailableMemoryFromProcMeminfo} from './from-proc-meminfo';
export const getAvailableMemory = (logLevel: LogLevel) => {
- // If we are in Lambda, the most reliable way is to read the environment variable
- // This is also pretty much free
- const maxMemory = getMaxLambdaMemory();
-
- if (maxMemory !== null) {
- const nodeMemory = freemem();
- return Math.min(nodeMemory, maxMemory);
+ let hostMemory = freemem();
+ let source = 'os.freemem()';
+ if (platform() === 'linux') {
+ const procInfo = getAvailableMemoryFromProcMeminfo(logLevel);
+ if (procInfo !== null) {
+ hostMemory = procInfo;
+ source = '/proc/meminfo MemAvailable';
+ }
+ } else if (platform() === 'darwin') {
+ const macosMemory = getAvailableMemoryFromMacos(logLevel);
+ if (macosMemory !== null) {
+ hostMemory = macosMemory;
+ source = 'macOS sysctl (free, file-backed and purgeable pages)';
+ }
}
- // If cgroup memory data is available, we are in Docker, and we should respect it
- const cgroupMemory = getAvailableMemoryFromCgroup();
- if (cgroupMemory !== null) {
+ let availableMemory = hostMemory;
+ const cgroupMemory =
+ platform() === 'linux' ? getAvailableMemoryFromCgroup() : null;
+ if (cgroupMemory !== null && Number.isFinite(cgroupMemory)) {
// There are 2 Docker memory configurations:
// 1. --memory=[num]
// 2. Global Docker memory limit
// If cgroup limit is higher than global memory, the global memory limit still applies
- const nodeMemory = freemem();
- const _procInfo = getFreeMemoryFromProcMeminfo(logLevel);
-
- if (cgroupMemory > nodeMemory * 1.25 && Number.isFinite(cgroupMemory)) {
+ if (cgroupMemory > hostMemory * 1.25) {
Log.warn({indent: false, logLevel}, 'Detected differing memory amounts:');
Log.warn(
{indent: false, logLevel},
`Memory reported by CGroup: ${(cgroupMemory / 1024 / 1024).toFixed(2)} MB`,
);
- if (_procInfo !== null) {
- Log.warn(
- {indent: false, logLevel},
- `Memory reported by /proc/meminfo: ${(_procInfo / 1024 / 1024).toFixed(2)} MB`,
- );
- }
-
Log.warn(
{indent: false, logLevel},
- `Memory reported by Node: ${(nodeMemory / 1024 / 1024).toFixed(2)} MB`,
+ `Memory reported by ${source}: ${(hostMemory / 1024 / 1024).toFixed(2)} MB`,
);
Log.warn(
{indent: false, logLevel},
@@ -53,14 +52,19 @@ export const getAvailableMemory = (logLevel: LogLevel) => {
);
}
- return Math.min(nodeMemory, cgroupMemory);
+ availableMemory = Math.min(availableMemory, Math.max(0, cgroupMemory));
+ source += ', bounded by remaining cgroup memory';
}
- const procInfo = getFreeMemoryFromProcMeminfo(logLevel);
-
- if (procInfo !== null) {
- return Math.min(freemem(), procInfo);
+ const maxLambdaMemory = getMaxLambdaMemory();
+ if (maxLambdaMemory !== null && Number.isFinite(maxLambdaMemory)) {
+ availableMemory = Math.min(availableMemory, Math.max(0, maxLambdaMemory));
+ source += ', bounded by Lambda memory';
}
- return freemem();
+ Log.verbose(
+ {indent: false, logLevel},
+ `Available memory: ${(availableMemory / 1024 / 1024).toFixed(2)} MiB (${source})`,
+ );
+ return availableMemory;
};