Skip to content

🎥 Make videos programmatically with React

Internal: Reduce local docs build concurrency

Igor Samokhovetsd61dbfca4dd2
diff --git a/packages/docs/build-concurrency.ts b/packages/docs/build-concurrency.ts
new file mode 100644
index 0000000000..8d65e60bcc
--- /dev/null
+++ b/packages/docs/build-concurrency.ts
@@ -0,0 +1,12 @@
+import {availableParallelism, cpus} from 'os';
+
+const cpuCount =
+ typeof availableParallelism === 'function'
+ ? availableParallelism()
+ : cpus().length;
+
+// Leave CPU capacity for other work during local docs builds.
+export const defaultDocsBuildConcurrency = Math.max(
+ 1,
+ Math.min(4, Math.floor(cpuCount / 2)),
+);
diff --git a/packages/docs/build-docs.ts b/packages/docs/build-docs.ts
index feeb725fe0..7f696241d4 100644
--- a/packages/docs/build-docs.ts
+++ b/packages/docs/build-docs.ts
@@ -1,4 +1,5 @@
import {spawn} from 'child_process';
+import {defaultDocsBuildConcurrency} from './build-concurrency';
const isVercel = process.env.VERCEL === '1' || process.env.VERCEL === 'true';
const lowMemoryBuild =
@@ -47,7 +48,18 @@ const run = (
const start = performance.now();
console.log(`[docs build] Starting ${label}...`);
const child = spawn(command, args, {
- env: {...process.env, ...env},
+ env: {
+ ...process.env,
+ // Rspack, SWC and Lightning CSS use Rayon for native parallel work.
+ ...(!isVercel
+ ? {
+ RAYON_NUM_THREADS:
+ process.env.RAYON_NUM_THREADS ??
+ String(defaultDocsBuildConcurrency),
+ }
+ : null),
+ ...env,
+ },
shell: false,
stdio: 'inherit',
});
@@ -99,10 +111,11 @@ const docusaurusEnv = {
DOCUSAURUS_PERF_LOGGER:
process.env.DOCUSAURUS_PERF_LOGGER ?? (isVercel ? 'true' : undefined),
NODE_OPTIONS: appendNodeOption(`--max-old-space-size=${nodeOldSpaceSize}`),
+ DOCUSAURUS_SSG_WORKER_THREAD_COUNT:
+ process.env.DOCUSAURUS_SSG_WORKER_THREAD_COUNT ??
+ String(lowMemoryBuild ? 1 : defaultDocsBuildConcurrency),
...(lowMemoryBuild
? {
- DOCUSAURUS_SSG_WORKER_THREAD_COUNT:
- process.env.DOCUSAURUS_SSG_WORKER_THREAD_COUNT ?? '1',
DOCUSAURUS_SSG_WORKER_THREAD_RECYCLER_MAX_MEMORY:
process.env.DOCUSAURUS_SSG_WORKER_THREAD_RECYCLER_MAX_MEMORY ??
String(512 * 1024 * 1024),
diff --git a/packages/docs/prewarm-twoslash.ts b/packages/docs/prewarm-twoslash.ts
index 3923074d4d..f4a3283f3a 100644
--- a/packages/docs/prewarm-twoslash.ts
+++ b/packages/docs/prewarm-twoslash.ts
@@ -2,7 +2,7 @@ import type {ChildProcessWithoutNullStreams} from 'child_process';
import {spawn} from 'child_process';
import {existsSync, mkdirSync, readdirSync, readFileSync, unlinkSync} from 'fs';
import {createRequire} from 'module';
-import {availableParallelism, cpus, totalmem} from 'os';
+import {totalmem} from 'os';
import {basename, dirname, join, resolve} from 'path';
import {createInterface} from 'readline';
import {Glob} from 'bun';
@@ -14,6 +14,7 @@ import {
readTwoslashCacheEntry,
} from '../docusaurus-plugin/src/twoslash-cache';
import {isTwoslashEnabled} from '../docusaurus-plugin/src/twoslash-enabled';
+import {defaultDocsBuildConcurrency} from './build-concurrency';
import {expandElementSourceReferences} from './plugins/element-source-utils';
const DOCS_ROOT = resolve(import.meta.dirname);
@@ -29,11 +30,6 @@ const GIB = 1024 * 1024 * 1024;
const isVercel = process.env.VERCEL === '1' || process.env.VERCEL === 'true';
const lowMemoryBuild =
isVercel || process.env.REMOTION_DOCS_LOW_MEMORY_BUILD === '1';
-const cpuCount =
- typeof availableParallelism === 'function'
- ? availableParallelism()
- : cpus().length;
-const cpuCap = Math.max(1, cpuCount - 1);
// Each worker holds a TypeScript language service that can grow to >1GB.
// Cap the worker count so the combined RSS leaves room for the OS and
// other processes on low-memory machines.
@@ -46,7 +42,7 @@ const NUM_WORKERS =
? Math.max(1, workerCountOverride)
: lowMemoryBuild
? 2
- : Math.min(cpuCap, memCap);
+ : Math.min(defaultDocsBuildConcurrency, memCap);
// Per-worker RSS threshold after which a worker is replaced with a fresh
// process. Derived from a global budget that leaves 25% of memory for the OS
diff --git a/turbo.json b/turbo.json
index b05c50dc0e..40a8f5b5ec 100644
--- a/turbo.json
+++ b/turbo.json
@@ -131,7 +131,9 @@
"REMOTION_DOCS_ENABLE_TWOSLASH"
],
"passThroughEnv": [
+ "DOCUSAURUS_SSG_WORKER_THREAD_COUNT",
"NODE_OPTIONS",
+ "RAYON_NUM_THREADS",
"REMOTION_DOCS_LOW_MEMORY_BUILD",
"TWOSLASH_RECYCLE_LIMIT_BYTES",
"TWOSLASH_STALL_TIMEOUT_MS",