3D engine that AI agents drive
JavaScript 3d3d-engineagentic-aiai-agentsblenderclaude-codegame-enginegaussian-splattinggltfjavascriptmcpmcp-servermodel-context-protocolpath-tracerpath-tracingpbrrenderingthreejs-alternativewebgl2webgpu
Release read render jobs when the store is full (#2286)
Nguyen Xuan An6872667d39f8
diff --git a/changes/2286.md b/changes/2286.mdnew file mode 100644index 00000000..6a8b06bb--- /dev/null+++ b/changes/2286.md@@ -0,0 +1 @@+- A render job store that is full releases completed jobs whose result was already read, oldest first, before it refuses a new job, and the refusal now names `scene_render_release` with the MiB needed and free (#2286)diff --git a/docs/agent-render-jobs.md b/docs/agent-render-jobs.mdindex ea900b5e..5d918837 100644--- a/docs/agent-render-jobs.md+++ b/docs/agent-render-jobs.md@@ -423,6 +423,7 @@ Admission reserves the maximum output budget plus snapshot and metadata space.The optional `limits` object can reduce controller budgets for bounded tests.It cannot increase production limits.Partial files count toward the store budget and remain until explicit release.+Each submit needs room for its snapshot and a full output reservation (128 MiB by default). When the store is too full, the controller first releases completed jobs whose `scene_render_result` was already read, oldest first, and stops as soon as the new job fits; the files of a released job are gone, so copy outputs you keep before you submit the next job. A job whose result was never read is never released this way. If the job still does not fit, the error says how many MiB it needs and how many are free and names `scene_render_release`. A job read before a server restart counts as unread again.## CPU checksdiff --git a/scripts/agent-render-jobs.mjs b/scripts/agent-render-jobs.mjsindex ca493d79..09a3b629 100644--- a/scripts/agent-render-jobs.mjs+++ b/scripts/agent-render-jobs.mjs@@ -767,6 +767,41 @@ export async function createAgentRenderJobs({ root, run, acquireGpu, limits: inpreturn bytes;}+ async function releaseFinishedJob(jobId, job) {+ await job.persistence.catch(() => {});+ // A retry after a failed release finds a half-deleted folder: finish it here instead of failing on the missing owner file.+ try {+ if (!await discardRemnant(jobId)) await removeJobDirectory(await directoryFor(jobId));+ } catch (error) {+ if (!FILE_IN_USE_CODES.has(error.code)) throw error;+ throw new Error(`Job ${jobId} is only partly released: a file in ${join(root, jobId)} is open in another program. Close it and call release again.`, { cause: error });+ }+ jobs.delete(jobId);+ }++ /** Completed jobs whose result was read, oldest first: the only jobs the store may release on its own. */+ function fetchedJobsOldestFirst() {+ return [...jobs.entries()].filter(([, job]) => job.fetched && job.record.state === 'completed' && active !== job);+ }++ /**+ * Frees `neededBytes` for a new job. When the store is too full, it releases jobs whose output was already read through+ * result(), oldest first, and stops as soon as the job fits. A job whose output was never read stays.+ */+ async function makeRoomInStore(neededBytes) {+ let existingBytes = await storeBytes();+ const releasable = fetchedJobsOldestFirst();+ while (existingBytes + neededBytes > limits.maxStoreBytes && releasable.length) {+ const [jobId, job] = releasable.shift();+ await releaseFinishedJob(jobId, job);+ existingBytes = await storeBytes();+ }+ if (existingBytes + neededBytes <= limits.maxStoreBytes) return;+ const freeMiB = Math.max(0, Math.floor((limits.maxStoreBytes - existingBytes) / 1024 ** 2));+ const neededMiB = Math.ceil(neededBytes / 1024 ** 2);+ throw new RangeError(`Job store lacks space for the snapshot and output reservation: it needs ${neededMiB} MiB free and has ${freeMiB} MiB. Call scene_render_release on finished jobs you no longer need (jobs whose result you already read are released automatically, oldest first).`);+ }+async function storeBytes() {let bytes = 0;for (const name of await readdir(root)) {@@ -1022,8 +1057,8 @@ export async function createAgentRenderJobs({ root, run, acquireGpu, limits: inpconst operation = (async () => {if (jobs.size >= limits.maxRecords) throw new RangeError('Job record budget exhausted. Release a finished job.');const snapshotText = JSON.stringify(snapshotMetadata);- const existingBytes = await storeBytes();- if (existingBytes + Buffer.byteLength(snapshotText) + (snapshot.assetBytes?.length ?? 0) + limits.maxOutputBytes + 131072 > limits.maxStoreBytes) throw new RangeError('Job store lacks space for the snapshot and output reservation.');+ const neededBytes = Buffer.byteLength(snapshotText) + (snapshot.assetBytes?.length ?? 0) + limits.maxOutputBytes + 131072;+ await makeRoomInStore(neededBytes);const jobId = randomUUID();const directory = join(root, jobId);const staging = join(root, `.pending-${storeId}-${jobId}`);@@ -1059,23 +1094,17 @@ export async function createAgentRenderJobs({ root, run, acquireGpu, limits: inpreturn structuredClone(job.record);},result(jobId) {- const record = getJob(jobId).record;+ const job = getJob(jobId);+ const record = job.record;if (record.state !== 'completed') throw new Error(`Render job is ${record.state}. No completed result is available.`);+ job.fetched = true;return structuredClone({ jobId, sourceRevision: record.sourceRevision, artifacts: record.artifacts, metrics: record.metrics });},async release(jobId) {if (closed) throw new Error('Render job controller is closed.');const job = getJob(jobId);if (active === job || !TERMINAL.has(job.record.state)) throw new Error('Cancel and settle the active job before release.');- await job.persistence.catch(() => {});- // A retry after a failed release finds a half-deleted folder: finish it here instead of failing on the missing owner file.- try {- if (!await discardRemnant(jobId)) await removeJobDirectory(await directoryFor(jobId));- } catch (error) {- if (!FILE_IN_USE_CODES.has(error.code)) throw error;- throw new Error(`Job ${jobId} is only partly released: a file in ${join(root, jobId)} is open in another program. Close it and call release again.`, { cause: error });- }- jobs.delete(jobId);+ await releaseFinishedJob(jobId, job);return { jobId, released: true };},async close() {diff --git a/tests/agent-render-job-store.test.js b/tests/agent-render-job-store.test.jsnew file mode 100644index 00000000..356ac3c4--- /dev/null+++ b/tests/agent-render-job-store.test.js@@ -0,0 +1,62 @@+import assert from 'node:assert/strict';+import test from 'node:test';+import { mkdtemp, rm, writeFile, access } from 'node:fs/promises';+import { tmpdir } from 'node:os';+import { join } from 'node:path';++import { createAgentRenderJobs } from '../scripts/agent-render-jobs.mjs';++const OUTPUT_BYTES_PER_JOB = 160 * 1024;+const limits = { maxOutputBytes: 300 * 1024, maxFrameBytes: 200 * 1024, maxStoreBytes: 1024 ** 2, cleanupMs: 100 };+const submission = { sourceRevision: 1, snapshot: { recipe: { format: 'fixture' } }, settings: { kind: 'image', width: 64, height: 64 } };++async function writeOutputRunner({ directory, progress }) {+ await writeFile(join(directory, 'render.png.partial'), Buffer.alloc(OUTPUT_BYTES_PER_JOB, 1));+ await progress({ frames: 1, maximumPendingFrames: 1 });+ return { artifacts: [{ temporary: 'render.png.partial', filename: 'render.png', mimeType: 'image/png' }], metrics: {} };+}++async function withController(context) {+ const root = await mkdtemp(join(tmpdir(), 'crucible-render-store-'));+ const controller = await createAgentRenderJobs({ root, limits, run: writeOutputRunner });+ context.after(async () => {+ await controller.close();+ await rm(root, { recursive: true, force: true });+ });+ return { controller, root };+}++async function submitAndSettle(controller) {+ const { jobId } = await controller.submit(submission);+ while (!['completed', 'failed', 'cancelled'].includes(controller.status(jobId).state)) await new Promise(resolve => setTimeout(resolve, 2));+ assert.equal(controller.status(jobId).state, 'completed', controller.status(jobId).reason);+ return jobId;+}++test('a full store releases completed jobs whose result was read, oldest first, before it refuses', async context => {+ const { controller, root } = await withController(context);+ const jobIds = [];+ for (let index = 0; index < 4; index += 1) {+ const jobId = await submitAndSettle(controller);+ controller.result(jobId);+ jobIds.push(jobId);+ }+ const fifth = await submitAndSettle(controller);+ const remaining = controller.list().map(record => record.jobId);+ assert.ok(!remaining.includes(jobIds[0]), 'the oldest fetched job is released');+ assert.ok(remaining.includes(fifth));+ await assert.rejects(access(join(root, jobIds[0])), { code: 'ENOENT' });+ assert.ok(remaining.includes(jobIds[3]), 'newer fetched jobs stay while the oldest release is enough');+});++test('a job whose result was never read is never released to make room, and the refusal names the release tool and the sizes', async context => {+ const { controller } = await withController(context);+ const unread = [];+ for (let index = 0; index < 4; index += 1) unread.push(await submitAndSettle(controller));+ await assert.rejects(controller.submit(submission), error => {+ assert.match(error.message, /scene_render_release/);+ assert.match(error.message, /needs \d+ MiB free and has \d+ MiB/);+ return true;+ });+ assert.deepEqual(controller.list().map(record => record.jobId), unread);+});