3D engine that AI agents drive
Bench: keep the silhouette score when the look images fail (issue #548)
Nguyen Xuan An47e04cbe3b67
diff --git a/bench/agent-modeling/look.mjs b/bench/agent-modeling/look.mjsindex 9e171436..86a2370b 100644--- a/bench/agent-modeling/look.mjs+++ b/bench/agent-modeling/look.mjs@@ -118,14 +118,19 @@ export async function renderLookImages(jobs) {});}-/** The look record of one finished task: silhouette scores and the images they came from. Judges are added later. */-export async function scoreLook(taskId, recipe, { imageDirectory } = {}) {+/** The look record of one finished task: silhouette scores and the images they came from (or imagesError). Judges are added later. */+export async function scoreLook(taskId, recipe, { imageDirectory, renderImages = renderLookImages } = {}) {const silhouette = await scoreSilhouette(taskId, recipe);if (!silhouette) return { skipped: 'asset recipes have no static silhouette' };const look = { silhouette };if (imageDirectory) {- const [images] = await renderLookImages([{ recipe, taskId, prefix: taskId, directory: imageDirectory }]);- look.images = images;+ // The silhouette needs no picture, so a failed image step is recorded and the score is kept.+ try {+ const [images] = await renderImages([{ recipe, taskId, prefix: taskId, directory: imageDirectory }]);+ look.images = images;+ } catch (error) {+ look.imagesError = error.message.split('\n')[0];+ }}return look;}diff --git a/changes/548.md b/changes/548.mdnew file mode 100644index 00000000..b4a0b7a2--- /dev/null+++ b/changes/548.md@@ -0,0 +1 @@+- Keep the silhouette score when the look images fail (issue #548): `scoreLook` now records `imagesError` instead of dropping the whole look record.diff --git a/tests/agent-look-score.test.js b/tests/agent-look-score.test.jsindex ea5975b8..5214df94 100644--- a/tests/agent-look-score.test.js+++ b/tests/agent-look-score.test.js@@ -1,7 +1,7 @@import assert from 'node:assert/strict';import test from 'node:test';import { judgeRecord } from '../bench/agent-modeling/judge.mjs';-import { scoreSilhouette } from '../bench/agent-modeling/look.mjs';+import { scoreLook, scoreSilhouette } from '../bench/agent-modeling/look.mjs';import { REFERENCES } from '../bench/agent-modeling/references.mjs';import { intersectionOverUnion, rasterizeSilhouette } from '../bench/agent-modeling/silhouette.mjs';import { TASKS } from '../bench/agent-modeling/tasks.mjs';@@ -70,3 +70,13 @@ test('a judge record needs four whole scores in range and keeps its criteria apaassert.throws(() => judgeRecord([4, 0, 0, 0]), RangeError);assert.throws(() => judgeRecord([1.5, 0, 0, 0]), RangeError);});++test('a failing image step keeps the silhouette score and records imagesError', async () => {+ const failingImages = async () => {+ throw new Error('could not fetch the boolean module\nstack detail');+ };+ const look = await scoreLook('table', REFERENCES.table, { imageDirectory: 'unused', renderImages: failingImages });+ assert.equal(look.silhouette.mean, 1);+ assert.equal(look.imagesError, 'could not fetch the boolean module');+ assert.equal(look.images, undefined);+});