Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Bench: keep the silhouette score when the look images fail (issue #548)

Nguyen Xuan An47e04cbe3b67
diff --git a/bench/agent-modeling/look.mjs b/bench/agent-modeling/look.mjs
index 9e171436..86a2370b 100644
--- a/bench/agent-modeling/look.mjs
+++ b/bench/agent-modeling/look.mjs
@@ -118,14 +118,19 @@ export async function renderLookImages(jobs) {
});
}
-/** The look record of one finished task: silhouette scores and the images they came from. Judges are added later. */
-export async function scoreLook(taskId, recipe, { imageDirectory } = {}) {
+/** The look record of one finished task: silhouette scores and the images they came from (or imagesError). Judges are added later. */
+export async function scoreLook(taskId, recipe, { imageDirectory, renderImages = renderLookImages } = {}) {
const silhouette = await scoreSilhouette(taskId, recipe);
if (!silhouette) return { skipped: 'asset recipes have no static silhouette' };
const look = { silhouette };
if (imageDirectory) {
- const [images] = await renderLookImages([{ recipe, taskId, prefix: taskId, directory: imageDirectory }]);
- look.images = images;
+ // The silhouette needs no picture, so a failed image step is recorded and the score is kept.
+ try {
+ const [images] = await renderImages([{ recipe, taskId, prefix: taskId, directory: imageDirectory }]);
+ look.images = images;
+ } catch (error) {
+ look.imagesError = error.message.split('\n')[0];
+ }
}
return look;
}
diff --git a/changes/548.md b/changes/548.md
new file mode 100644
index 00000000..b4a0b7a2
--- /dev/null
+++ b/changes/548.md
@@ -0,0 +1 @@
+- Keep the silhouette score when the look images fail (issue #548): `scoreLook` now records `imagesError` instead of dropping the whole look record.
diff --git a/tests/agent-look-score.test.js b/tests/agent-look-score.test.js
index ea5975b8..5214df94 100644
--- a/tests/agent-look-score.test.js
+++ b/tests/agent-look-score.test.js
@@ -1,7 +1,7 @@
import assert from 'node:assert/strict';
import test from 'node:test';
import { judgeRecord } from '../bench/agent-modeling/judge.mjs';
-import { scoreSilhouette } from '../bench/agent-modeling/look.mjs';
+import { scoreLook, scoreSilhouette } from '../bench/agent-modeling/look.mjs';
import { REFERENCES } from '../bench/agent-modeling/references.mjs';
import { intersectionOverUnion, rasterizeSilhouette } from '../bench/agent-modeling/silhouette.mjs';
import { TASKS } from '../bench/agent-modeling/tasks.mjs';
@@ -70,3 +70,13 @@ test('a judge record needs four whole scores in range and keeps its criteria apa
assert.throws(() => judgeRecord([4, 0, 0, 0]), RangeError);
assert.throws(() => judgeRecord([1.5, 0, 0, 0]), RangeError);
});
+
+test('a failing image step keeps the silhouette score and records imagesError', async () => {
+ const failingImages = async () => {
+ throw new Error('could not fetch the boolean module\nstack detail');
+ };
+ const look = await scoreLook('table', REFERENCES.table, { imageDirectory: 'unused', renderImages: failingImages });
+ assert.equal(look.silhouette.mean, 1);
+ assert.equal(look.imagesError, 'could not fetch the boolean module');
+ assert.equal(look.images, undefined);
+});