Skip to content

armgit / 67.js

Public Imported from GitHub
0

3D engine that AI agents drive

Add the job material identity and Cycles parity checks (#1036)

Nguyen Xuan An014283bbb211
diff --git a/bench/job-materials/figure.mjs b/bench/job-materials/figure.mjs
new file mode 100644
index 00000000..e6d4bd1e
--- /dev/null
+++ b/bench/job-materials/figure.mjs
@@ -0,0 +1,48 @@
+// The before / after / Cycles picture of one parity scene as a JPEG.
+// node bench/job-materials/figure.mjs <folder> <scene> <output.jpg>
+// Reads <scene>-before.npy, <scene>-cook.npy and <scene>-cycles.npy (linear RGB, rows from the top), shows them with the
+// display transform of a job (ACES, gamma 2.2) side by side, left to right, and writes a JPEG under 400 KB.
+import { readFileSync, writeFileSync } from 'node:fs';
+import { join } from 'node:path';
+import sharp from 'sharp';
+import { toneMapToRgba8 } from '../../src/path-tracer.js';
+
+function readNpy(path) {
+ const bytes = readFileSync(path);
+ const headerLength = bytes.readUInt16LE(8);
+ const header = bytes.toString('ascii', 10, 10 + headerLength);
+ const [height, width] = header.match(/\((\d+), (\d+), 3\)/).slice(1).map(Number);
+ const data = new Float32Array(bytes.buffer.slice(bytes.byteOffset + 10 + headerLength, bytes.byteOffset + bytes.length));
+ return { width, height, data };
+}
+
+const LABEL_HEIGHT = 28;
+
+async function panel(path, label) {
+ const { width, height, data } = readNpy(path);
+ const rgba = toneMapToRgba8(data, width, height);
+ const labelSvg = Buffer.from(`<svg xmlns="http://www.w3.org/2000/svg" width="${width}" height="${LABEL_HEIGHT}"><rect width="100%" height="100%" fill="#111"/>`
+ + `<text x="10" y="19" font-family="sans-serif" font-size="15" fill="#eee">${label}</text></svg>`);
+ const image = await sharp(Buffer.from(rgba.buffer), { raw: { width, height, channels: 4 } }).removeAlpha().png().toBuffer();
+ return { width, height: height + LABEL_HEIGHT, input: await sharp({ create: { width, height: height + LABEL_HEIGHT, channels: 3, background: '#111' } })
+ .composite([{ input: image, top: 0, left: 0 }, { input: labelSvg, top: height, left: 0 }]).png().toBuffer() };
+}
+
+const [folder, scene, output] = process.argv.slice(2);
+if (!folder || !scene || !output) throw new RangeError('Usage: figure.mjs <folder> <scene> <output.jpg>');
+const panels = [await panel(join(folder, `${scene}-before.npy`), 'before'), await panel(join(folder, `${scene}-cook.npy`), 'after'), await panel(join(folder, `${scene}-cycles.npy`), 'Cycles 4096 spp')];
+const totalWidth = panels.reduce((sum, entry) => sum + entry.width, 0);
+let left = 0;
+const composites = panels.map(entry => {
+ const placed = { input: entry.input, top: 0, left };
+ left += entry.width;
+ return placed;
+});
+for (const quality of [85, 75, 65, 55, 45]) {
+ const jpeg = await sharp({ create: { width: totalWidth, height: panels[0].height, channels: 3, background: '#111' } }).composite(composites).jpeg({ quality }).toBuffer();
+ if (jpeg.length < 400 * 1024 || quality === 45) {
+ writeFileSync(output, jpeg);
+ console.log(JSON.stringify({ output, bytes: jpeg.length, quality }));
+ break;
+ }
+}
diff --git a/bench/job-materials/job-run.mjs b/bench/job-materials/job-run.mjs
new file mode 100644
index 00000000..e24ecccd
--- /dev/null
+++ b/bench/job-materials/job-run.mjs
@@ -0,0 +1,64 @@
+// Real native render jobs through the MCP server of one checkout, to compare two checkouts byte for byte.
+// node bench/job-materials/job-run.mjs <mcp entry> <output folder> <samples> <scene...> [--browser <chrome path>]
+// Each scene of scenes.mjs `recipeFor` is rendered as one native path-traced image job. <scene>.png and <scene>.json (the job
+// report and the SHA-256 of the PNG) land in the output folder; run it once per checkout and compare the digests.
+import { createHash } from 'node:crypto';
+import { mkdirSync, mkdtempSync, readFileSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join, resolve } from 'node:path';
+import { setTimeout as delay } from 'node:timers/promises';
+import { Client } from '@modelcontextprotocol/client';
+import { StdioClientTransport } from '@modelcontextprotocol/client/stdio';
+import { recipeFor } from './scenes.mjs';
+
+const WIDTH = 192;
+const HEIGHT = 128;
+const [entry, outputFolder, samples, ...rest] = process.argv.slice(2);
+const browserIndex = rest.indexOf('--browser');
+const browser = browserIndex >= 0 ? rest[browserIndex + 1] : undefined;
+const scenes = rest.filter((value, index) => value !== '--browser' && index !== browserIndex + 1);
+if (!entry || !outputFolder || !samples || scenes.length === 0) {
+ throw new RangeError('Usage: job-run.mjs <mcp entry> <output folder> <samples> <scene...> [--browser <chrome path>]');
+}
+
+async function renderScene(sceneName) {
+ const workFolder = mkdtempSync(join(tmpdir(), '67js-job-materials-'));
+ const transport = new StdioClientTransport({ command: process.execPath,
+ args: [resolve(entry), '--backend', 'gpu', '--memory', '192', '--jobs', join(workFolder, 'jobs'), ...(browser ? ['--browser', browser] : [])],
+ cwd: workFolder, env: { ...process.env }, stderr: 'pipe' });
+ let diagnostics = '';
+ transport.stderr.on('data', bytes => { diagnostics = (diagnostics + bytes).slice(-4096); });
+ const client = new Client({ name: '67js-job-materials', version: '1' });
+ await client.connect(transport);
+ const call = async (name, argumentsValue) => {
+ const result = await client.callTool({ name, arguments: argumentsValue }, { timeout: 120000 });
+ if (result.isError) throw new Error(`${name}: ${result.content?.find(item => item.type === 'text')?.text}`);
+ return result.structuredContent ?? result;
+ };
+ try {
+ const created = await call('scene_create', { recipe: recipeFor(sceneName) });
+ const submitted = await call('scene_render_submit', { sessionId: created.sessionId, expectedRevision: created.revision ?? 0, settings: {
+ kind: 'image', width: WIDTH, height: HEIGHT,
+ renderer: { type: 'pathTracer', backend: 'native', samples: Number(samples), bounces: 4, denoise: false, seed: 1, maxSeconds: 300 },
+ } });
+ let record;
+ for (let attempt = 0; attempt < 1200; attempt++) {
+ record = await call('scene_render_status', { jobId: submitted.jobId });
+ if (['completed', 'failed', 'cancelled', 'interrupted'].includes(record.state)) break;
+ await delay(500);
+ }
+ if (record.state !== 'completed') throw new Error(`job ${record.state}: ${record.reason ?? ''} ${diagnostics}`);
+ const result = await call('scene_render_result', { jobId: submitted.jobId });
+ const bytes = readFileSync(result.artifacts[0].path);
+ writeFileSync(join(outputFolder, `${sceneName}.png`), bytes);
+ const summary = { scene: sceneName, pngSha256: createHash('sha256').update(bytes).digest('hex'), report: result.metrics?.pathTracer };
+ writeFileSync(join(outputFolder, `${sceneName}.json`), JSON.stringify(summary, null, 2));
+ console.log(JSON.stringify({ scene: sceneName, pngSha256: summary.pngSha256, backend: summary.report?.backend, nativeBackend: summary.report?.nativeBackend,
+ kernel: summary.report?.kernel, instances: summary.report?.instances, instancing: summary.report?.instancing, ignored: summary.report?.ignored }));
+ } finally {
+ await client.close();
+ }
+}
+
+mkdirSync(outputFolder, { recursive: true });
+for (const sceneName of scenes) await renderScene(sceneName);
diff --git a/bench/job-materials/parity.mjs b/bench/job-materials/parity.mjs
new file mode 100644
index 00000000..9249a1a4
--- /dev/null
+++ b/bench/job-materials/parity.mjs
@@ -0,0 +1,60 @@
+// The material parity of a native job against Cycles, per material field.
+// node bench/job-materials/parity.mjs describe <folder> [scene...]
+// Writes <scene>.json, the scene description render-cycles.py reads (bench/cycles-parity).
+// node bench/job-materials/parity.mjs cook|before <folder> <samples> [scene...]
+// Reads <scene>.json, exports the scene the way a native render job does (FORMAT-v2 tables for the "cycles" kernel),
+// traces it with the runner function of the job (traceFrameNative) and writes <scene>-cook.npy (linear RGB); `before` writes <scene>-before.npy from the v1 tables.
+import { mkdir, readFile, writeFile } from 'node:fs/promises';
+import { join, resolve } from 'node:path';
+import { pathToFileURL } from 'node:url';
+import { PARITY_SCENE_NAMES, parityScene } from './scenes.mjs';
+import { reconstructTraceSceneSource, serializeSceneDescription } from '../cycles-parity/scene-bridge.mjs';
+import { writeFloat32Npy } from '../cycles-parity/npy.mjs';
+
+export const WIDTH = 192;
+export const HEIGHT = 128;
+
+async function describe(folder, names) {
+ for (const name of names) {
+ const description = await serializeSceneDescription(name, parityScene(name));
+ await writeFile(join(folder, `${name}.json`), JSON.stringify(description));
+ }
+}
+
+// `before` exports the way a job did before #1036: the v1 tables, which drop the fields the kernel now receives.
+async function cook(folder, samples, names, before) {
+ const [{ exportNativeSceneTables }, { createCamera }, pack, { buildTraceScene, refuseLocalVolumes, resolveTexture }, { buildNativeInput }, jobInput, { traceFrameNative }, tracer] = await Promise.all([
+ import('../../scripts/native-bake-page.js'), import('../../src/path-tracer.js'), import('../../src/path-tracer-gpu-pack.js'),
+ import('../../src/path-tracer-scene.js'), import('../../packages/render-native/pack-input.mjs'), import('../../packages/render-native/job-input.mjs'),
+ import('../../scripts/agent-native-trace.mjs'), import('../../packages/render-native/index.mjs'),
+ ]);
+ const modules = { buildTraceScene, refuseLocalVolumes, resolveTexture, buildNativeInput, buildJobNativeInput: jobInput.buildJobNativeInput,
+ buildFlattenedJobNativeInput: before ? () => { throw new RangeError('before #1036'); } : jobInput.buildFlattenedJobNativeInput, pack };
+ for (const name of names) {
+ const description = JSON.parse(await readFile(join(folder, `${name}.json`), 'utf8'));
+ const source = await reconstructTraceSceneSource(description);
+ const camera = createCamera(source.camera, WIDTH, HEIGHT);
+ const options = { width: WIDTH, height: HEIGHT, bounces: 8, seed: 1, fireflyClamp: 20, noiseThreshold: 0, samples };
+ const exportedTables = exportNativeSceneTables({ parts: source.parts, environment: source.environment, camera, options, kernel: 'cycles', instancing: true, modules });
+ const exported = JSON.parse(JSON.stringify({ ...exportedTables, ignored: [] }));
+ let traced;
+ const recordingTracer = { ...tracer, traceNative(tables, traceOptions) {
+ traced = tracer.traceNative(tables, traceOptions);
+ return traced;
+ } };
+ const result = await traceFrameNative({ tracer: recordingTracer, exported, width: WIDTH, height: HEIGHT, samples, bounces: 8, seed: 1, denoise: 'none', kernel: 'cycles' });
+ const linear = new Float32Array(WIDTH * HEIGHT * 3);
+ for (let pixel = 0; pixel < WIDTH * HEIGHT; pixel++) linear.set(traced.rgba.subarray(pixel * 4, pixel * 4 + 3), pixel * 3);
+ await writeFloat32Npy(join(folder, `${name}-${before ? 'before' : 'cook'}.npy`), linear, [HEIGHT, WIDTH, 3]);
+ console.log(JSON.stringify({ scene: name, wholeMaterials: exported.wholeMaterials, schemaVersion: exported.schemaVersion, tracedSeconds: result.report.tracedSeconds }));
+ }
+}
+
+const [command, folderArgument, ...rest] = process.argv.slice(2);
+if (process.argv[1] && import.meta.url === pathToFileURL(process.argv[1]).href) {
+ const folder = resolve(folderArgument ?? '');
+ await mkdir(folder, { recursive: true });
+ if (command === 'describe') await describe(folder, rest.length ? rest : PARITY_SCENE_NAMES);
+ else if (command === 'cook' || command === 'before') await cook(folder, Number(rest[0]), rest.length > 1 ? rest.slice(1) : PARITY_SCENE_NAMES, command === 'before');
+ else throw new RangeError('Usage: describe <folder> [scene...], or cook|before <folder> <samples> [scene...].');
+}
diff --git a/bench/job-materials/scenes.mjs b/bench/job-materials/scenes.mjs
new file mode 100644
index 00000000..49667184
--- /dev/null
+++ b/bench/job-materials/scenes.mjs
@@ -0,0 +1,101 @@
+// The scenes of the native job material checks.
+// parityScene(name): raw parts, environment and camera (the reference scene shape of bench/reference) for one material
+// field the v1 job tables used to drop, or the composite of a blend pane and a sheen cloth. Cycles renders the same
+// description (bench/cycles-parity/render-cycles.py) and Cook traces it through the export a native job runs.
+// recipeFor(name): a scene recipe for a real MCP job, to compare a job on this branch with one on the base branch.
+import { readFileSync } from 'node:fs';
+import { box, sphere } from '../../src/geometry.js';
+
+export const PARITY_SCENE_NAMES = ['blend', 'sheen', 'specular-tint', 'anisotropy', 'iridescence', 'absorption', 'composite'];
+
+const SUN_COLOR = [3, 2.8, 2.5];
+
+function placed(name, geometry, translation, scale, material) {
+ const matrix = new Float32Array([scale[0], 0, 0, 0, 0, scale[1], 0, 0, 0, 0, scale[2], 0, translation[0], translation[1], translation[2], 1]);
+ return { name, geometry, matrix, material: { tag: 'surface', metallic: 0, roughness: 0.5, emission: 0, ...material } };
+}
+
+function stage() {
+ return [
+ placed('floor', box(0, 1), [0, -1.05, 0], [12, 0.1, 12], { color: [0.5, 0.5, 0.5], roughness: 0.8 }),
+ placed('wall', box(0, 1), [0, 1, -3], [12, 6, 0.1], { color: [0.45, 0.5, 0.55], roughness: 0.9 }),
+ ];
+}
+
+function subject(material) {
+ return placed('subject', sphere(64, 40), [0, 0, 0], [0.9, 0.9, 0.9], material);
+}
+
+function stripes() {
+ return [
+ placed('stripe-red', box(0, 1), [-0.9, 0, -1.2], [0.8, 1.6, 0.1], { color: [0.9, 0.15, 0.1], roughness: 0.6 }),
+ placed('stripe-white', box(0, 1), [0.9, 0, -1.2], [0.8, 1.6, 0.1], { color: [0.9, 0.9, 0.9], roughness: 0.6 }),
+ ];
+}
+
+const pane = () => placed('pane', box(0, 1), [0, 0, 1.2], [2.6, 1.8, 0.04], { color: [0.4, 0.65, 1], roughness: 0.1, opacity: 0.3, alphaMode: 'blend' });
+
+const subjectMaterials = {
+ sheen: { color: [0.25, 0.05, 0.35], roughness: 0.8, sheenColor: [1, 1, 1], sheenRoughness: 0.4 },
+ 'specular-tint': { color: [0.9, 0.8, 0.5], metallic: 1, roughness: 0.3, specular: 0.8, specularColor: [1, 0.5, 0.2] },
+ anisotropy: { color: [0.9, 0.8, 0.5], metallic: 1, roughness: 0.35, anisotropy: 0.7, anisotropyRotation: 0.5 },
+ iridescence: { color: [0.8, 0.8, 0.8], roughness: 0.15, iridescence: 1, iridescenceIor: 1.4, iridescenceThickness: 500 },
+ absorption: { color: [1, 1, 1], roughness: 0.02, transmission: 1, ior: 1.45, attenuationDistance: 1, attenuationColor: [0.2, 0.8, 0.4] },
+};
+
+export function parityScene(name) {
+ let parts;
+ if (name === 'blend') parts = [...stage(), ...stripes(), pane()];
+ else if (name === 'composite') parts = [...stage(), ...stripes(), placed('cloth', sphere(64, 40), [0, 0, 0], [0.7, 0.7, 0.7], subjectMaterials.sheen), pane()];
+ else if (subjectMaterials[name]) parts = [...stage(), subject(subjectMaterials[name])];
+ else throw new RangeError(`Unknown parity scene "${name}". Pass one of: ${PARITY_SCENE_NAMES.join(', ')}.`);
+ return {
+ parts,
+ camera: { eye: [0, 0.6, 5.2], target: [0, 0.2, 0], fov: 40, near: 0.1, far: 60 },
+ environment: { world: true, fillIntensity: 0, ambientIntensity: 1, fogDensity: 0, lightDirection: [-3, 6, 4], keyColor: SUN_COLOR, pointLights: [], spotLights: [], areaLights: [] },
+ blenderWorld: { type: 'constant', value: 0.4 },
+ };
+}
+
+export const IDENTITY_SCENE_NAMES = ['room', 'textured', 'shared', 'glass'];
+
+const IDENTITY_MATERIALS = [
+ { id: 'brick', color: [1, 1, 1], roughness: 0.9, pattern: { type: 'bricks', size: 1.5, count: 6, bump: 1.2, seed: 4 } },
+ { id: 'wood', color: [1, 1, 1], roughness: 0.55, pattern: { type: 'planks', size: 1.3, count: 5, bump: 0.8, seed: 3 } },
+ { id: 'stone', color: [0.5, 0.55, 0.6], roughness: 1, pattern: { type: 'concrete', size: 2, bump: 0.5, seed: 9 } },
+ { id: 'red', color: [0.8, 0.2, 0.15], roughness: 0.5 },
+ { id: 'blue', color: [0.15, 0.4, 0.85], roughness: 0.3, metallic: 0.8 },
+ { id: 'glow', color: [1, 0.7, 0.4], roughness: 0.7, emission: 4 },
+ { id: 'glass', color: [1, 1, 1], roughness: 0.05, transmission: 0.9, ior: 1.45 },
+ { id: 'lacquer', color: [0.7, 0.1, 0.1], roughness: 0.3, clearcoat: 0.8, clearcoatRoughness: 0.1 },
+];
+
+function shape(index) {
+ return { id: `solid-${index}`, type: index % 2 ? 'box' : 'sphere', ...(index % 2 ? { bevel: 0 } : { segments: 16 + index, rings: 8 }) };
+}
+
+function recipeOf(geometries, nodes) {
+ return {
+ format: '67js.scene', schemaVersion: 1, geometries, materials: IDENTITY_MATERIALS, nodes,
+ camera: { eye: [0, 0, 14], target: [0, 0, -2], fov: 50, near: 0.1, far: 60 },
+ };
+}
+
+export function recipeFor(name) {
+ if (name === 'room') return JSON.parse(readFileSync(new URL('../../tests/fixtures/demo-room.recipe.json', import.meta.url), 'utf8'));
+ const place = (index, geometry, material, extra = {}) => ({ id: `n-${index}`, geometry, material, translation: [((index % 4) - 1.5) * 1.6, (Math.floor(index / 4) - 1) * 1.6, 0], scale: [0.5, 0.5, 0.5], ...extra });
+ if (name === 'textured') {
+ const materials = ['brick', 'wood', 'stone'];
+ return recipeOf(Array.from({ length: 12 }, (_, index) => shape(index)), Array.from({ length: 12 }, (_, index) => place(index, `solid-${index}`, materials[index % 3])));
+ }
+ if (name === 'glass') {
+ const materials = ['glass', 'lacquer', 'blue', 'glow'];
+ return recipeOf(Array.from({ length: 12 }, (_, index) => shape(index)), Array.from({ length: 12 }, (_, index) => place(index, `solid-${index}`, materials[index % 4])));
+ }
+ if (name === 'shared') {
+ const materials = ['red', 'blue', 'glow', 'stone'];
+ return recipeOf([{ id: 'ball', type: 'sphere', segments: 16, rings: 8 }], Array.from({ length: 40 }, (_, index) => place(index, 'ball', materials[index % 4],
+ { translation: [((index % 8) - 3.5) * 1.1, (Math.floor(index / 8) - 2) * 1.1, 0], scale: [0.4, 0.4, 0.4] })));
+ }
+ throw new RangeError(`Unknown identity scene "${name}". Pass one of: ${IDENTITY_SCENE_NAMES.join(', ')}.`);
+}
diff --git a/bench/job-materials/vast-run.sh b/bench/job-materials/vast-run.sh
new file mode 100644
index 00000000..67b7644a
--- /dev/null
+++ b/bench/job-materials/vast-run.sh
@@ -0,0 +1,63 @@
+#!/usr/bin/env bash
+# The checks of #1036 on a GPU box with Blender, Chrome and the Rust toolchain. Run from the repository root:
+# bash bench/job-materials/vast-run.sh <base branch> <stage...>
+# Stages: build, tests, identity, parity, size. Outputs go to artifacts/job-materials/ (identity-<tree>/, parity/).
+# The base branch is built in a second work tree next to this one, so one run compares real jobs of both.
+set -uo pipefail
+base_branch="$1"
+shift
+out="artifacts/job-materials"
+mkdir -p "$out"
+here="$(pwd)"
+base_tree="$here/../job-materials-base"
+chrome_wrapper="/root/chrome-gpu.sh"
+printf '#!/bin/sh\nexec "%s" --use-angle=vulkan --enable-features=Vulkan --ignore-gpu-blocklist --no-sandbox "$@"\n' "$CHROME_PATH" > "$chrome_wrapper"
+chmod +x "$chrome_wrapper"
+
+build_tree() {
+ (cd "$1" && npm run build >/dev/null 2>&1 && (cd packages/render-native && node build.mjs 2>&1 | tail -2)) || echo "BUILD-FAILED $1"
+}
+
+for stage in "$@"; do
+ echo "=== stage $stage"
+ case "$stage" in
+ build)
+ if [ ! -d "$base_tree" ]; then
+ git fetch -q origin "$base_branch" && git worktree add --force "$base_tree" FETCH_HEAD >/dev/null 2>&1
+ ln -s "$here/node_modules" "$base_tree/node_modules"
+ fi
+ build_tree "$here"
+ build_tree "$base_tree"
+ ;;
+ tests)
+ node --test tests/native-job-materials.test.js tests/native-instancing-fallback.test.js tests/native-instancing-materials.test.js tests/native-instancing-transport.test.js tests/native-instancing-equivalence.test.js 2>&1 | grep -E "^(✖|ℹ (tests|pass|fail|skipped))|Error" | head -20
+ ;;
+ identity)
+ for tree in "$here" "$base_tree"; do
+ label="$(basename "$tree")"
+ echo "--- $label"
+ node bench/job-materials/job-run.mjs "$tree/bin/67js-mcp.mjs" "$out/identity-$label" 256 room textured shared glass --browser "$chrome_wrapper" 2>&1 | tail -12
+ done
+ ;;
+ parity)
+ folder="$out/parity"
+ mkdir -p "$folder"
+ node bench/job-materials/parity.mjs describe "$folder" 2>&1 | tail -3
+ for scene in blend sheen specular-tint anisotropy iridescence absorption composite; do
+ blender -b --factory-startup --python bench/cycles-parity/render-cycles.py -- "$folder/$scene.json" "$folder/$scene-cycles" --samples 4096 --bounces 8 --seed 1 --width 192 --height 128 2>&1 | grep -E "CYCLES_TRACE|Error|Traceback" | cut -c1-200
+ done
+ ls "$folder" | head -40
+ node bench/job-materials/parity.mjs cook "$folder" 4096 2>&1 | tail -10
+ node bench/job-materials/parity.mjs before "$folder" 4096 2>&1 | tail -10
+ for scene in blend sheen specular-tint anisotropy iridescence absorption composite; do
+ for variant in cook before; do
+ python3 bench/cycles-parity/compare.py pair --no-flip --json-out "$folder/$scene-$variant-compare.json" "$folder/$scene-cycles.npy" "$folder/$scene-$variant.npy" >/dev/null 2>&1 || echo "COMPARE-FAILED $scene $variant"
+ done
+ done
+ node bench/job-materials/figure.mjs "$folder" composite "$out/composite.jpg" 2>&1 | tail -2
+ ;;
+ size)
+ npm run size 2>&1 | tail -30
+ ;;
+ esac
+done