Turn one illustration into an animated 2D mesh avatar with a coding agent and a local editor
Switch overlay layers on the Live page and in the stream view
shinshin86ba759d53cf19
diff --git a/README.ja.md b/README.ja.mdindex 995a93d..417fcfa 100644--- a/README.ja.md+++ b/README.ja.md@@ -118,6 +118,8 @@ npm run create -- --random --prompt "銀髪のショートヘア、制服"「ライティング」を有効にすると、ドラッグで動かせる光源に合わせてアバターに陰影が付きます。初期状態はオフです。配信の操作画面の下のほう、またはエディタのプレビューの下にある欄で設定でき、設定はOBS用URLにも含まれます。詳しくは[リファレンス](docs/reference.md#lighting)(英語)を参照してください。+眼鏡などの重ねるレイヤー(`built/overlays/`)があるアバターでは、配信の操作画面の「重ねるレイヤー」で表示を切り替えられます。配信画面にもすぐ反映され、表示中のレイヤーはOBS用URLにも含まれます。+操作画面は専用のウィンドウで開いたままにしてください。OBS用URLはOBSの中か別のタブで開き、操作画面のタブには貼り付けないでください。画面が隠れて追跡が停止・低速になったという表示が出た場合は、操作画面のウィンドウを前面に戻してください。diff --git a/README.md b/README.mdindex 3e4b559..3563b3c 100644--- a/README.md+++ b/README.md@@ -174,6 +174,10 @@ Optional **Lighting** adds shading that follows a light you drag over the avatarby default; open its section near the bottom of the Live page, or below the preview in theeditor. The OBS URL includes the lighting settings. See [Lighting](docs/reference.md#lighting).+If the avatar has overlay layers such as glasses (`built/overlays/`), the **Overlays** section+of the Live page switches them on and off. The stream view follows at once, and the OBS URL+includes the overlays that are shown.+Keep the Live page open in its own window. Open the OBS URL inside OBS or in anothertab; do not paste it into the tab running Live. If the page reports that tracking hasstopped or slowed while hidden, bring its window to the front.diff --git a/README.zh.md b/README.zh.mdindex f6821c7..2e73153 100644--- a/README.zh.md+++ b/README.zh.md@@ -115,6 +115,8 @@ npm run create -- --random --prompt "银色短发,校服"可选的**光照**会在你拖动的光源处生成跟随的阴影。默认关闭;在直播页面底部或编辑器预览下方打开对应区域即可。OBS 地址包含光照设置,详见[光照](docs/reference.md#lighting)。+如果虚拟形象带有眼镜等叠加层(`built/overlays/`),可以在直播页面的**叠加层**中切换显示。直播画面会立即同步,OBS 地址也会包含当前显示的叠加层。+把直播页面放在单独的窗口里保持打开。在 OBS 内部或另一个标签页打开 OBS 地址;不要把它粘到正在运行的直播标签页里。如果页面提示在被隐藏时追踪已停止或变慢,把它切回前台。diff --git a/e2e/live-streaming.spec.ts b/e2e/live-streaming.spec.tsindex 7bebf64..27a9ab8 100644--- a/e2e/live-streaming.spec.ts+++ b/e2e/live-streaming.spec.ts@@ -1,5 +1,7 @@import { expect, test, type BrowserContext, type Page } from '@playwright/test';import { PNG } from 'pngjs';+import { cp, mkdir, mkdtemp, readFile, rm, writeFile } from 'node:fs/promises';+import { basename, join, resolve } from 'node:path';import { samplePresent, sampleSkipReason } from './sample';test.beforeEach(() => { test.skip(!samplePresent, sampleSkipReason); });@@ -205,3 +207,49 @@ test('hidden tracking stall warns in all languages and clears when fresh framesawait page.evaluate(() => { (window as Window & { freezeVideo?: boolean }).freezeVideo = false; });await expect(page.getByTestId('background-status')).toHaveCount(0, { timeout: 5000 });});++test('overlay switches on the Live page reach the stream view and its OBS URL', async ({ context, page }) => {+ const outside = await guardNetwork(context);+ // A copy of the sample with one synthetic overlay: a full-size transparent PNG with a dark block on the face.+ const rig = JSON.parse(await readFile(resolve('samples/miko-qipao/rig.json'), 'utf8'));+ await mkdir(resolve('projects'), { recursive: true });+ const directory = await mkdtemp(join(resolve('projects'), 'live-overlay-')), name = basename(directory);+ try {+ await cp(resolve('samples/miko-qipao'), directory, { recursive: true, filter: path => !path.includes(join(resolve('samples/miko-qipao'), 'review')) });+ const { width, height } = rig.image, overlay = new PNG({ width, height });+ const [x0, y0] = [Math.round(rig.face.nose.cx) - 80, Math.round(rig.face.nose.cy) - 30];+ for (let y = y0; y < y0 + 60; y++) for (let x = x0; x < x0 + 160; x++) overlay.data.set([20, 20, 20, 255], (y * width + x) * 4);+ await mkdir(join(directory, 'built/overlays'), { recursive: true });+ await writeFile(join(directory, 'built/overlays/glasses.png'), PNG.sync.write(overlay));+ await writeFile(join(directory, 'built/overlays/overlays.json'), JSON.stringify({ version: 1, overlays: [{ name: 'glasses', file: 'glasses.png' }] }));++ await ready(page, `/stream.html?project=${name}&bg=green&idle=0`);+ const live = await context.newPage(); await ready(live, `/live.html?project=${name}`);+ const section = live.getByTestId('overlay-section'), toggle = section.getByRole('button', { name: 'glasses', exact: true });+ await expect(toggle).toHaveAttribute('aria-pressed', 'false');+ const canvas = page.locator('#avatar');+ const shot = async () => PNG.sync.read(await canvas.screenshot());+ const changed = (a: PNG, b: PNG) => { let count = 0; for (let i = 0; i < a.data.length; i += 4) if (Math.abs(a.data[i] - b.data[i]) > 40) count++; return count; };+ const before = await shot();+ await toggle.click();+ await expect(toggle).toHaveAttribute('aria-pressed', 'true');+ await expect.poll(async () => changed(before, await shot())).toBeGreaterThan(1000);+ expect(new URL(await live.locator('.obs-url').inputValue()).searchParams.get('overlays')).toBe('glasses');+ // a stream view opened from that URL starts with the overlay shown+ const fromUrl = await context.newPage();+ await ready(fromUrl, `${new URL(await live.locator('.obs-url').inputValue()).pathname}?project=${name}&bg=green&idle=0&overlays=glasses`);+ await expect.poll(async () => changed(before, PNG.sync.read(await fromUrl.locator('#avatar').screenshot()))).toBeGreaterThan(1000);+ const shown = changed(before, await shot());+ await toggle.click();+ await expect(toggle).toHaveAttribute('aria-pressed', 'false');+ // The tassels keep swinging, so hiding returns most of the way rather than to identical pixels.+ await expect.poll(async () => changed(before, await shot())).toBeLessThan(shown / 3);+ // the choice is remembered for the project+ await live.reload(); await expect(live.locator('canvas[data-state="ready"]')).toBeVisible();+ await expect(live.getByTestId('overlay-section').getByRole('button', { name: 'glasses', exact: true })).toHaveAttribute('aria-pressed', 'false');+ // the sample has no overlays, so the section is not shown+ const sample = await context.newPage(); await ready(sample, '/live.html');+ await expect(sample.getByTestId('overlay-section')).toHaveCount(0);+ } finally { await rm(directory, { recursive: true, force: true }); }+ expect(outside).toEqual([]);+});diff --git a/src/live/LiveApp.tsx b/src/live/LiveApp.tsxindex 45ac6cf..b2bf7c0 100644--- a/src/live/LiveApp.tsx+++ b/src/live/LiveApp.tsx@@ -5,17 +5,19 @@ import { loadLightingPresets, sameLighting, saveLightingPresets } from '../lightimport { useEffect, useRef, useState } from 'react';import { useI18n } from '../editor/i18n';import { createAvatarView } from './avatar-view';-import { viewSettings, streamUrl, backgroundColor } from './settings';+import { viewSettings, streamUrl, backgroundColor, loadOverlays, saveOverlays } from './settings';import { FacePose, type TrackingOptions } from './tracking';import { CameraCapture, MicrophoneCapture, type CameraState, type MicState, type BackgroundTracking } from './media';import { liveText } from './i18n';import { Icon } from '../editor/Icon';-import { createLiveSender, receiveExpression, receiveLightingPreset, sendLighting, watchDuplicateLivePages } from './relay';+import { createLiveSender, receiveExpression, receiveLightingPreset, sendLighting, sendOverlays, watchDuplicateLivePages } from './relay';import { EXPRESSION_TOKEN_PATH, LIGHTING_PRESET_PATH } from './expression-protocol';export function LiveApp() {const { language, setLanguage } = useI18n(), t = liveText[language];- const [settings, setSettings] = useState(() => { const view = viewSettings(location.search); return { ...view, lighting: view.lighting ?? loadLighting(view.project) }; });+ const [settings, setSettings] = useState(() => { const view = viewSettings(location.search); return { ...view, lighting: view.lighting ?? loadLighting(view.project), overlays: view.overlays ?? loadOverlays(view.project) }; });+ // Overlay layers the loaded avatar has (built/overlays), listed once it has loaded.+ const [availableOverlays, setAvailableOverlays] = useState<string[]>([]);const [lightingOpen, setLightingOpen] = useState(false);const [presets, setPresets] = useState(() => loadLightingPresets(settings.project));const presetsRef = useRef(presets); presetsRef.current = presets;@@ -36,6 +38,7 @@ export function LiveApp() {const camera = useRef<CameraCapture | null>(null), microphone = useRef<MicrophoneCapture | null>(null);const avatarRef = useRef<MeshAvatar | null>(null);const lightRef = useRef(settings.lighting); lightRef.current = settings.lighting;+ const overlaysRef = useRef(settings.overlays); overlaysRef.current = settings.overlays;const pose = useRef(new FacePose());const controls = useRef({ options, gain, micState, cameraState }); controls.current = { options, gain, micState, cameraState };const refreshDevices = () => { void navigator.mediaDevices?.enumerateDevices().then(setDevices).catch(() => undefined); };@@ -68,7 +71,7 @@ export function LiveApp() {}, (avatar, now) => {// An expression alone also drives the stream view, including its fade back to neutral.if (controls.current.cameraState === 'running' || controls.current.micState === 'micOn' || avatar.getExpression().active) send(avatar.getParameters(), now);- }).then(value => { if (cancelled) value.destroy(); else { view = value; avatarRef.current = value.avatar; value.avatar.setLighting(lightRef.current); value.avatar.setExpression(expressionRef.current); setViewState('ready'); } }).catch(() => { if (!cancelled) setViewState('projectError'); });+ }).then(value => { if (cancelled) value.destroy(); else { view = value; avatarRef.current = value.avatar; value.avatar.setLighting(lightRef.current); value.avatar.setExpression(expressionRef.current); value.avatar.setOverlays(overlaysRef.current); setAvailableOverlays(value.overlays); setViewState('ready'); } }).catch(() => { if (!cancelled) setViewState('projectError'); });return () => { cancelled = true; clock.terminate(); view?.destroy(); avatarRef.current = null; };}, [settings.project, settings.fit]);useEffect(() => {@@ -85,6 +88,13 @@ export function LiveApp() {return () => clearInterval(timer);}, [settings.project]);const changeLighting = (lighting: typeof settings.lighting) => setSettings(current => ({ ...current, lighting }));+ useEffect(() => {+ saveOverlays(settings.project, settings.overlays);+ avatarRef.current?.setOverlays(settings.overlays);+ sendOverlays(settings.project, settings.overlays);+ }, [settings.project, settings.overlays]);+ const toggleOverlay = (name: string) => setSettings(current => ({ ...current,+ overlays: current.overlays.includes(name) ? current.overlays.filter(item => item !== name) : [...current.overlays, name] }));useEffect(() => { avatarRef.current?.setExpression(expression); }, [expression]);useEffect(() => watchDuplicateLivePages(settings.project, setDuplicatePage), [settings.project]);useEffect(() => receiveExpression(settings.project, setExpression), [settings.project]);@@ -175,6 +185,12 @@ export function LiveApp() {</> : <small>{t.remoteUnavailable}</small>}<a href="https://github.com/shinshin86/mesh-avatar-studio/blob/main/docs/reference.md#switching-expressions-from-other-apps" target="_blank" rel="noreferrer">{t.remoteDocs}</a></div></details>+ {availableOverlays.length > 0 && <details className="live-overlays" data-testid="overlay-section" open>+ <summary><Icon name="eye" />{t.overlays}{settings.overlays.some(name => availableOverlays.includes(name)) && <span className="overlays-on">ON</span>}</summary>+ <div className="overlay-buttons" role="group" aria-label={t.overlays}>+ {availableOverlays.map(name => <button key={name} type="button" aria-pressed={settings.overlays.includes(name)} onClick={() => toggleOverlay(name)}>{name}</button>)}+ </div><small>{t.overlaysHint}</small>+ </details>}<details className="live-lighting" data-testid="lighting-section" onToggle={event => setLightingOpen(event.currentTarget.open)}><summary><Icon name="light" />{lightingText[language].title}{settings.lighting.enabled && <span className="lighting-on">ON</span>}</summary><div className="lighting-presets"><h3>{t.presets}</h3>diff --git a/src/live/avatar-view.ts b/src/live/avatar-view.tsindex 25dc0e1..eab49d9 100644--- a/src/live/avatar-view.ts+++ b/src/live/avatar-view.ts@@ -4,12 +4,21 @@ import { SAMPLE_PROJECT, type ViewSettings } from './settings';export const neutralParameters: Record<string, number> = Object.fromEntries(PARAMS.map(param => [param.id, param.def]));export interface FrameTiming { frames: number; totalMs: number; maxMs: number }+/** Names of the avatar's overlay layers (built/overlays/overlays.json), or none. */+async function listOverlays(assets: Record<string, string>): Promise<string[]> {+ const url = assets['overlays/overlays.json'];+ if (!url) return [];+ try {+ const sheet = await (await fetch(url)).json();+ return Array.isArray(sheet?.overlays) ? sheet.overlays.map((item: { name?: unknown }) => item?.name).filter((name: unknown): name is string => typeof name === 'string') : [];+ } catch { return []; }+}export async function createAvatarView(canvas: HTMLCanvasElement, settings: ViewSettings,beforeFrame?: (avatar: MeshAvatar, now: number, dt: number) => void,afterFrame?: (avatar: MeshAvatar, now: number) => void) {canvas.dataset.state = 'loading';const options = { manual: true, fit: settings.fit, preserveMouthForm: true };- let avatar: MeshAvatar;+ let avatar: MeshAvatar, overlays: string[] = [];if (settings.avatar) avatar = await loadMeshAvatar(canvas, settings.avatar, options);else {const projects = await localProjects(), project = projects?.find(entry => entry.name === settings.project);@@ -17,8 +26,10 @@ export async function createAvatarView(canvas: HTMLCanvasElement, settings: Viewconst loaded = project ? await openLocalProject(project) : await openSampleAvatar();const lipSync = 'manifest' in loaded ? loaded.manifest.lipSync : loaded.lipSync;avatar = await createMeshAvatar(canvas, { ...options, lipSync, rig: loaded.rig!, assets: loaded.assets });+ overlays = await listOverlays(loaded.assets);}if (settings.lighting) avatar.setLighting(settings.lighting);+ if (settings.overlays) avatar.setOverlays(settings.overlays);avatar.setAutoIdle(settings.idle); avatar.setAutoMotion(settings.idle);if (!settings.idle) avatar.setParameters(neutralParameters);canvas.dataset.state = 'ready';@@ -35,7 +46,7 @@ export async function createAvatarView(canvas: HTMLCanvasElement, settings: View};const frame = (now: number) => { lastPaint = now; update(now); raf = requestAnimationFrame(frame); };raf = requestAnimationFrame(frame);- return { avatar, timing,+ return { avatar, timing, overlays,paintPaused(now: number) { return now - lastPaint > 1000; },updateIfStalled(now: number) { if (now - previous >= 1000 / 30 - 2) update(now, false); },destroy() { cancelAnimationFrame(raf); avatar.destroy(); } };diff --git a/src/live/i18n.ts b/src/live/i18n.tsindex 69bdcde..5151ddc 100644--- a/src/live/i18n.ts+++ b/src/live/i18n.ts@@ -21,6 +21,7 @@ export const liveEn = {micBlocked: 'Microphone access was blocked. Allow access in your browser and try again.', micUnavailable: 'Microphone unavailable. Check the selected device.',liveUnavailable: 'Open this project from the local project list before using Live.',expression: 'Expression', expressionHint: 'While this page is focused, keys 1–8 switch expressions. Key 1 returns to neutral.',+ overlays: 'Overlays', overlaysHint: 'Show or hide the avatar\'s overlay layers, such as glasses. The stream view follows these switches.',neutral: 'Neutral', smile: 'Smile', shy: 'Shy', surprise: 'Surprised', halfLidded: 'Half-lidded', angry: 'Angry', sad: 'Sad', wink: 'Wink',duplicatePage: 'This project is also open in another Live tab. The stream view follows only one of them, and switches from other apps reach both. Close the tab you are not using.',remoteTitle: 'Switch from other apps', remoteHelp: 'Keyboard-shortcut apps and button panels can switch expressions even when this page is not in front. Send a POST request with the token header and a JSON body such as {"expression": "smile"}.',@@ -53,6 +54,7 @@ const liveJa: typeof liveEn = {micBlocked: 'マイクへのアクセスが拒否されました。ブラウザで許可してから、もう一度お試しください。', micUnavailable: 'マイクを使用できません。選択したデバイスを確認してください。',liveUnavailable: '配信するには、エディタのローカルプロジェクト一覧からこのプロジェクトを開いてください。',expression: '表情', expressionHint: 'この画面を操作している間は、キー1〜8で表情を切り替えられます。通常に戻すときはキー1を押します。',+ overlays: '重ねるレイヤー', overlaysHint: '眼鏡などの重ねるレイヤーの表示を切り替えます。配信画面にも反映されます。',neutral: '通常', smile: '笑顔', shy: '照れ', surprise: '驚き', halfLidded: 'ジト目', angry: '怒り', sad: '悲しみ', wink: 'ウインク',duplicatePage: 'このプロジェクトの配信画面が別のタブでも開いています。配信用画面はどちらか一方の動きだけを表示し、他のアプリからの切り替えは両方に届きます。使わないタブは閉じてください。',remoteTitle: '他のアプリから切り替える', remoteHelp: 'キーボードショートカットのアプリやボタン操作盤から、この画面が前面になくても表情を切り替えられます。トークンをヘッダーに付けて、{"expression": "smile"} のような JSON を POST で送ってください。',@@ -85,6 +87,7 @@ const liveZh: typeof liveEn = {micBlocked: '麦克风访问被拒绝。请在浏览器中允许访问后重试。', micUnavailable: '麦克风不可用。请检查所选设备。',liveUnavailable: '请先从本地项目列表打开此项目,再使用直播功能。',expression: '表情', expressionHint: '在此页面操作时,可用 1–8 键切换表情。按 1 键恢复普通表情。',+ overlays: '叠加层', overlaysHint: '显示或隐藏眼镜等叠加层。直播画面也会同步切换。',neutral: '普通', smile: '微笑', shy: '害羞', surprise: '惊讶', halfLidded: '半眯眼', angry: '生气', sad: '难过', wink: '眨眼',duplicatePage: '此项目还在另一个直播标签页中打开。直播画面只跟随其中一个,其他应用发来的切换会同时到达两个页面。请关闭不用的标签页。',remoteTitle: '从其他应用切换', remoteHelp: '即使此页面不在前台,也可以通过快捷键应用或按键面板切换表情。请在请求头中附上令牌,用 POST 发送 {"expression": "smile"} 这样的 JSON。',diff --git a/src/live/live.css b/src/live/live.cssindex e3b8499..1d44346 100644--- a/src/live/live.css+++ b/src/live/live.css@@ -50,6 +50,15 @@ input[type="color"] { width: 100%; height: 32px; border: 0; padding: 0; }.live-expression > summary::-webkit-details-marker { display: none; }.live-expression > summary::before { content: '▸'; }.live-expression[open] > summary::before { content: '▾'; }+.live-overlays { border: 1px solid #d5dfeb; border-radius: 12px; background: white; }+.live-overlays > summary { display: flex; align-items: center; gap: 8px; cursor: pointer; padding: 12px 16px; font-weight: 600; list-style: none; }+.live-overlays > summary::-webkit-details-marker { display: none; }+.live-overlays > summary::before { content: '▸'; }+.live-overlays[open] > summary::before { content: '▾'; }+.live-overlays > small { display: block; padding: 0 16px 14px; color: #5a6a80; }+.overlay-buttons { display: flex; flex-wrap: wrap; gap: 6px; padding: 0 16px 8px; }+.overlay-buttons button[aria-pressed="true"] { border-color: #3977d8; background: #e8f0fd; color: #1d4f9c; font-weight: 600; }+.overlays-on { margin-left: auto; padding: 1px 7px; border-radius: 999px; background: #e8f0fd; color: #1d4f9c; font-size: 11px; font-weight: 600; }.expression-on { margin-left: auto; padding: 1px 7px; border-radius: 999px; background: #e8f0fd; color: #1d4f9c; font-size: 11px; font-weight: 600; }.live-expression-body { padding: 0 16px 16px; }.live-expression-body h3 { margin: 16px 0 4px; font-size: 13px; }diff --git a/src/live/overlay-protocol.ts b/src/live/overlay-protocol.tsnew file mode 100644index 0000000..4903993--- /dev/null+++ b/src/live/overlay-protocol.ts@@ -0,0 +1,18 @@+// Overlay layers (built/overlays, such as glasses) shown on the Live page, relayed to stream views.+export const OVERLAYS_EVENT = 'studio:overlays';+export interface OverlaysMessage { project: string; overlays: string[] }+const NAME = /^[A-Za-z0-9_-]{1,64}$/;+export const MAX_OVERLAYS = 32;++/** The valid, unique overlay names in a list, or null when the list itself is not usable. */+export function overlayNames(input: unknown): string[] | null {+ if (!Array.isArray(input) || input.length > MAX_OVERLAYS || !input.every(name => typeof name === 'string' && NAME.test(name))) return null;+ return [...new Set(input as string[])];+}+export function overlaysMessage(input: unknown): OverlaysMessage | null {+ if (!input || typeof input !== 'object' || Array.isArray(input)) return null;+ const data = input as Record<string, unknown>;+ if (Object.keys(data).length !== 2 || typeof data.project !== 'string' || !/^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/.test(data.project)) return null;+ const overlays = overlayNames(data.overlays);+ return overlays ? { project: data.project, overlays } : null;+}diff --git a/src/live/relay.ts b/src/live/relay.tsindex c3455d5..82b7b73 100644--- a/src/live/relay.ts+++ b/src/live/relay.ts@@ -2,6 +2,7 @@ import { LIGHTING_EVENT, lightingWire, lightingMessage, lightingValue } from '..import type { LightingSettings } from 'mesh-avatar';import { LIVE_EVENT, liveMessage } from './protocol';import { EXPRESSION_EVENT, LIGHTING_PRESET_EVENT, expressionMessage, lightingPresetMessage, type RemoteExpression } from './expression-protocol';+import { OVERLAYS_EVENT, overlaysMessage } from './overlay-protocol';export function createLiveSender(project: string, sender = randomSender()) {let sent = -Infinity;@@ -26,6 +27,16 @@ export function receiveLighting(project: string, callback: (value: LightingSettireturn () => import.meta.hot?.off(LIGHTING_EVENT, receive);}+export function sendOverlays(project: string, overlays: string[]) {+ const message = overlaysMessage({ project, overlays });+ if (message) import.meta.hot?.send(OVERLAYS_EVENT, message);+}+export function receiveOverlays(project: string, callback: (overlays: string[]) => void) {+ const receive = (data: unknown) => { const message = overlaysMessage(data); if (message?.project === project) callback(message.overlays); };+ import.meta.hot?.on(OVERLAYS_EVENT, receive);+ return () => import.meta.hot?.off(OVERLAYS_EVENT, receive);+}+/** Expressions sent from other apps; a message without a project applies to every Live page. */export function receiveExpression(project: string, callback: (expression: RemoteExpression) => void) {const receive = (data: unknown) => { const message = expressionMessage(data); if (message && (message.project === null || message.project === project)) callback(message.expression); };diff --git a/src/live/settings.ts b/src/live/settings.tsindex 0ad234a..be9636c 100644--- a/src/live/settings.ts+++ b/src/live/settings.ts@@ -1,5 +1,6 @@import type { LightingSettings } from 'mesh-avatar';import { lightingFromQuery, writeLightingQuery } from '../lighting/storage';+import { overlayNames } from './overlay-protocol';export const SAMPLE_PROJECT = 'sample-miko-qipao';export interface ViewSettings {project: string;@@ -8,6 +9,8 @@ export interface ViewSettings {fit: 'contain' | 'cover';idle: boolean;lighting?: LightingSettings;+ /** Overlay layers shown on the avatar, such as glasses. */+ overlays?: string[];}export function backgroundColor(value: string | null): string {if (value === 'green') return '#00ff00';@@ -27,12 +30,22 @@ export function viewSettings(search: string, origin = globalThis.location?.origi} catch { /* Invalid or external URLs fall back to the sample. */ }return { project: project && /^[A-Za-z0-9][A-Za-z0-9._-]*$/.test(project) ? project : SAMPLE_PROJECT,...(avatar ? { avatar } : {}),- background: backgroundColor(query.get('bg')), fit: query.get('fit') === 'cover' ? 'cover' : 'contain', idle: query.get('idle') !== '0', ...(query.has('light') ? { lighting: lightingFromQuery(query) } : {}) };+ background: backgroundColor(query.get('bg')), fit: query.get('fit') === 'cover' ? 'cover' : 'contain', idle: query.get('idle') !== '0', ...(query.has('light') ? { lighting: lightingFromQuery(query) } : {}),+ ...(query.has('overlays') ? { overlays: overlayNames(query.get('overlays')!.split(',').filter(Boolean)) ?? [] } : {}) };+}+const overlaysKey = (project: string) => `mesh-avatar-live-overlays:${project}`;+/** Overlays last shown for a project on this Live page; storage may be unavailable. */+export function loadOverlays(project: string): string[] {+ try { return overlayNames(JSON.parse(localStorage.getItem(overlaysKey(project)) ?? '[]')) ?? []; } catch { return []; }+}+export function saveOverlays(project: string, overlays: string[]) {+ try { localStorage.setItem(overlaysKey(project), JSON.stringify(overlays)); } catch { /* the choice is kept for this page only */ }}export function streamUrl(settings: ViewSettings, origin: string): string {const url = new URL('/stream.html', origin);url.search = new URLSearchParams({ project: settings.project, bg: settings.background, fit: settings.fit, idle: settings.idle ? '1' : '0' }).toString();if (settings.avatar) { url.searchParams.delete('project'); url.searchParams.set('avatar', settings.avatar); }if (settings.lighting) writeLightingQuery(url.searchParams, settings.lighting);+ if (settings.overlays?.length) url.searchParams.set('overlays', settings.overlays.join(','));return url.href;}diff --git a/src/live/stream.ts b/src/live/stream.tsindex 4bd2bea..e9e3007 100644--- a/src/live/stream.ts+++ b/src/live/stream.ts@@ -1,7 +1,7 @@import { createAvatarView, neutralParameters } from './avatar-view';import { viewSettings } from './settings';import { LivePose } from './protocol';-import { receiveLighting, receiveLiveParameters } from './relay';+import { receiveLighting, receiveLiveParameters, receiveOverlays } from './relay';import './stream.css';const settings = viewSettings(location.search);@@ -10,6 +10,7 @@ const canvas = document.querySelector<HTMLCanvasElement>('#avatar')!;const pose = new LivePose(settings.project);let avatarInstance: import('mesh-avatar').MeshAvatar | undefined;const unsubscribeLighting = receiveLighting(settings.project, value => { settings.lighting = value; avatarInstance?.setLighting(value); });+const unsubscribeOverlays = receiveOverlays(settings.project, value => { settings.overlays = value; avatarInstance?.setOverlays(value); });const unsubscribe = receiveLiveParameters(data => pose.receive(data, performance.now()));void createAvatarView(canvas, settings, (avatar, now, dt) => {const sampled = pose.sample(now, dt);@@ -24,7 +25,9 @@ void createAvatarView(canvas, settings, (avatar, now, dt) => {}).then(view => {avatarInstance = view.avatar;if (settings.lighting) view.avatar.setLighting(settings.lighting);- const destroy = () => { unsubscribe(); unsubscribeLighting(); avatarInstance = undefined; view.destroy(); };+ // An update relayed while the avatar was loading wins over the URL.+ if (settings.overlays) view.avatar.setOverlays(settings.overlays);+ const destroy = () => { unsubscribe(); unsubscribeLighting(); unsubscribeOverlays(); avatarInstance = undefined; view.destroy(); };window.addEventListener('pagehide', destroy, { once: true });import.meta.hot?.dispose(destroy);-}).catch(() => { unsubscribe(); unsubscribeLighting(); canvas.dataset.state = 'error'; });+}).catch(() => { unsubscribe(); unsubscribeLighting(); unsubscribeOverlays(); canvas.dataset.state = 'error'; });diff --git a/src/server/live-relay.ts b/src/server/live-relay.tsindex 356f5f7..81fe9d8 100644--- a/src/server/live-relay.ts+++ b/src/server/live-relay.ts@@ -4,6 +4,7 @@ import { LIVE_EVENT, liveMessage } from '../live/protocol';import { resolve } from 'node:path';import { EXPRESSION_EVENT, LIGHTING_PRESET_EVENT } from '../live/expression-protocol';import { expressionRemote } from './expression-remote';+import { OVERLAYS_EVENT, overlaysMessage } from '../live/overlay-protocol';export function liveRelay(root: string): Plugin {return {@@ -18,6 +19,13 @@ export function liveRelay(root: string): Plugin {lastLighting.set(client.socket, now);server.ws.send(LIGHTING_EVENT, message);});+ const lastOverlays = new WeakMap<WebSocketClient['socket'], number>();+ server.ws.on(OVERLAYS_EVENT, (data, client) => {+ const message = overlaysMessage(data), now = performance.now();+ if (!message || now - (lastOverlays.get(client.socket) ?? -Infinity) < 1000 / 60) return;+ lastOverlays.set(client.socket, now);+ server.ws.send(OVERLAYS_EVENT, message);+ });const lastSent = new WeakMap<WebSocketClient['socket'], number>();server.ws.on(LIVE_EVENT, (data, client) => {const now = performance.now();diff --git a/tests/stream-settings.test.ts b/tests/stream-settings.test.tsindex 570d949..19cf387 100644--- a/tests/stream-settings.test.ts+++ b/tests/stream-settings.test.ts@@ -1,5 +1,6 @@import { expect, test } from 'vitest';import { backgroundColor, streamUrl, viewSettings } from '../src/live/settings';+import { overlaysMessage } from '../src/live/overlay-protocol';test('stream defaults are transparent, contained, idle and use the bundled sample', () => {expect(viewSettings('')).toEqual({ project: 'sample-miko-qipao', background: 'transparent', fit: 'contain', idle: true });@@ -29,3 +30,23 @@ test('stream avatar sources stay on the current server origin', () => {expect(viewSettings(`?avatar=${encodeURIComponent(value)}`, origin).avatar).toBe(value);}});++test('overlay names round-trip through the stream URL and invalid lists are dropped', () => {+ const settings = viewSettings('?project=mio&overlays=glasses,hat');+ expect(settings.overlays).toEqual(['glasses', 'hat']);+ expect(new URL(streamUrl(settings, 'http://127.0.0.1:5173')).searchParams.get('overlays')).toBe('glasses,hat');+ expect(viewSettings('?project=mio').overlays).toBeUndefined();+ expect(viewSettings('?project=mio&overlays=').overlays).toEqual([]);+ expect(viewSettings('?project=mio&overlays=glasses,../x').overlays).toEqual([]);+ expect(new URL(streamUrl({ ...settings, overlays: [] }, 'http://127.0.0.1:5173')).searchParams.has('overlays')).toBe(false);+});++test('overlay relay messages accept only a project and a short list of safe, unique names', () => {+ expect(overlaysMessage({ project: 'mio', overlays: ['glasses', 'glasses', 'hat'] })).toEqual({ project: 'mio', overlays: ['glasses', 'hat'] });+ expect(overlaysMessage({ project: 'mio', overlays: [] })).toEqual({ project: 'mio', overlays: [] });+ for (const bad of [null, [], { project: 'mio' }, { project: '../x', overlays: [] }, { project: 'mio', overlays: 'glasses' },+ { project: 'mio', overlays: ['has space'] }, { project: 'mio', overlays: [1] }, { project: 'mio', overlays: [], extra: 1 },+ { project: 'mio', overlays: Array.from({ length: 33 }, (_, i) => `o${i}`) }]) {+ expect(overlaysMessage(bad)).toBeNull();+ }+});