Skip to content

Generate clean 2D game sprites & animation atlases — component-row pipeline: state rows, alpha cleanup, frame extraction, runtime atlases. Codex/Claude skill.

feat: generate cached game sounds with normalized browser exports

Soohong Kima06e1b561610
diff --git a/sprite_gen/_modules.py b/sprite_gen/_modules.py
index 19cbcc7..1caa94b 100644
--- a/sprite_gen/_modules.py
+++ b/sprite_gen/_modules.py
@@ -6,6 +6,7 @@ the reorg tooling. Keep in sync with the physical folder layout.
"""
MODULE_DOMAIN = {
+ 'sound': 'audio',
'guide': 'workflow',
'preferences': 'workflow',
'runio': 'spec',
@@ -76,6 +77,7 @@ MODULE_DOMAIN = {
DOMAINS: list[tuple[str, str]] = [
("workflow", "Start here — image or sprite choices, access checks, saved defaults"),
("gen", "Generation — prepare a run, generate stills / rows / clips"),
+ ("audio", "Audio — named effects, music, loop processing and browser exports"),
("video", "Video → loop — canvas, keyed frames, seamless cycle, batch"),
("frames", "Frames — chroma/white removal, row extraction, sheet slicing, atlas unpacking"),
("curate", "Curation — direction anchors and the curation sidecar"),
diff --git a/sprite_gen/audio/__init__.py b/sprite_gen/audio/__init__.py
new file mode 100644
index 0000000..c7a7365
--- /dev/null
+++ b/sprite_gen/audio/__init__.py
@@ -0,0 +1,2 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Game audio generation, durable request cache and browser exports."""
diff --git a/sprite_gen/audio/cache.py b/sprite_gen/audio/cache.py
new file mode 100644
index 0000000..46e2935
--- /dev/null
+++ b/sprite_gen/audio/cache.py
@@ -0,0 +1,38 @@
+# SPDX-License-Identifier: Apache-2.0
+"""SQLite claim before POST, atomic audio receipt after POST. No expiring paid-request lease."""
+from __future__ import annotations
+
+from contextlib import closing
+import hashlib
+import json
+from pathlib import Path
+import sqlite3
+
+
+def digest(document):
+ return hashlib.sha256(json.dumps(document, sort_keys=True, separators=(',', ':'), allow_nan=False).encode()).hexdigest()
+
+
+def obtain(path, document, make, *, prepare=None):
+ path = Path(path)
+ path.parent.mkdir(parents=True, exist_ok=True)
+ key = digest(document)
+ with closing(sqlite3.connect(path, timeout=30)) as db:
+ db.execute('CREATE TABLE IF NOT EXISTS audio (id TEXT PRIMARY KEY, audio BLOB, metadata TEXT)')
+ # The committed NULL receipt is a durable in-flight/unknown outcome. Another process,
+ # a crash or a timeout cannot turn it into an automatic second purchase.
+ with db:
+ db.execute('BEGIN IMMEDIATE')
+ row = db.execute('SELECT audio, metadata FROM audio WHERE id=?', (key,)).fetchone()
+ if row is not None:
+ if row[0] is None:
+ raise ValueError(f'sound request {key} is unresolved (in progress or failed); inspect the provider request before any new purchase')
+ return bytes(row[0]), json.loads(row[1]), True
+ if prepare is not None:
+ prepare()
+ db.execute('INSERT INTO audio(id) VALUES (?)', (key,))
+ audio, metadata = make()
+ with db:
+ db.execute('UPDATE audio SET audio=?, metadata=? WHERE id=?',
+ (audio, json.dumps(metadata, allow_nan=False), key))
+ return audio, metadata, False
diff --git a/sprite_gen/audio/processing.py b/sprite_gen/audio/processing.py
new file mode 100644
index 0000000..eea3644
--- /dev/null
+++ b/sprite_gen/audio/processing.py
@@ -0,0 +1,106 @@
+# SPDX-License-Identifier: Apache-2.0
+"""PCM loop overlap, EBU R128 normalization and measured browser audio encodes."""
+from __future__ import annotations
+
+import hashlib
+import json
+import math
+from pathlib import Path
+import re
+import shutil
+import subprocess
+
+import numpy as np
+
+RATE = 48000
+
+
+def require_tools():
+ for name in ('ffmpeg', 'ffprobe'):
+ if not shutil.which(name):
+ raise ValueError(f'{name} is required on PATH for sound processing')
+
+
+def _ffmpeg(args, data=None):
+ result = subprocess.run(['ffmpeg', '-hide_banner', '-nostdin', '-y', *args],
+ input=data, capture_output=True, timeout=180)
+ if result.returncode:
+ # Source metadata can contain untrusted prose. Keep diagnostics local and bounded.
+ raise ValueError('ffmpeg audio processing failed; check input audio and codec availability')
+ return result
+
+
+def decode(path):
+ result = _ffmpeg(['-v', 'error', '-i', str(path), '-map', '0:a:0', '-ac', '2', '-ar', str(RATE),
+ '-f', 'f32le', 'pipe:1'])
+ samples = np.frombuffer(result.stdout, dtype='<f4').reshape(-1, 2).copy()
+ if not len(samples) or not np.isfinite(samples).all():
+ raise ValueError('source has no finite audio samples')
+ return samples
+
+
+def make_loop(samples, seconds):
+ count = round(seconds * RATE)
+ if count < 2 or count * 2 >= len(samples):
+ raise ValueError('loop crossfade must be positive and shorter than half the source')
+ weight = (.5 - .5 * np.cos(np.linspace(0, np.pi, count)))[:, None]
+ overlap = samples[-count:] * (1 - weight) + samples[:count] * weight
+ return np.concatenate((samples[count:-count], overlap)).astype('<f4')
+
+
+def _pcm_args():
+ return ['-f', 'f32le', '-ar', str(RATE), '-ac', '2', '-i', 'pipe:0']
+
+
+def measure(samples, target=-18):
+ result = _ffmpeg([*_pcm_args(), '-af', f'loudnorm=I={target}:TP=-2:LRA=11:print_format=json',
+ '-f', 'null', '-'], samples.astype('<f4').tobytes())
+ matches = re.findall(r'\{\s*"input_i".*?\}', result.stderr.decode(errors='replace'), re.S)
+ if not matches:
+ raise ValueError('ffmpeg did not report loudness')
+ values = json.loads(matches[-1])
+ return {k: float(v) for k, v in values.items() if k != 'normalization_type'}
+
+
+def normalize(samples, target, loop):
+ # Repeated context avoids loudnorm's startup/lookahead envelope at the loop boundary.
+ source = np.tile(samples, (3, 1)) if loop else samples
+ measured = measure(source, target)
+ if not all(math.isfinite(measured[k]) for k in ('input_i', 'input_tp', 'input_thresh', 'target_offset')):
+ raise ValueError('silent or unmeasurable audio cannot be loudness normalized')
+ filt = (f'loudnorm=I={target}:TP=-2:LRA=11:measured_I={measured["input_i"]}:'
+ f'measured_TP={measured["input_tp"]}:measured_LRA={measured["input_lra"]}:'
+ f'measured_thresh={measured["input_thresh"]}:offset={measured["target_offset"]}:linear=true')
+ result = _ffmpeg([*_pcm_args(), '-af', filt, '-ar', str(RATE), '-f', 'f32le', 'pipe:1'],
+ source.astype('<f4').tobytes())
+ output = np.frombuffer(result.stdout, dtype='<f4').reshape(-1, 2).copy()
+ return output[len(samples):2*len(samples)] if loop else output[:len(samples)]
+
+
+def export(source: Path, out_dir: Path, *, target_lufs, loop, crossfade):
+ samples = decode(source)
+ source_seconds = len(samples) / RATE
+ if loop:
+ samples = make_loop(samples, crossfade)
+ samples = normalize(samples, target_lufs, loop)
+ duration = len(samples) / RATE
+ files = {}
+ for fmt, codec in (('ogg', ['-c:a', 'libvorbis', '-q:a', '6']), ('m4a', ['-c:a', 'aac', '-b:a', '192k', '-movflags', '+faststart'])):
+ path = out_dir / f'sound.{fmt}'
+ _ffmpeg([*_pcm_args(), '-map_metadata', '-1', *codec, str(path)], samples.astype('<f4').tobytes())
+ decoded = decode(path)
+ if len(decoded) < len(samples) - 2:
+ raise ValueError(f'{fmt} truncated the audio')
+ playback = decoded[:len(samples)] # AAC may retain a final partial codec frame.
+ measured = measure(np.tile(playback, (3, 1)) if loop else playback, target_lufs)
+ lufs, peak = measured['input_i'], measured['input_tp']
+ if not math.isfinite(lufs) or abs(lufs - target_lufs) > 1 or peak > -1:
+ raise ValueError(f'{fmt} failed loudness/peak check ({lufs:g} LUFS, {peak:g} dBTP); cached source retained')
+ files[fmt] = {'path': path.name, 'duration_seconds': len(decoded)/RATE,
+ 'lufs': lufs, 'true_peak_dbfs': peak,
+ 'sha256': hashlib.sha256(path.read_bytes()).hexdigest()}
+ if loop:
+ files[fmt]['seam_delta'] = float(np.max(np.abs(playback[-1] - playback[0])))
+ return {'files': files, 'duration_seconds': duration, 'source_duration_seconds': source_seconds,
+ 'sample_rate': RATE, 'loop_start_seconds': 0, 'loop_end_seconds': duration,
+ 'crossfade_seconds': crossfade if loop else 0}
diff --git a/sprite_gen/audio/provider.py b/sprite_gen/audio/provider.py
new file mode 100644
index 0000000..ebf063b
--- /dev/null
+++ b/sprite_gen/audio/provider.py
@@ -0,0 +1,89 @@
+# SPDX-License-Identifier: Apache-2.0
+"""ElevenLabs HTTP contract; retry decisions belong to gen.retry."""
+from __future__ import annotations
+
+import json
+import math
+import os
+import re
+import sys
+from urllib.error import HTTPError
+from urllib.request import Request, urlopen
+
+from sprite_gen.gen import retry
+
+BASE = 'https://api.elevenlabs.io/v1'
+FORMAT = 'mp3_44100_128'
+SAFE_CODES = frozenset({'system_busy', 'too_many_concurrent_requests', 'quota_exceeded',
+ 'invalid_api_key', 'missing_permissions', 'bad_prompt', 'bad_composition_plan',
+ 'content_policy_violation', 'unauthorized', 'invalid_request'})
+
+
+def request(kind, prompt, duration, loop):
+ if kind not in ('sfx', 'bgm'):
+ raise ValueError('kind must be sfx or bgm')
+ if not isinstance(prompt, str) or not prompt.strip() or len(prompt) > 4100:
+ raise ValueError('prompt requires 1 to 4100 characters')
+ duration = float(duration)
+ low, high = (0.5, 30) if kind == 'sfx' else (3, 600)
+ if not math.isfinite(duration) or not low <= duration <= high:
+ raise ValueError(f'{kind} duration must be between {low} and {high} seconds')
+ if type(loop) is not bool:
+ raise ValueError('loop must be a boolean')
+ body = ({'text': prompt, 'duration_seconds': duration, 'loop': loop,
+ 'model_id': 'eleven_text_to_sound_v2', 'prompt_influence': 0.3} if kind == 'sfx'
+ else {'prompt': prompt, 'music_length_ms': round(duration * 1000),
+ 'model_id': 'music_v1', 'force_instrumental': True})
+ return {'endpoint': 'sound-generation' if kind == 'sfx' else 'music',
+ 'output_format': FORMAT, 'body': body}
+
+
+def credential():
+ key = os.environ.get('ELEVENLABS_API_KEY', '').strip()
+ if not key:
+ raise ValueError('ELEVENLABS_API_KEY is required for a new sound request')
+ return key
+
+
+def _code(body):
+ detail = body.get('detail') if isinstance(body, dict) else None
+ code = detail.get('status') if isinstance(detail, dict) else None
+ return code if isinstance(code, str) and code in SAFE_CODES else None
+
+
+def generate(document):
+ key = credential()
+ req = Request(f"{BASE}/{document['endpoint']}?output_format={document['output_format']}",
+ data=json.dumps(document['body']).encode(),
+ headers={'xi-api-key': key, 'Content-Type': 'application/json'}, method='POST')
+
+ def send():
+ try:
+ with urlopen(req, timeout=600) as response:
+ return response.status, response.read(), response.headers
+ except HTTPError as exc:
+ try:
+ body = json.loads(exc.read())
+ except (ValueError, OSError):
+ body = {}
+ return exc.code, body, exc.headers
+ except Exception:
+ # Transport exceptions may include headers or body text. Never print them.
+ raise ValueError('ElevenLabs response unavailable; request outcome unresolved; no resubmission') from None
+
+ print('[sound] ElevenLabs paid API request; source audio will be cached', file=sys.stderr)
+ status, body, headers, asks = retry.ask(send, provider='elevenlabs', who='ElevenLabs', label='sound', kind=_code)
+ if not 200 <= status < 300:
+ raise ValueError(f'ElevenLabs refused sound (HTTP {status}, {_code(body) or "unclassified"}); {retry.tail(asks)}')
+ if not isinstance(body, bytes) or not body:
+ raise ValueError('ElevenLabs returned no audio; request outcome unresolved')
+ meta = {'provider': 'elevenlabs', 'model': document['body']['model_id'], 'asks': asks}
+ # Only documented, constrained metadata; provider prose never enters a report or log.
+ for header, field in (('request-id', 'request_id'), ('song-id', 'song_id')):
+ value = headers.get(header, '')
+ if value and key not in value and re.fullmatch(r'[A-Za-z0-9_-]{1,128}', value):
+ meta[field] = value
+ value = headers.get('character-cost', '')
+ if str(value).isdigit() and len(str(value)) < 12:
+ meta['character_cost'] = int(value)
+ return body, meta
diff --git a/sprite_gen/audio/sound.py b/sprite_gen/audio/sound.py
new file mode 100644
index 0000000..0d5b6c3
--- /dev/null
+++ b/sprite_gen/audio/sound.py
@@ -0,0 +1,97 @@
+# SPDX-License-Identifier: Apache-2.0
+"""Generate one named game sound and publish its immutable audio bundle."""
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import os
+from pathlib import Path
+import re
+import shutil
+import tempfile
+
+from sprite_gen.audio import cache, processing, provider
+from sprite_gen.util.cli_kwargs import namespace_from_kwargs
+
+
+def generate(*, kind, prompt, duration, event, out_dir, cache_dir=None, loop=None, target_lufs=None, crossfade=None):
+ loop = kind == 'bgm' if loop is None else loop
+ request = provider.request(kind, prompt, duration, loop)
+ if not isinstance(event, str) or not re.fullmatch(r'[A-Za-z0-9][A-Za-z0-9_.:-]{0,127}', event):
+ raise ValueError('event requires 1 to 128 letters, digits, dots, colons, dashes or underscores')
+ target = (-18.0 if kind == 'sfx' else -20.0) if target_lufs is None else float(target_lufs)
+ if not math.isfinite(target) or not -36 <= target <= -9:
+ raise ValueError('target_lufs must be between -36 and -9')
+ crossfade = (min(.25, float(duration)/8) if crossfade is None else float(crossfade)) if loop else 0
+ if loop and (not math.isfinite(crossfade) or not 2/processing.RATE <= crossfade < float(duration)/2):
+ raise ValueError('crossfade must be positive and shorter than half the duration')
+ out_dir = Path(out_dir).expanduser().resolve()
+ recipe = {'request': cache.digest(request), 'event': event, 'loop': loop,
+ 'target_lufs': target, 'crossfade_seconds': crossfade, 'processing_version': 1}
+ if out_dir.exists():
+ manifest = json.loads((out_dir / 'sound.json').read_text()) if (out_dir / 'sound.json').is_file() else {}
+ if manifest.get('recipe') != recipe:
+ raise ValueError('output directory already exists with a different sound; use a new directory')
+ for file in manifest['files'].values():
+ path = out_dir / file['path']
+ if not path.is_file() or cache_hash(path) != file['sha256']:
+ raise ValueError('published sound is damaged; use a new output directory with the same cache')
+ return {**manifest, 'cache_hit': True}
+ processing.require_tools()
+ cache_dir = Path(cache_dir).expanduser() if cache_dir else Path(os.environ.get('XDG_CACHE_HOME', Path.home()/'.cache'))/'sprite-gen'/'audio'
+ # Resolve missing credentials before claiming a request, but allow fully offline cache use.
+ def make():
+ return provider.generate(request)
+ if cache_dir.resolve() == out_dir or out_dir in cache_dir.resolve().parents:
+ raise ValueError('cache directory must be outside the output bundle')
+ data, receipt, hit = cache.obtain(cache_dir/'requests.sqlite3', request, make, prepare=provider.credential)
+ out_dir.parent.mkdir(parents=True, exist_ok=True)
+ stage = Path(tempfile.mkdtemp(prefix=f'.{out_dir.name}-', dir=out_dir.parent))
+ try:
+ source = stage/'source.mp3'
+ source.write_bytes(data)
+ details = processing.export(source, stage, target_lufs=target, loop=loop, crossfade=crossfade)
+ source.unlink()
+ manifest = {'kind': 'sprite-gen-sound', 'version': 1, 'event': event, 'audio_kind': kind,
+ 'loop': loop, 'target_lufs': target, 'recipe': recipe, 'provider': receipt, **details}
+ (stage/'sound.json').write_text(json.dumps(manifest, indent=2, allow_nan=False)+'\n')
+ stage.rename(out_dir)
+ finally:
+ shutil.rmtree(stage, ignore_errors=True)
+ return {**manifest, 'cache_hit': hit}
+
+
+def cache_hash(path):
+ import hashlib
+ return hashlib.sha256(path.read_bytes()).hexdigest()
+
+
+def add_arguments(parser):
+ parser.add_argument('--kind', choices=('sfx', 'bgm'), required=True)
+ prompts = parser.add_mutually_exclusive_group(required=True)
+ prompts.add_argument('--prompt')
+ prompts.add_argument('--prompt-file', type=Path)
+ parser.add_argument('--duration', type=float, required=True, help='provider duration in seconds; loops subtract crossfade')
+ parser.add_argument('--event', required=True)
+ parser.add_argument('--out-dir', type=Path, required=True)
+ parser.add_argument('--cache-dir', type=Path)
+ parser.add_argument('--loop', action=argparse.BooleanOptionalAction, default=None)
+ parser.add_argument('--target-lufs', type=float)
+ parser.add_argument('--crossfade', type=float, help='loop overlap seconds (default <=0.25s)')
+
+
+def run(**kwargs):
+ parser = argparse.ArgumentParser()
+ add_arguments(parser)
+ args = namespace_from_kwargs(parser, kwargs)
+ try:
+ prompt = args.prompt_file.read_text() if args.prompt_file else args.prompt
+ result = generate(kind=args.kind, prompt=prompt, duration=args.duration, event=args.event,
+ out_dir=args.out_dir, cache_dir=args.cache_dir, loop=args.loop,
+ target_lufs=args.target_lufs, crossfade=args.crossfade)
+ except (ValueError, OSError) as exc:
+ raise SystemExit(f'sound: {exc}') from None
+ print(json.dumps({'ok': True, 'manifest': str(args.out_dir/'sound.json'),
+ 'cache_hit': result['cache_hit'], 'duration_seconds': result['duration_seconds']}))
+ return 0
diff --git a/sprite_gen/cli.py b/sprite_gen/cli.py
index b821f29..096d3c6 100644
--- a/sprite_gen/cli.py
+++ b/sprite_gen/cli.py
@@ -8,6 +8,7 @@ from pathlib import Path
from typing import Callable
from sprite_gen import _modules, gen
+from sprite_gen.audio import sound
from sprite_gen.background import tile
from sprite_gen.curate import anchor
from sprite_gen.compose import compose_atlas, compose_cycle, compose_gif, compose_layers, export_aseprite, export_pngs
@@ -245,6 +246,7 @@ def _add_correction_loop(p: argparse.ArgumentParser) -> None:
COMMANDS: dict[str, tuple[str, Callable[[argparse.ArgumentParser], None], Callable[..., int]]] = {
"game-pack": ("Resize finished assets into game atlases with playback policies.", game_pack.add_arguments, game_pack.run),
+ "sound": ("Generate a named game sound or instrumental music with browser audio exports.", sound.add_arguments, sound.run),
"background-tile": ("Quilt a repeating RGBA background and measure its join.", tile.add_arguments, tile.run),
"shadow": ("Project silhouette shadows from any supported asset.", shadow.add_arguments, shadow.run),
"inspect-motion": ("Measure asset timing and contact evidence without changing animation.", motion.add_arguments, motion.run),
diff --git a/sprite_gen/gen/retry.py b/sprite_gen/gen/retry.py
index 4d17d30..911d871 100644
--- a/sprite_gen/gen/retry.py
+++ b/sprite_gen/gen/retry.py
@@ -56,12 +56,14 @@ UNTAKEN: dict[str, frozenset[int]] = {
"anthropic": frozenset({429, 500, 529}),
"openai": frozenset({429, 503}),
"xai": frozenset({429}),
+ "elevenlabs": frozenset({429}),
}
ACT_FIRST: dict[str, frozenset[str]] = {
"anthropic": frozenset({"enforced_spend_limit_reached"}),
"openai": frozenset({"credit_balance_exhausted", "organization_spend_limit_exceeded",
"project_spend_limit_exceeded", "organization_usage_limit_exceeded", "insufficient_quota"}),
"xai": frozenset(),
+ "elevenlabs": frozenset(),
}
GET_AGAIN = frozenset({429, 500, 502, 503, 504, 529})
@@ -96,6 +98,12 @@ def _act_first(provider: str, body: Any) -> bool:
def again_after(provider: str, method: str, status: int, body: Any, headers: Any, failures: int) -> float | None:
"""Seconds to wait before sending the request again after its `failures`-th failed answer in a row,
or None when it is not sent again (the rule in the module docstring)."""
+ # ElevenLabs documents only these two 429 codes as requests it could not process.
+ # https://elevenlabs.io/docs/help-center/technical/api-error-code-429 (2026-10-13).
+ if provider == "elevenlabs" and method != "GET":
+ detail = body.get("detail") if isinstance(body, dict) else None
+ if not isinstance(detail, dict) or detail.get("status") not in ("system_busy", "too_many_concurrent_requests"):
+ return None
if failures > ASKS_AGAIN:
return None
if method == "GET":
diff --git a/tests/audio/test_sound.py b/tests/audio/test_sound.py
index db9c905..005e751 100644
--- a/tests/audio/test_sound.py
+++ b/tests/audio/test_sound.py
@@ -13,6 +13,11 @@ from sprite_gen.audio import cache, provider, processing, sound
from sprite_gen.gen import retry
[email protected](autouse=True)
+def fake_credential(monkeypatch):
+ monkeypatch.setenv('ELEVENLABS_API_KEY', 'synthetic-key')
+
+
def wav_bytes(seconds=3):
t = np.arange(round(seconds * 48000)) / 48000
x = .08 * np.sin(2 * np.pi * 321.3 * t)