World's first open-source, agentic video production system. 12 production pipelines, 100+ tools, 700+ agent skill and production-knowledge files. Turn your AI coding assistant into a full video production studio.
Python agentagentic-aiaiclaudecopilotcursorelevenlabsffmpegfluximage-generationopen-sourceopenaipythonremotionstable-diffusiontext-to-speechtext-to-videovideo-generationvideo-production
feat(elevenlabs_tts): optional word-level timestamps (#752)
hojanyazd380b69126c2
diff --git a/.agents/skills/elevenlabs/SKILL.md b/.agents/skills/elevenlabs/SKILL.mdindex d81dc390..db8e3cda 100644--- a/.agents/skills/elevenlabs/SKILL.md+++ b/.agents/skills/elevenlabs/SKILL.md@@ -279,6 +279,34 @@ export const sceneDurations = manifest.scenes.reduce((acc, scene) => {// <Series.Sequence durationInFrames={sceneDurations.title}>```+### Word-Level Timings (direct `elevenlabs_tts`)++When captions, word-synced headlines or animation cues must follow the real+speech, pass `with_timestamps: true` to the direct `elevenlabs_tts` tool instead+of transcribing the audio afterwards:++```python+result = ElevenLabsTTS().execute({+ "text": paragraph,+ "voice_id": voice_id,+ "previous_text": prev_paragraph, # continuity across separately synthesized paragraphs+ "next_text": next_paragraph,+ "with_timestamps": True,+ "output_path": "projects/<id>/assets/audio/tts/para_03.mp3",+})+result.data["word_timestamps"] # [{"word": "Доверие.", "start": 0.12, "end": 0.71}, ...]+result.data["alignment_path"] # para_03.mp3.alignment.json (raw character alignment)+```++- Times are seconds from the start of that audio file. When paragraphs are placed+ on one timeline, add each paragraph's placement offset.+- Words are whitespace-delimited and keep their punctuation, so they line up with+ `text.split()` of the script.+- `alignment_matches_text` is `false` when the service normalised the text+ (numbers, symbols); timings then follow the spoken form in the alignment.+- Synthesizing per paragraph gives exact paragraph boundaries for scene cuts;+ the word timings place captions and cues inside each paragraph.+### Audio Timing Patterns```tsxdiff --git a/tests/tools/test_elevenlabs_timestamps.py b/tests/tools/test_elevenlabs_timestamps.pynew file mode 100644index 00000000..00111eac--- /dev/null+++ b/tests/tools/test_elevenlabs_timestamps.py@@ -0,0 +1,108 @@+"""ElevenLabs TTS: optional word-level timings via the /with-timestamps endpoint."""++from __future__ import annotations++import base64+import json+import wave+from unittest.mock import Mock++from tools.audio.elevenlabs_tts import ElevenLabsTTS+++def _alignment(text: str, step: float = 0.1) -> dict:+ chars = list(text)+ return {+ "characters": chars,+ "character_start_times_seconds": [round(i * step, 3) for i in range(len(chars))],+ "character_end_times_seconds": [round((i + 1) * step, 3) for i in range(len(chars))],+ }+++def _json_response(text: str, audio: bytes) -> Mock:+ body = {"audio_base64": base64.b64encode(audio).decode(), "alignment": _alignment(text)}+ return Mock(json=Mock(return_value=body), headers={"request-id": "r1"}, content=b"")+++def test_with_timestamps_calls_endpoint_and_returns_words(monkeypatch, tmp_path):+ monkeypatch.setenv("ELEVENLABS_API_KEY", "test")+ text = "Много знакомств. А кому позвонить?"+ post = Mock(return_value=_json_response(text, b"ID3fake-mp3"))+ monkeypatch.setattr("requests.post", post)+ out = tmp_path / "para.mp3"++ result = ElevenLabsTTS().execute({"text": text, "with_timestamps": True, "output_path": str(out)})++ assert result.success, result.error+ assert post.call_args.args[0].endswith("/with-timestamps")+ assert post.call_args.kwargs["headers"]["Accept"] == "application/json"+ assert "with_timestamps" not in post.call_args.kwargs["json"] # tool option, not an API field+ assert out.read_bytes() == b"ID3fake-mp3"++ words = result.data["word_timestamps"]+ assert [w["word"] for w in words] == text.split()+ assert words[0] == {"word": "Много", "start": 0.0, "end": 0.5}+ assert words[1]["start"] == 0.6 and words[1]["end"] == 1.6 # "знакомств." incl. punctuation+ assert result.data["alignment_matches_text"] is True++ saved = json.loads((tmp_path / "para.mp3.alignment.json").read_text(encoding="utf-8"))+ assert saved["text"] == text and saved["words"] == words+ assert str(tmp_path / "para.mp3.alignment.json") in result.artifacts+++def test_with_timestamps_pcm_output_is_wrapped_as_wav(monkeypatch, tmp_path):+ monkeypatch.setenv("ELEVENLABS_API_KEY", "test")+ post = Mock(return_value=_json_response("Hi there", b"\x00\x00" * 50))+ monkeypatch.setattr("requests.post", post)+ out = tmp_path / "voice.wav"++ result = ElevenLabsTTS().execute(+ {"text": "Hi there", "with_timestamps": True, "output_format": "pcm_16000", "output_path": str(out)}+ )++ assert result.success, result.error+ with wave.open(str(out)) as wav:+ assert wav.getframerate() == 16000 and wav.getnframes() == 50+ assert [w["word"] for w in result.data["word_timestamps"]] == ["Hi", "there"]+++def test_default_path_is_unchanged(monkeypatch, tmp_path):+ monkeypatch.setenv("ELEVENLABS_API_KEY", "test")+ post = Mock(return_value=Mock(content=b"ID3audio", headers={"request-id": "r2"}))+ monkeypatch.setattr("requests.post", post)++ result = ElevenLabsTTS().execute({"text": "Hello", "output_path": str(tmp_path / "a.mp3")})++ assert result.success, result.error+ assert not post.call_args.args[0].endswith("/with-timestamps")+ assert post.call_args.kwargs["headers"]["Accept"] == "audio/mpeg"+ assert "word_timestamps" not in result.data+ assert result.artifacts == [str(tmp_path / "a.mp3")]+++def test_missing_alignment_fails_loudly(monkeypatch, tmp_path):+ monkeypatch.setenv("ELEVENLABS_API_KEY", "test")+ body = {"audio_base64": base64.b64encode(b"x").decode()}+ monkeypatch.setattr("requests.post", Mock(return_value=Mock(json=Mock(return_value=body), headers={})))++ result = ElevenLabsTTS().execute({"text": "Hello", "with_timestamps": True, "output_path": str(tmp_path / "a.mp3")})++ assert not result.success+ assert "alignment" in result.error+++def test_words_from_alignment_handles_multichar_entries():+ alignment = {+ "characters": ["Hel", "lo", " ", "wo", "rld!"],+ "character_start_times_seconds": [0.0, 0.3, 0.5, 0.6, 0.8],+ "character_end_times_seconds": [0.3, 0.5, 0.6, 0.8, 1.1],+ }+ assert ElevenLabsTTS.words_from_alignment(alignment) == [+ {"word": "Hello", "start": 0.0, "end": 0.5},+ {"word": "world!", "start": 0.6, "end": 1.1},+ ]+++def test_with_timestamps_is_part_of_idempotency_key():+ assert "with_timestamps" in ElevenLabsTTS.idempotency_key_fields+ assert ElevenLabsTTS.input_schema["properties"]["with_timestamps"]["default"] is Falsediff --git a/tools/audio/elevenlabs_tts.py b/tools/audio/elevenlabs_tts.pyindex f122738b..b994e557 100644--- a/tools/audio/elevenlabs_tts.py+++ b/tools/audio/elevenlabs_tts.py@@ -2,7 +2,10 @@from __future__ import annotations+import base64+import jsonimport os+import reimport timefrom pathlib import Pathfrom typing import Any@@ -142,6 +145,16 @@ class ElevenLabsTTS(BaseTool):"default": "mp3_44100_128","enum": ["mp3_44100_128", "mp3_44100_192", "pcm_16000", "pcm_24000"],},+ "with_timestamps": {+ "type": "boolean",+ "default": False,+ "description": (+ "Use the /with-timestamps endpoint and return word-level timings "+ "(data.word_timestamps) plus the raw character alignment, saved next to "+ "the audio as <output>.alignment.json. Use for caption sync and "+ "word-cued animation without a separate transcription pass."+ ),+ },},}@@ -166,6 +179,7 @@ class ElevenLabsTTS(BaseTool):"previous_request_ids","next_request_ids","output_format",+ "with_timestamps",]side_effects = ["writes audio file to output_path", "calls ElevenLabs API"]user_visible_verification = ["Listen to generated audio for natural speech quality"]@@ -246,12 +260,16 @@ class ElevenLabsTTS(BaseTool):):if field in inputs:payload[field] = inputs[field]+ with_timestamps = bool(inputs.get("with_timestamps", False))+ endpoint = f"https://api.elevenlabs.io/v1/text-to-speech/{voice_id}"+ if with_timestamps:+ endpoint += "/with-timestamps"response = requests.post(- f"https://api.elevenlabs.io/v1/text-to-speech/{voice_id}",+ endpoint,headers={"xi-api-key": api_key,"Content-Type": "application/json",- "Accept": "audio/mpeg",+ "Accept": "application/json" if with_timestamps else "audio/mpeg",},json=payload,params={"output_format": output_format},@@ -259,6 +277,14 @@ class ElevenLabsTTS(BaseTool):)response.raise_for_status()+ alignment = None+ if with_timestamps:+ body = response.json()+ audio_bytes = base64.b64decode(body["audio_base64"])+ alignment = body.get("alignment") or body.get("normalized_alignment")+ else:+ audio_bytes = response.content+ext = "mp3" if "mp3" in output_format else "wav"output_path = Path(inputs.get("output_path", f"tts_output.{ext}"))output_path.parent.mkdir(parents=True, exist_ok=True)@@ -269,9 +295,29 @@ class ElevenLabsTTS(BaseTool):wav.setnchannels(1)wav.setsampwidth(2)wav.setframerate(int(output_format.split("_")[1]))- wav.writeframes(response.content)+ wav.writeframes(audio_bytes)else:- output_path.write_bytes(response.content)+ output_path.write_bytes(audio_bytes)++ artifacts = [str(output_path)]+ extra: dict[str, Any] = {}+ if with_timestamps:+ if not alignment:+ raise ValueError("ElevenLabs returned no alignment for a with-timestamps request")+ words = self.words_from_alignment(alignment)+ alignment_path = output_path.with_name(output_path.name + ".alignment.json")+ alignment_path.write_text(+ json.dumps({"text": text, "alignment": alignment, "words": words}, ensure_ascii=False),+ encoding="utf-8",+ )+ artifacts.append(str(alignment_path))+ extra = {+ "word_timestamps": words,+ "alignment_path": str(alignment_path),+ # False when the service normalised the text (numbers, symbols); timings+ # then refer to the spoken form returned in the alignment.+ "alignment_matches_text": "".join(alignment["characters"]) == text,+ }return ToolResult(success=True,@@ -284,7 +330,31 @@ class ElevenLabsTTS(BaseTool):"output": str(output_path),"format": output_format,"request_id": response.headers.get("request-id"),+ **extra,},- artifacts=[str(output_path)],+ artifacts=artifacts,model=model_id,)++ @staticmethod+ def words_from_alignment(alignment: dict[str, Any]) -> list[dict[str, Any]]:+ """Group ElevenLabs character timings into whitespace-delimited words.++ Each word gets the start of its first character and the end of its last one,+ in seconds from the start of the returned audio.+ """+ pieces = alignment["characters"]+ starts = alignment["character_start_times_seconds"]+ ends = alignment["character_end_times_seconds"]+ text, owner = "", [] # owner[k] = alignment entry that produced text[k]+ for i, piece in enumerate(pieces):+ text += piece+ owner.extend([i] * len(piece))+ return [+ {+ "word": m.group(),+ "start": round(starts[owner[m.start()]], 3),+ "end": round(ends[owner[m.end() - 1]], 3),+ }+ for m in re.finditer(r"\S+", text)+ ]