Skip to content

World's first open-source, agentic video production system. 12 production pipelines, 100+ tools, 700+ agent skill and production-knowledge files. Turn your AI coding assistant into a full video production studio.

feat(elevenlabs_tts): optional word-level timestamps (#752)

hojanyazd380b69126c2
diff --git a/.agents/skills/elevenlabs/SKILL.md b/.agents/skills/elevenlabs/SKILL.md
index d81dc390..db8e3cda 100644
--- a/.agents/skills/elevenlabs/SKILL.md
+++ b/.agents/skills/elevenlabs/SKILL.md
@@ -279,6 +279,34 @@ export const sceneDurations = manifest.scenes.reduce((acc, scene) => {
// <Series.Sequence durationInFrames={sceneDurations.title}>
```
+### Word-Level Timings (direct `elevenlabs_tts`)
+
+When captions, word-synced headlines or animation cues must follow the real
+speech, pass `with_timestamps: true` to the direct `elevenlabs_tts` tool instead
+of transcribing the audio afterwards:
+
+```python
+result = ElevenLabsTTS().execute({
+ "text": paragraph,
+ "voice_id": voice_id,
+ "previous_text": prev_paragraph, # continuity across separately synthesized paragraphs
+ "next_text": next_paragraph,
+ "with_timestamps": True,
+ "output_path": "projects/<id>/assets/audio/tts/para_03.mp3",
+})
+result.data["word_timestamps"] # [{"word": "Доверие.", "start": 0.12, "end": 0.71}, ...]
+result.data["alignment_path"] # para_03.mp3.alignment.json (raw character alignment)
+```
+
+- Times are seconds from the start of that audio file. When paragraphs are placed
+ on one timeline, add each paragraph's placement offset.
+- Words are whitespace-delimited and keep their punctuation, so they line up with
+ `text.split()` of the script.
+- `alignment_matches_text` is `false` when the service normalised the text
+ (numbers, symbols); timings then follow the spoken form in the alignment.
+- Synthesizing per paragraph gives exact paragraph boundaries for scene cuts;
+ the word timings place captions and cues inside each paragraph.
+
### Audio Timing Patterns
```tsx
diff --git a/tests/tools/test_elevenlabs_timestamps.py b/tests/tools/test_elevenlabs_timestamps.py
new file mode 100644
index 00000000..00111eac
--- /dev/null
+++ b/tests/tools/test_elevenlabs_timestamps.py
@@ -0,0 +1,108 @@
+"""ElevenLabs TTS: optional word-level timings via the /with-timestamps endpoint."""
+
+from __future__ import annotations
+
+import base64
+import json
+import wave
+from unittest.mock import Mock
+
+from tools.audio.elevenlabs_tts import ElevenLabsTTS
+
+
+def _alignment(text: str, step: float = 0.1) -> dict:
+ chars = list(text)
+ return {
+ "characters": chars,
+ "character_start_times_seconds": [round(i * step, 3) for i in range(len(chars))],
+ "character_end_times_seconds": [round((i + 1) * step, 3) for i in range(len(chars))],
+ }
+
+
+def _json_response(text: str, audio: bytes) -> Mock:
+ body = {"audio_base64": base64.b64encode(audio).decode(), "alignment": _alignment(text)}
+ return Mock(json=Mock(return_value=body), headers={"request-id": "r1"}, content=b"")
+
+
+def test_with_timestamps_calls_endpoint_and_returns_words(monkeypatch, tmp_path):
+ monkeypatch.setenv("ELEVENLABS_API_KEY", "test")
+ text = "Много знакомств. А кому позвонить?"
+ post = Mock(return_value=_json_response(text, b"ID3fake-mp3"))
+ monkeypatch.setattr("requests.post", post)
+ out = tmp_path / "para.mp3"
+
+ result = ElevenLabsTTS().execute({"text": text, "with_timestamps": True, "output_path": str(out)})
+
+ assert result.success, result.error
+ assert post.call_args.args[0].endswith("/with-timestamps")
+ assert post.call_args.kwargs["headers"]["Accept"] == "application/json"
+ assert "with_timestamps" not in post.call_args.kwargs["json"] # tool option, not an API field
+ assert out.read_bytes() == b"ID3fake-mp3"
+
+ words = result.data["word_timestamps"]
+ assert [w["word"] for w in words] == text.split()
+ assert words[0] == {"word": "Много", "start": 0.0, "end": 0.5}
+ assert words[1]["start"] == 0.6 and words[1]["end"] == 1.6 # "знакомств." incl. punctuation
+ assert result.data["alignment_matches_text"] is True
+
+ saved = json.loads((tmp_path / "para.mp3.alignment.json").read_text(encoding="utf-8"))
+ assert saved["text"] == text and saved["words"] == words
+ assert str(tmp_path / "para.mp3.alignment.json") in result.artifacts
+
+
+def test_with_timestamps_pcm_output_is_wrapped_as_wav(monkeypatch, tmp_path):
+ monkeypatch.setenv("ELEVENLABS_API_KEY", "test")
+ post = Mock(return_value=_json_response("Hi there", b"\x00\x00" * 50))
+ monkeypatch.setattr("requests.post", post)
+ out = tmp_path / "voice.wav"
+
+ result = ElevenLabsTTS().execute(
+ {"text": "Hi there", "with_timestamps": True, "output_format": "pcm_16000", "output_path": str(out)}
+ )
+
+ assert result.success, result.error
+ with wave.open(str(out)) as wav:
+ assert wav.getframerate() == 16000 and wav.getnframes() == 50
+ assert [w["word"] for w in result.data["word_timestamps"]] == ["Hi", "there"]
+
+
+def test_default_path_is_unchanged(monkeypatch, tmp_path):
+ monkeypatch.setenv("ELEVENLABS_API_KEY", "test")
+ post = Mock(return_value=Mock(content=b"ID3audio", headers={"request-id": "r2"}))
+ monkeypatch.setattr("requests.post", post)
+
+ result = ElevenLabsTTS().execute({"text": "Hello", "output_path": str(tmp_path / "a.mp3")})
+
+ assert result.success, result.error
+ assert not post.call_args.args[0].endswith("/with-timestamps")
+ assert post.call_args.kwargs["headers"]["Accept"] == "audio/mpeg"
+ assert "word_timestamps" not in result.data
+ assert result.artifacts == [str(tmp_path / "a.mp3")]
+
+
+def test_missing_alignment_fails_loudly(monkeypatch, tmp_path):
+ monkeypatch.setenv("ELEVENLABS_API_KEY", "test")
+ body = {"audio_base64": base64.b64encode(b"x").decode()}
+ monkeypatch.setattr("requests.post", Mock(return_value=Mock(json=Mock(return_value=body), headers={})))
+
+ result = ElevenLabsTTS().execute({"text": "Hello", "with_timestamps": True, "output_path": str(tmp_path / "a.mp3")})
+
+ assert not result.success
+ assert "alignment" in result.error
+
+
+def test_words_from_alignment_handles_multichar_entries():
+ alignment = {
+ "characters": ["Hel", "lo", " ", "wo", "rld!"],
+ "character_start_times_seconds": [0.0, 0.3, 0.5, 0.6, 0.8],
+ "character_end_times_seconds": [0.3, 0.5, 0.6, 0.8, 1.1],
+ }
+ assert ElevenLabsTTS.words_from_alignment(alignment) == [
+ {"word": "Hello", "start": 0.0, "end": 0.5},
+ {"word": "world!", "start": 0.6, "end": 1.1},
+ ]
+
+
+def test_with_timestamps_is_part_of_idempotency_key():
+ assert "with_timestamps" in ElevenLabsTTS.idempotency_key_fields
+ assert ElevenLabsTTS.input_schema["properties"]["with_timestamps"]["default"] is False
diff --git a/tools/audio/elevenlabs_tts.py b/tools/audio/elevenlabs_tts.py
index f122738b..b994e557 100644
--- a/tools/audio/elevenlabs_tts.py
+++ b/tools/audio/elevenlabs_tts.py
@@ -2,7 +2,10 @@
from __future__ import annotations
+import base64
+import json
import os
+import re
import time
from pathlib import Path
from typing import Any
@@ -142,6 +145,16 @@ class ElevenLabsTTS(BaseTool):
"default": "mp3_44100_128",
"enum": ["mp3_44100_128", "mp3_44100_192", "pcm_16000", "pcm_24000"],
},
+ "with_timestamps": {
+ "type": "boolean",
+ "default": False,
+ "description": (
+ "Use the /with-timestamps endpoint and return word-level timings "
+ "(data.word_timestamps) plus the raw character alignment, saved next to "
+ "the audio as <output>.alignment.json. Use for caption sync and "
+ "word-cued animation without a separate transcription pass."
+ ),
+ },
},
}
@@ -166,6 +179,7 @@ class ElevenLabsTTS(BaseTool):
"previous_request_ids",
"next_request_ids",
"output_format",
+ "with_timestamps",
]
side_effects = ["writes audio file to output_path", "calls ElevenLabs API"]
user_visible_verification = ["Listen to generated audio for natural speech quality"]
@@ -246,12 +260,16 @@ class ElevenLabsTTS(BaseTool):
):
if field in inputs:
payload[field] = inputs[field]
+ with_timestamps = bool(inputs.get("with_timestamps", False))
+ endpoint = f"https://api.elevenlabs.io/v1/text-to-speech/{voice_id}"
+ if with_timestamps:
+ endpoint += "/with-timestamps"
response = requests.post(
- f"https://api.elevenlabs.io/v1/text-to-speech/{voice_id}",
+ endpoint,
headers={
"xi-api-key": api_key,
"Content-Type": "application/json",
- "Accept": "audio/mpeg",
+ "Accept": "application/json" if with_timestamps else "audio/mpeg",
},
json=payload,
params={"output_format": output_format},
@@ -259,6 +277,14 @@ class ElevenLabsTTS(BaseTool):
)
response.raise_for_status()
+ alignment = None
+ if with_timestamps:
+ body = response.json()
+ audio_bytes = base64.b64decode(body["audio_base64"])
+ alignment = body.get("alignment") or body.get("normalized_alignment")
+ else:
+ audio_bytes = response.content
+
ext = "mp3" if "mp3" in output_format else "wav"
output_path = Path(inputs.get("output_path", f"tts_output.{ext}"))
output_path.parent.mkdir(parents=True, exist_ok=True)
@@ -269,9 +295,29 @@ class ElevenLabsTTS(BaseTool):
wav.setnchannels(1)
wav.setsampwidth(2)
wav.setframerate(int(output_format.split("_")[1]))
- wav.writeframes(response.content)
+ wav.writeframes(audio_bytes)
else:
- output_path.write_bytes(response.content)
+ output_path.write_bytes(audio_bytes)
+
+ artifacts = [str(output_path)]
+ extra: dict[str, Any] = {}
+ if with_timestamps:
+ if not alignment:
+ raise ValueError("ElevenLabs returned no alignment for a with-timestamps request")
+ words = self.words_from_alignment(alignment)
+ alignment_path = output_path.with_name(output_path.name + ".alignment.json")
+ alignment_path.write_text(
+ json.dumps({"text": text, "alignment": alignment, "words": words}, ensure_ascii=False),
+ encoding="utf-8",
+ )
+ artifacts.append(str(alignment_path))
+ extra = {
+ "word_timestamps": words,
+ "alignment_path": str(alignment_path),
+ # False when the service normalised the text (numbers, symbols); timings
+ # then refer to the spoken form returned in the alignment.
+ "alignment_matches_text": "".join(alignment["characters"]) == text,
+ }
return ToolResult(
success=True,
@@ -284,7 +330,31 @@ class ElevenLabsTTS(BaseTool):
"output": str(output_path),
"format": output_format,
"request_id": response.headers.get("request-id"),
+ **extra,
},
- artifacts=[str(output_path)],
+ artifacts=artifacts,
model=model_id,
)
+
+ @staticmethod
+ def words_from_alignment(alignment: dict[str, Any]) -> list[dict[str, Any]]:
+ """Group ElevenLabs character timings into whitespace-delimited words.
+
+ Each word gets the start of its first character and the end of its last one,
+ in seconds from the start of the returned audio.
+ """
+ pieces = alignment["characters"]
+ starts = alignment["character_start_times_seconds"]
+ ends = alignment["character_end_times_seconds"]
+ text, owner = "", [] # owner[k] = alignment entry that produced text[k]
+ for i, piece in enumerate(pieces):
+ text += piece
+ owner.extend([i] * len(piece))
+ return [
+ {
+ "word": m.group(),
+ "start": round(starts[owner[m.start()]], 3),
+ "end": round(ends[owner[m.end() - 1]], 3),
+ }
+ for m in re.finditer(r"\S+", text)
+ ]