Skip to content

利用 AI 大模型和自动化工作流,根据主题或关键词一键生成高清短视频。Generate HD short videos from a topic or keyword with an automated AI workflow.

Bound Sonilo streaming events before JSON parsing

Rudy Celekli5bc715f07cc4
diff --git a/app/services/sonilo.py b/app/services/sonilo.py
index 2fe661f5..66b311ca 100644
--- a/app/services/sonilo.py
+++ b/app/services/sonilo.py
@@ -24,6 +24,9 @@ MAX_PROMPT_LENGTH = 2000
MAX_PROXY_BYTES = 300 * 1024 * 1024
MAX_GENERATED_AUDIO_BYTES = 30 * 1024 * 1024
MAX_ERROR_BODY_BYTES = 500
+# One event can hold the full permitted audio as base64, plus JSON metadata.
+MAX_STREAM_EVENT_BYTES = 4 * ((MAX_GENERATED_AUDIO_BYTES + 2) // 3) + 64 * 1024
+_STREAM_READ_BYTES = 64 * 1024
VIDEO_TO_MUSIC_SERVICE_ID = "video_to_music"
@@ -235,6 +238,27 @@ def _create_video_proxy(video_path: str) -> str:
return proxy_path
+def _iter_bounded_events(response: requests.Response):
+ """Read NDJSON without Requests' unbounded unterminated-line buffer."""
+ pending = bytearray()
+ for chunk in response.iter_content(chunk_size=_STREAM_READ_BYTES):
+ if not chunk:
+ continue
+ # Split only the bounded transport chunk; never concatenate an
+ # oversized line before checking its accumulated size.
+ pieces = chunk.split(b"\n")
+ for index, piece in enumerate(pieces):
+ if len(pending) + len(piece) > MAX_STREAM_EVENT_BYTES:
+ raise SoniloError("Sonilo streaming event exceeds the size limit")
+ pending.extend(piece)
+ if index < len(pieces) - 1:
+ if pending:
+ yield bytes(pending).rstrip(b"\r")
+ pending.clear()
+ if pending:
+ yield bytes(pending).rstrip(b"\r")
+
+
def _parse_event(raw_line: bytes) -> dict[str, Any]:
"""严格解析单条 NDJSON,禁止静默忽略截断或非对象响应。"""
try:
@@ -257,7 +281,7 @@ def _stream_audio(response: requests.Response, temp_audio_path: str) -> tuple[in
title = ""
completed = False
with open(temp_audio_path, "wb") as output:
- for raw_line in response.iter_lines():
+ for raw_line in _iter_bounded_events(response):
if not raw_line:
continue
event = _parse_event(raw_line)
diff --git a/test/services/test_sonilo.py b/test/services/test_sonilo.py
index 7442673d..a03b8748 100644
--- a/test/services/test_sonilo.py
+++ b/test/services/test_sonilo.py
@@ -37,7 +37,7 @@ class _StreamingResponse:
def iter_content(self, chunk_size):
if self.iter_error:
raise self.iter_error
- return iter(())
+ return iter(event + b"\n" for event in self.events)
def iter_lines(self):
if self.iter_error:
diff --git a/test/services/test_sonilo_event_bound.py b/test/services/test_sonilo_event_bound.py
new file mode 100644
index 00000000..ec4b055c
--- /dev/null
+++ b/test/services/test_sonilo_event_bound.py
@@ -0,0 +1,45 @@
+import base64
+import json
+from unittest.mock import patch
+
+import pytest
+import requests
+
+from app.services import sonilo
+
+
+class ChunkedResponse(requests.Response):
+ def __init__(self, chunks):
+ super().__init__()
+ self.chunks = chunks
+ self.consumed = 0
+
+ def iter_content(self, chunk_size=1, decode_unicode=False):
+ for chunk in self.chunks:
+ self.consumed += 1
+ yield chunk
+
+
[email protected]("terminated", [True, False])
+def test_oversized_sonilo_event_rejected_before_json_or_entire_response(tmp_path, terminated):
+ limit = 64
+ # Last chunk must not be consumed: even an unterminated line is bounded.
+ response = ChunkedResponse([b"x" * 33, b"x" * 33 + (b"\n" if terminated else b""), b"tail"])
+ with (
+ patch.object(sonilo, "MAX_STREAM_EVENT_BYTES", limit, create=True),
+ patch.object(sonilo, "_parse_event") as parse,
+ pytest.raises(sonilo.SoniloError, match="event.*limit"),
+ ):
+ sonilo._stream_audio(response, str(tmp_path / "staged.m4a"))
+ parse.assert_not_called()
+ assert response.consumed == 2
+
+
+def test_sonilo_split_and_multiple_events_preserve_audio(tmp_path):
+ audio = b"fixture-audio"
+ event = json.dumps({"type": "audio_chunk", "data": base64.b64encode(audio).decode()}).encode()
+ complete = b'{"type":"complete"}'
+ response = ChunkedResponse([event[:10], event[10:] + b"\r\n\n" + complete[:5], complete[5:]])
+ target = tmp_path / "staged.m4a"
+ assert sonilo._stream_audio(response, str(target)) == (len(audio), "")
+ assert target.read_bytes() == audio