Skip to content

利用 AI 大模型和自动化工作流,根据主题或关键词一键生成高清短视频。Generate HD short videos from a topic or keyword with an automated AI workflow.

fix: harden file paths and task queue limits

Harryef9877a31da2
diff --git a/app/controllers/manager/base_manager.py b/app/controllers/manager/base_manager.py
index 81251419..34804798 100644
--- a/app/controllers/manager/base_manager.py
+++ b/app/controllers/manager/base_manager.py
@@ -1,10 +1,17 @@
import threading
from typing import Any, Callable, Dict
+from loguru import logger
+
+
+class TaskQueueFullError(ValueError):
+ pass
+
class TaskManager:
- def __init__(self, max_concurrent_tasks: int):
+ def __init__(self, max_concurrent_tasks: int, max_queued_tasks: int = 100):
self.max_concurrent_tasks = max_concurrent_tasks
+ self.max_queued_tasks = max_queued_tasks
self.current_tasks = 0
self.lock = threading.Lock()
self.queue = self.create_queue()
@@ -15,11 +22,24 @@ class TaskManager:
def add_task(self, func: Callable, *args: Any, **kwargs: Any):
with self.lock:
if self.current_tasks < self.max_concurrent_tasks:
- print(f"add task: {func.__name__}, current_tasks: {self.current_tasks}")
+ logger.info(
+ f"add task: {func.__name__}, current_tasks: {self.current_tasks}"
+ )
self.execute_task(func, *args, **kwargs)
else:
- print(
- f"enqueue task: {func.__name__}, current_tasks: {self.current_tasks}"
+ queue_size = self.queue_size()
+ # 并发数已满时才进入排队。队列必须有上限,否则匿名接口可以持续
+ # 堆积任务对象和请求参数,最终造成内存耗尽或第三方 API 成本失控。
+ if queue_size >= self.max_queued_tasks:
+ logger.warning(
+ f"reject task: {func.__name__}, queue_size: {queue_size}, "
+ f"max_queued_tasks: {self.max_queued_tasks}"
+ )
+ raise TaskQueueFullError("task queue is full, please try again later")
+
+ logger.info(
+ f"enqueue task: {func.__name__}, current_tasks: {self.current_tasks}, "
+ f"queue_size: {queue_size}"
)
self.enqueue({"func": func, "args": args, "kwargs": kwargs})
@@ -62,3 +82,6 @@ class TaskManager:
def is_queue_empty(self):
raise NotImplementedError()
+
+ def queue_size(self):
+ raise NotImplementedError()
diff --git a/app/controllers/manager/memory_manager.py b/app/controllers/manager/memory_manager.py
index cf7321f5..4a2612bd 100644
--- a/app/controllers/manager/memory_manager.py
+++ b/app/controllers/manager/memory_manager.py
@@ -6,7 +6,7 @@ from app.controllers.manager.base_manager import TaskManager
class InMemoryTaskManager(TaskManager):
def create_queue(self):
- return Queue()
+ return Queue(maxsize=self.max_queued_tasks)
def enqueue(self, task: Dict):
self.queue.put(task)
@@ -16,3 +16,6 @@ class InMemoryTaskManager(TaskManager):
def is_queue_empty(self):
return self.queue.empty()
+
+ def queue_size(self):
+ return self.queue.qsize()
diff --git a/app/controllers/manager/redis_manager.py b/app/controllers/manager/redis_manager.py
index cad1912a..861af0f4 100644
--- a/app/controllers/manager/redis_manager.py
+++ b/app/controllers/manager/redis_manager.py
@@ -14,9 +14,14 @@ FUNC_MAP = {
class RedisTaskManager(TaskManager):
- def __init__(self, max_concurrent_tasks: int, redis_url: str):
+ def __init__(
+ self,
+ max_concurrent_tasks: int,
+ redis_url: str,
+ max_queued_tasks: int = 100,
+ ):
self.redis_client = redis.Redis.from_url(redis_url)
- super().__init__(max_concurrent_tasks)
+ super().__init__(max_concurrent_tasks, max_queued_tasks=max_queued_tasks)
def create_queue(self):
return "task_queue"
@@ -54,3 +59,6 @@ class RedisTaskManager(TaskManager):
def is_queue_empty(self):
return self.redis_client.llen(self.queue) == 0
+
+ def queue_size(self):
+ return self.redis_client.llen(self.queue)
diff --git a/app/controllers/v1/video.py b/app/controllers/v1/video.py
index 330b2ab2..5822fd88 100644
--- a/app/controllers/v1/video.py
+++ b/app/controllers/v1/video.py
@@ -11,6 +11,7 @@ from loguru import logger
from app.config import config
from app.controllers import base
+from app.controllers.manager.base_manager import TaskQueueFullError
from app.controllers.manager.memory_manager import InMemoryTaskManager
from app.controllers.manager.redis_manager import RedisTaskManager
from app.controllers.v1.base import new_router
@@ -30,7 +31,7 @@ from app.models.schema import (
)
from app.services import state as sm
from app.services import task as tm
-from app.utils import utils
+from app.utils import file_security, utils
# 认证依赖项
# router = new_router(dependencies=[Depends(base.verify_token)])
@@ -42,15 +43,21 @@ _redis_port = config.app.get("redis_port", 6379)
_redis_db = config.app.get("redis_db", 0)
_redis_password = config.app.get("redis_password", None)
_max_concurrent_tasks = config.app.get("max_concurrent_tasks", 5)
+_max_queued_tasks = config.app.get("max_queued_tasks", 100)
redis_url = f"redis://:{_redis_password}@{_redis_host}:{_redis_port}/{_redis_db}"
# 根据配置选择合适的任务管理器
if _enable_redis:
task_manager = RedisTaskManager(
- max_concurrent_tasks=_max_concurrent_tasks, redis_url=redis_url
+ max_concurrent_tasks=_max_concurrent_tasks,
+ redis_url=redis_url,
+ max_queued_tasks=_max_queued_tasks,
)
else:
- task_manager = InMemoryTaskManager(max_concurrent_tasks=_max_concurrent_tasks)
+ task_manager = InMemoryTaskManager(
+ max_concurrent_tasks=_max_concurrent_tasks,
+ max_queued_tasks=_max_queued_tasks,
+ )
def _sanitize_upload_filename(filename: str, request_id: str) -> str:
@@ -67,35 +74,42 @@ def _sanitize_upload_filename(filename: str, request_id: str) -> str:
def _resolve_path_within_directory(base_dir: str, unsafe_path: str, request_id: str) -> str:
- # 对用户传入的相对路径做归一化,并强制要求结果仍然落在指定目录内,
- # 这样可以阻止通过 ../ 或绝对路径逃逸到任务目录之外读取任意文件。
- base_dir_real = os.path.realpath(base_dir)
- resolved_path = os.path.realpath(os.path.join(base_dir_real, unsafe_path))
-
try:
- common_path = os.path.commonpath([base_dir_real, resolved_path])
- except ValueError:
+ return file_security.resolve_path_within_directory(base_dir, unsafe_path)
+ except ValueError as exc:
+ logger.warning(
+ f"reject unsafe file path, request_id: {request_id}, path: {unsafe_path}, "
+ f"error: {str(exc)}"
+ )
raise HttpException(
task_id=request_id,
- status_code=403,
+ status_code=404 if str(exc) == "file does not exist" else 403,
message=f"{request_id}: invalid file path",
)
- if common_path != base_dir_real:
- raise HttpException(
- task_id=request_id,
- status_code=403,
- message=f"{request_id}: access to the requested file is forbidden",
- )
+def _task_file_to_uri(file: str, endpoint: str, task_dir: str, request_id: str) -> str:
+ if not isinstance(file, str):
+ return file
- if not os.path.isfile(resolved_path):
- raise HttpException(
- task_id=request_id,
- status_code=404,
- message=f"{request_id}: file not found",
+ if file.startswith(("http://", "https://")):
+ return file
+
+ try:
+ resolved_path = file_security.resolve_path_within_directory(task_dir, file)
+ except ValueError as exc:
+ # 任务状态理论上只应保存任务目录内的产物路径。这里不再继续拼接 URL,
+ # 避免把异常路径包装成可访问链接;同时保留原值,便于排查历史脏数据。
+ logger.warning(
+ f"skip unsafe task output path, request_id: {request_id}, path: {file}, "
+ f"error: {str(exc)}"
)
+ return file
- return resolved_path
+ relative_path = os.path.relpath(resolved_path, task_dir).replace("\\", "/")
+ uri_path = f"tasks/{relative_path}"
+ if endpoint:
+ return f"{endpoint.rstrip('/')}/{uri_path}"
+ return f"/{uri_path}"
@router.post("/videos", response_model=TaskResponse, summary="Generate a short video")
@@ -136,6 +150,14 @@ def create_task(
task_manager.add_task(tm.start, task_id=task_id, params=body, stop_at=stop_at)
logger.success(f"Task created: {utils.to_json(task)}")
return utils.get_response(200, task)
+ except TaskQueueFullError as e:
+ sm.state.delete_task(task_id)
+ logger.warning(
+ f"reject task because queue is full, request_id: {request_id}, task_id: {task_id}"
+ )
+ raise HttpException(
+ task_id=task_id, status_code=429, message=f"{request_id}: {str(e)}"
+ )
except ValueError as e:
raise HttpException(
task_id=task_id, status_code=400, message=f"{request_id}: {str(e)}"
@@ -166,37 +188,24 @@ def get_task(
task_id: str = Path(..., description="Task ID"),
query: TaskQueryRequest = Depends(),
):
- endpoint = config.app.get("endpoint", "")
- if not endpoint:
- endpoint = str(request.base_url)
- endpoint = endpoint.rstrip("/")
-
request_id = base.get_task_id(request)
+ endpoint = config.app.get("endpoint", "").rstrip("/")
task = sm.state.get_task(task_id)
if task:
task_dir = utils.task_dir()
-
- def file_to_uri(file):
- if not file.startswith(endpoint):
- _uri_path = file.replace(task_dir, "tasks").replace("\\", "/")
- _uri_path = f"{endpoint}/{_uri_path}"
- else:
- _uri_path = file
- return _uri_path
+ response_task = dict(task)
if "videos" in task:
- videos = task["videos"]
- urls = []
- for v in videos:
- urls.append(file_to_uri(v))
- task["videos"] = urls
+ response_task["videos"] = [
+ _task_file_to_uri(v, endpoint, task_dir, request_id)
+ for v in task["videos"]
+ ]
if "combined_videos" in task:
- combined_videos = task["combined_videos"]
- urls = []
- for v in combined_videos:
- urls.append(file_to_uri(v))
- task["combined_videos"] = urls
- return utils.get_response(200, task)
+ response_task["combined_videos"] = [
+ _task_file_to_uri(v, endpoint, task_dir, request_id)
+ for v in task["combined_videos"]
+ ]
+ return utils.get_response(200, response_task)
raise HttpException(
task_id=task_id, status_code=404, message=f"{request_id}: task not found"
@@ -235,11 +244,14 @@ def get_bgm_list(request: Request):
files = glob.glob(os.path.join(song_dir, suffix))
bgm_list = []
for file in files:
+ filename = os.path.basename(file)
bgm_list.append(
{
- "name": os.path.basename(file),
+ "name": filename,
"size": os.path.getsize(file),
- "file": file,
+ # 只返回文件名,避免把服务器绝对路径暴露给调用方。
+ # 服务端后续会把该文件名解析回 songs 白名单目录。
+ "file": filename,
}
)
response = {"files": bgm_list}
@@ -263,7 +275,7 @@ def upload_bgm_file(request: Request, file: UploadFile = File(...)):
# If the file already exists, it will be overwritten
file.file.seek(0)
buffer.write(file.file.read())
- response = {"file": save_path}
+ response = {"file": safe_filename}
return utils.get_response(200, response)
raise HttpException(
@@ -284,11 +296,14 @@ def get_video_materials_list(request: Request):
files.sort(key=lambda file_path: os.path.basename(file_path).lower())
video_materials_list = []
for file in files:
+ filename = os.path.basename(file)
video_materials_list.append(
{
- "name": os.path.basename(file),
+ "name": filename,
"size": os.path.getsize(file),
- "file": file,
+ # 与 BGM 一样,只返回文件名;创建任务时再在 local_videos
+ # 白名单目录内解析,避免 API 泄露宿主机绝对路径。
+ "file": filename,
}
)
response = {"files": video_materials_list}
@@ -315,7 +330,7 @@ def upload_video_material_file(request: Request, file: UploadFile = File(...)):
# If the file already exists, it will be overwritten
file.file.seek(0)
buffer.write(file.file.read())
- response = {"file": save_path}
+ response = {"file": safe_filename}
return utils.get_response(200, response)
raise HttpException(
diff --git a/app/services/video.py b/app/services/video.py
index 0c1d0989..ae41ca07 100644
--- a/app/services/video.py
+++ b/app/services/video.py
@@ -31,7 +31,7 @@ from app.models.schema import (
VideoTransitionMode,
)
from app.services.utils import video_effects
-from app.utils import utils
+from app.utils import file_security, utils
class SubClippedVideoClip:
def __init__(self, file_path, start_time=None, end_time=None, width=None, height=None, duration=None):
@@ -55,6 +55,7 @@ audio_codec = "aac"
audio_bitrate = "192k"
video_codec = "libx264"
fps = 30
+_BGM_EXTENSIONS = (".mp3",)
def get_ffmpeg_binary():
@@ -237,8 +238,26 @@ def get_bgm_file(bgm_type: str = "random", bgm_file: str = ""):
if not bgm_type:
return ""
- if bgm_file and os.path.exists(bgm_file):
- return bgm_file
+ if bgm_file:
+ song_dir = utils.song_dir()
+ try:
+ resolved_bgm_file = file_security.resolve_path_within_directory(
+ song_dir, bgm_file
+ )
+ except ValueError as exc:
+ # API 请求里的 bgm_file 来自用户输入,不能直接把任意绝对路径交给
+ # MoviePy 打开。这里强制限制到 resource/songs 目录,阻止读取
+ # /etc/passwd、配置文件、密钥等非背景音乐文件。
+ logger.warning(
+ f"reject unsafe bgm file: {bgm_file}, song_dir: {song_dir}, error: {str(exc)}"
+ )
+ return ""
+
+ if not resolved_bgm_file.lower().endswith(_BGM_EXTENSIONS):
+ logger.warning(f"reject unsupported bgm file extension: {resolved_bgm_file}")
+ return ""
+
+ return resolved_bgm_file
if bgm_type == "random":
suffix = "*.mp3"
@@ -639,23 +658,41 @@ def preprocess_video(materials: List[MaterialInfo], clip_duration=4):
# 仅返回通过预处理校验的素材,避免低分辨率图片继续进入后续的视频合成流程。
valid_materials = []
+ local_videos_dir = utils.storage_dir("local_videos", create=True)
for material in materials:
if not material.url:
continue
- ext = utils.parse_extension(material.url)
- material_source_path = material.url
+ try:
+ material_source_path = file_security.resolve_path_within_directory(
+ local_videos_dir, material.url
+ )
+ except ValueError as exc:
+ # local video_source 的素材路径来自 API 参数,必须限制在专用素材目录。
+ # 允许用户传文件名,也兼容历史返回的绝对路径,但不允许逃逸到系统
+ # 其他目录,避免任意文件读取或通过 MoviePy 探测本地敏感文件。
+ logger.warning(
+ f"skip unsafe local material: {material.url}, "
+ f"local_videos_dir: {local_videos_dir}, error: {str(exc)}"
+ )
+ continue
+
+ ext = utils.parse_extension(material_source_path)
try:
# 图片素材直接按图片方式读取,避免先走 VideoFileClip 误判后触发不稳定的回退分支。
if ext in const.FILE_TYPE_IMAGES:
- clip, material_source_path = _open_image_clip_with_fallback(material.url)
+ clip, material_source_path = _open_image_clip_with_fallback(
+ material_source_path
+ )
else:
- clip = _open_video_clip_quietly(material.url)
+ clip = _open_video_clip_quietly(material_source_path)
except Exception:
# 非标准扩展名或探测失败时再回退到图片模式,兼容历史上直接传本地图片路径的情况。
try:
- clip, material_source_path = _open_image_clip_with_fallback(material.url)
+ clip, material_source_path = _open_image_clip_with_fallback(
+ material_source_path
+ )
except Exception as exc:
logger.warning(
f"skip unreadable local material: {material.url}, error: {str(exc)}"
diff --git a/app/utils/file_security.py b/app/utils/file_security.py
new file mode 100644
index 00000000..00366fcd
--- /dev/null
+++ b/app/utils/file_security.py
@@ -0,0 +1,35 @@
+import os
+
+
+def resolve_path_within_directory(
+ base_dir: str,
+ unsafe_path: str,
+ *,
+ require_file: bool = True,
+) -> str:
+ # 用户传入的路径可能是文件名、相对路径、绝对路径,也可能夹带 `../`。
+ # 这里统一解析成真实路径,并用 commonpath 判断它是否仍在允许目录内。
+ # 这样比简单判断字符串前缀可靠,可以覆盖符号链接、重复分隔符、相对路径
+ # 等场景,适用于上传目录、素材目录、任务产物目录这类白名单目录。
+ if not unsafe_path:
+ raise ValueError("empty path is not allowed")
+
+ base_dir_real = os.path.realpath(base_dir)
+ candidate_path = unsafe_path
+ if not os.path.isabs(candidate_path):
+ candidate_path = os.path.join(base_dir_real, candidate_path)
+
+ resolved_path = os.path.realpath(candidate_path)
+ try:
+ common_path = os.path.commonpath([base_dir_real, resolved_path])
+ except ValueError as exc:
+ # Windows 下不同盘符会触发 ValueError,这类路径一定不属于允许目录。
+ raise ValueError("path is outside the allowed directory") from exc
+
+ if common_path != base_dir_real:
+ raise ValueError("path is outside the allowed directory")
+
+ if require_file and not os.path.isfile(resolved_path):
+ raise ValueError("file does not exist")
+
+ return resolved_path
diff --git a/config.example.toml b/config.example.toml
index 39f8ed20..60ca7873 100644
--- a/config.example.toml
+++ b/config.example.toml
@@ -223,6 +223,9 @@ redis_password = ""
# 文生视频时的最大并发任务数
max_concurrent_tasks = 5
+# 并发任务已满后的最大排队任务数,超过后接口返回 429,避免匿名请求无限堆积
+max_queued_tasks = 100
+
[whisper]
# Only effective when subtitle_provider is "whisper"
diff --git a/test/services/test_video.py b/test/services/test_video.py
index 60add022..5031c98e 100644
--- a/test/services/test_video.py
+++ b/test/services/test_video.py
@@ -1,7 +1,9 @@
import unittest
import os
+import shutil
import sys
+import tempfile
import types
from contextlib import redirect_stdout
from io import StringIO
@@ -12,12 +14,71 @@ from moviepy import (
)
# add project root to python path
sys.path.insert(0, str(Path(__file__).parent.parent.parent))
+from app.config import config
+from app.controllers.manager.base_manager import TaskQueueFullError
+from app.controllers.manager.memory_manager import InMemoryTaskManager
+from app.controllers.v1 import video as video_controller
+from app.models import const
from app.models.schema import MaterialInfo
+from app.services import state as sm
from app.services import video as vd
from app.utils import utils
resources_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "resources")
+
+class _FakeRequest:
+ def __init__(self):
+ self.headers = {"x-task-id": "test-request"}
+
+
+class TestSecurityControls(unittest.TestCase):
+ def setUp(self):
+ self.original_app_config = dict(config.app)
+
+ def tearDown(self):
+ config.app.clear()
+ config.app.update(self.original_app_config)
+
+ def test_task_query_returns_relative_task_url_without_mutating_state(self):
+ """
+ endpoint 未显式配置时,任务查询接口不能使用 Host 派生绝对 URL,
+ 也不能把展示 URL 回写到任务状态里,否则不同 Host 查询会污染结果。
+ """
+ task_id = "security-task-url"
+ task_dir = utils.task_dir(task_id)
+ video_path = os.path.join(task_dir, "final-1.mp4")
+ Path(video_path).write_bytes(b"fake-video")
+ config.app["endpoint"] = ""
+
+ try:
+ sm.state.update_task(
+ task_id,
+ state=const.TASK_STATE_COMPLETE,
+ videos=[video_path],
+ combined_videos=[video_path],
+ )
+
+ response = video_controller.get_task(_FakeRequest(), task_id=task_id)
+
+ self.assertEqual(response["data"]["videos"], [f"/tasks/{task_id}/final-1.mp4"])
+ self.assertEqual(sm.state.get_task(task_id)["videos"], [video_path])
+ finally:
+ sm.state.delete_task(task_id)
+ shutil.rmtree(task_dir, ignore_errors=True)
+
+ def test_in_memory_task_manager_rejects_when_queue_is_full(self):
+ """
+ 并发数用尽后,等待队列必须有硬上限。这里用 max_concurrent_tasks=0
+ 强制任务进入队列,验证超过 max_queued_tasks 时会拒绝继续入队。
+ """
+ manager = InMemoryTaskManager(max_concurrent_tasks=0, max_queued_tasks=1)
+
+ manager.add_task(lambda: None)
+
+ with self.assertRaises(TaskQueueFullError):
+ manager.add_task(lambda: None)
+
class TestVideoService(unittest.TestCase):
def setUp(self):
self.test_img_path = os.path.join(resources_dir, "1.png")
@@ -28,28 +89,70 @@ class TestVideoService(unittest.TestCase):
def test_preprocess_video(self):
if not os.path.exists(self.test_img_path):
self.fail(f"test image not found: {self.test_img_path}")
-
+
+ local_videos_dir = utils.storage_dir("local_videos", create=True)
+ safe_img_path = os.path.join(local_videos_dir, "test-preprocess-1.png")
+ shutil.copy2(self.test_img_path, safe_img_path)
+
# test preprocess_video function
m = MaterialInfo()
- m.url = self.test_img_path
+ m.url = os.path.basename(safe_img_path)
m.provider = "local"
print(m)
-
+
+ try:
+ materials = vd.preprocess_video([m], clip_duration=4)
+ print(materials)
+
+ # verify result
+ self.assertIsNotNone(materials)
+ self.assertEqual(len(materials), 1)
+ self.assertTrue(materials[0].url.endswith(".mp4"))
+
+ # moviepy get video info
+ clip = VideoFileClip(materials[0].url)
+ print(clip)
+
+ # clean generated test video file
+ if os.path.exists(materials[0].url):
+ os.remove(materials[0].url)
+ finally:
+ if os.path.exists(safe_img_path):
+ os.remove(safe_img_path)
+
+ def test_preprocess_video_rejects_material_outside_local_videos(self):
+ """
+ local 素材路径来自 API 参数,不能允许任意绝对路径进入 MoviePy。
+ 这里验证非 local_videos 白名单目录内的路径会被跳过,避免任意文件读取。
+ """
+ m = MaterialInfo(provider="local", url=self.test_img_path)
+
materials = vd.preprocess_video([m], clip_duration=4)
- print(materials)
-
- # verify result
- self.assertIsNotNone(materials)
- self.assertEqual(len(materials), 1)
- self.assertTrue(materials[0].url.endswith(".mp4"))
-
- # moviepy get video info
- clip = VideoFileClip(materials[0].url)
- print(clip)
-
- # clean generated test video file
- if os.path.exists(materials[0].url):
- os.remove(materials[0].url)
+
+ self.assertEqual(materials, [])
+
+ def test_get_bgm_file_accepts_song_directory_filename(self):
+ """
+ BGM 列表接口现在只暴露文件名;生成视频时应能把文件名安全解析回
+ resource/songs 白名单目录,保持正常使用路径可用。
+ """
+ song_dir = utils.song_dir()
+ bgm_path = os.path.join(song_dir, "test-safe-bgm.mp3")
+ Path(bgm_path).write_bytes(b"fake-mp3")
+
+ try:
+ self.assertEqual(vd.get_bgm_file(bgm_file="test-safe-bgm.mp3"), bgm_path)
+ finally:
+ if os.path.exists(bgm_path):
+ os.remove(bgm_path)
+
+ def test_get_bgm_file_rejects_path_outside_song_directory(self):
+ """
+ 用户传入的 bgm_file 不能直接作为本地路径打开,否则可能读取系统文件。
+ 即使外部文件存在,也必须因为不在 songs 目录内被拒绝。
+ """
+ with tempfile.NamedTemporaryFile(suffix=".mp3") as temp_bgm:
+ self.assertEqual(vd.get_bgm_file(bgm_file=temp_bgm.name), "")
def test_get_ffmpeg_binary_uses_configured_env_path(self):
"""配置中显式指定 ffmpeg 时,应优先使用该路径。"""