利用 AI 大模型和自动化工作流,根据主题或关键词一键生成高清短视频。Generate HD short videos from a topic or keyword with an automated AI workflow.
Python ai-video-generatorcontent-creationffmpeginstagram-reelsllmpythonshort-videosubtitlestext-to-speechtiktokvideo-automationvideo-workflowworkflow-automationyoutube-shorts
fix: harden file paths and task queue limits
Harryef9877a31da2
diff --git a/app/controllers/manager/base_manager.py b/app/controllers/manager/base_manager.pyindex 81251419..34804798 100644--- a/app/controllers/manager/base_manager.py+++ b/app/controllers/manager/base_manager.py@@ -1,10 +1,17 @@import threadingfrom typing import Any, Callable, Dict+from loguru import logger+++class TaskQueueFullError(ValueError):+ pass+class TaskManager:- def __init__(self, max_concurrent_tasks: int):+ def __init__(self, max_concurrent_tasks: int, max_queued_tasks: int = 100):self.max_concurrent_tasks = max_concurrent_tasks+ self.max_queued_tasks = max_queued_tasksself.current_tasks = 0self.lock = threading.Lock()self.queue = self.create_queue()@@ -15,11 +22,24 @@ class TaskManager:def add_task(self, func: Callable, *args: Any, **kwargs: Any):with self.lock:if self.current_tasks < self.max_concurrent_tasks:- print(f"add task: {func.__name__}, current_tasks: {self.current_tasks}")+ logger.info(+ f"add task: {func.__name__}, current_tasks: {self.current_tasks}"+ )self.execute_task(func, *args, **kwargs)else:- print(- f"enqueue task: {func.__name__}, current_tasks: {self.current_tasks}"+ queue_size = self.queue_size()+ # 并发数已满时才进入排队。队列必须有上限,否则匿名接口可以持续+ # 堆积任务对象和请求参数,最终造成内存耗尽或第三方 API 成本失控。+ if queue_size >= self.max_queued_tasks:+ logger.warning(+ f"reject task: {func.__name__}, queue_size: {queue_size}, "+ f"max_queued_tasks: {self.max_queued_tasks}"+ )+ raise TaskQueueFullError("task queue is full, please try again later")++ logger.info(+ f"enqueue task: {func.__name__}, current_tasks: {self.current_tasks}, "+ f"queue_size: {queue_size}")self.enqueue({"func": func, "args": args, "kwargs": kwargs})@@ -62,3 +82,6 @@ class TaskManager:def is_queue_empty(self):raise NotImplementedError()++ def queue_size(self):+ raise NotImplementedError()diff --git a/app/controllers/manager/memory_manager.py b/app/controllers/manager/memory_manager.pyindex cf7321f5..4a2612bd 100644--- a/app/controllers/manager/memory_manager.py+++ b/app/controllers/manager/memory_manager.py@@ -6,7 +6,7 @@ from app.controllers.manager.base_manager import TaskManagerclass InMemoryTaskManager(TaskManager):def create_queue(self):- return Queue()+ return Queue(maxsize=self.max_queued_tasks)def enqueue(self, task: Dict):self.queue.put(task)@@ -16,3 +16,6 @@ class InMemoryTaskManager(TaskManager):def is_queue_empty(self):return self.queue.empty()++ def queue_size(self):+ return self.queue.qsize()diff --git a/app/controllers/manager/redis_manager.py b/app/controllers/manager/redis_manager.pyindex cad1912a..861af0f4 100644--- a/app/controllers/manager/redis_manager.py+++ b/app/controllers/manager/redis_manager.py@@ -14,9 +14,14 @@ FUNC_MAP = {class RedisTaskManager(TaskManager):- def __init__(self, max_concurrent_tasks: int, redis_url: str):+ def __init__(+ self,+ max_concurrent_tasks: int,+ redis_url: str,+ max_queued_tasks: int = 100,+ ):self.redis_client = redis.Redis.from_url(redis_url)- super().__init__(max_concurrent_tasks)+ super().__init__(max_concurrent_tasks, max_queued_tasks=max_queued_tasks)def create_queue(self):return "task_queue"@@ -54,3 +59,6 @@ class RedisTaskManager(TaskManager):def is_queue_empty(self):return self.redis_client.llen(self.queue) == 0++ def queue_size(self):+ return self.redis_client.llen(self.queue)diff --git a/app/controllers/v1/video.py b/app/controllers/v1/video.pyindex 330b2ab2..5822fd88 100644--- a/app/controllers/v1/video.py+++ b/app/controllers/v1/video.py@@ -11,6 +11,7 @@ from loguru import loggerfrom app.config import configfrom app.controllers import base+from app.controllers.manager.base_manager import TaskQueueFullErrorfrom app.controllers.manager.memory_manager import InMemoryTaskManagerfrom app.controllers.manager.redis_manager import RedisTaskManagerfrom app.controllers.v1.base import new_router@@ -30,7 +31,7 @@ from app.models.schema import ()from app.services import state as smfrom app.services import task as tm-from app.utils import utils+from app.utils import file_security, utils# 认证依赖项# router = new_router(dependencies=[Depends(base.verify_token)])@@ -42,15 +43,21 @@ _redis_port = config.app.get("redis_port", 6379)_redis_db = config.app.get("redis_db", 0)_redis_password = config.app.get("redis_password", None)_max_concurrent_tasks = config.app.get("max_concurrent_tasks", 5)+_max_queued_tasks = config.app.get("max_queued_tasks", 100)redis_url = f"redis://:{_redis_password}@{_redis_host}:{_redis_port}/{_redis_db}"# 根据配置选择合适的任务管理器if _enable_redis:task_manager = RedisTaskManager(- max_concurrent_tasks=_max_concurrent_tasks, redis_url=redis_url+ max_concurrent_tasks=_max_concurrent_tasks,+ redis_url=redis_url,+ max_queued_tasks=_max_queued_tasks,)else:- task_manager = InMemoryTaskManager(max_concurrent_tasks=_max_concurrent_tasks)+ task_manager = InMemoryTaskManager(+ max_concurrent_tasks=_max_concurrent_tasks,+ max_queued_tasks=_max_queued_tasks,+ )def _sanitize_upload_filename(filename: str, request_id: str) -> str:@@ -67,35 +74,42 @@ def _sanitize_upload_filename(filename: str, request_id: str) -> str:def _resolve_path_within_directory(base_dir: str, unsafe_path: str, request_id: str) -> str:- # 对用户传入的相对路径做归一化,并强制要求结果仍然落在指定目录内,- # 这样可以阻止通过 ../ 或绝对路径逃逸到任务目录之外读取任意文件。- base_dir_real = os.path.realpath(base_dir)- resolved_path = os.path.realpath(os.path.join(base_dir_real, unsafe_path))-try:- common_path = os.path.commonpath([base_dir_real, resolved_path])- except ValueError:+ return file_security.resolve_path_within_directory(base_dir, unsafe_path)+ except ValueError as exc:+ logger.warning(+ f"reject unsafe file path, request_id: {request_id}, path: {unsafe_path}, "+ f"error: {str(exc)}"+ )raise HttpException(task_id=request_id,- status_code=403,+ status_code=404 if str(exc) == "file does not exist" else 403,message=f"{request_id}: invalid file path",)- if common_path != base_dir_real:- raise HttpException(- task_id=request_id,- status_code=403,- message=f"{request_id}: access to the requested file is forbidden",- )+def _task_file_to_uri(file: str, endpoint: str, task_dir: str, request_id: str) -> str:+ if not isinstance(file, str):+ return file- if not os.path.isfile(resolved_path):- raise HttpException(- task_id=request_id,- status_code=404,- message=f"{request_id}: file not found",+ if file.startswith(("http://", "https://")):+ return file++ try:+ resolved_path = file_security.resolve_path_within_directory(task_dir, file)+ except ValueError as exc:+ # 任务状态理论上只应保存任务目录内的产物路径。这里不再继续拼接 URL,+ # 避免把异常路径包装成可访问链接;同时保留原值,便于排查历史脏数据。+ logger.warning(+ f"skip unsafe task output path, request_id: {request_id}, path: {file}, "+ f"error: {str(exc)}")+ return file- return resolved_path+ relative_path = os.path.relpath(resolved_path, task_dir).replace("\\", "/")+ uri_path = f"tasks/{relative_path}"+ if endpoint:+ return f"{endpoint.rstrip('/')}/{uri_path}"+ return f"/{uri_path}"@router.post("/videos", response_model=TaskResponse, summary="Generate a short video")@@ -136,6 +150,14 @@ def create_task(task_manager.add_task(tm.start, task_id=task_id, params=body, stop_at=stop_at)logger.success(f"Task created: {utils.to_json(task)}")return utils.get_response(200, task)+ except TaskQueueFullError as e:+ sm.state.delete_task(task_id)+ logger.warning(+ f"reject task because queue is full, request_id: {request_id}, task_id: {task_id}"+ )+ raise HttpException(+ task_id=task_id, status_code=429, message=f"{request_id}: {str(e)}"+ )except ValueError as e:raise HttpException(task_id=task_id, status_code=400, message=f"{request_id}: {str(e)}"@@ -166,37 +188,24 @@ def get_task(task_id: str = Path(..., description="Task ID"),query: TaskQueryRequest = Depends(),):- endpoint = config.app.get("endpoint", "")- if not endpoint:- endpoint = str(request.base_url)- endpoint = endpoint.rstrip("/")-request_id = base.get_task_id(request)+ endpoint = config.app.get("endpoint", "").rstrip("/")task = sm.state.get_task(task_id)if task:task_dir = utils.task_dir()-- def file_to_uri(file):- if not file.startswith(endpoint):- _uri_path = file.replace(task_dir, "tasks").replace("\\", "/")- _uri_path = f"{endpoint}/{_uri_path}"- else:- _uri_path = file- return _uri_path+ response_task = dict(task)if "videos" in task:- videos = task["videos"]- urls = []- for v in videos:- urls.append(file_to_uri(v))- task["videos"] = urls+ response_task["videos"] = [+ _task_file_to_uri(v, endpoint, task_dir, request_id)+ for v in task["videos"]+ ]if "combined_videos" in task:- combined_videos = task["combined_videos"]- urls = []- for v in combined_videos:- urls.append(file_to_uri(v))- task["combined_videos"] = urls- return utils.get_response(200, task)+ response_task["combined_videos"] = [+ _task_file_to_uri(v, endpoint, task_dir, request_id)+ for v in task["combined_videos"]+ ]+ return utils.get_response(200, response_task)raise HttpException(task_id=task_id, status_code=404, message=f"{request_id}: task not found"@@ -235,11 +244,14 @@ def get_bgm_list(request: Request):files = glob.glob(os.path.join(song_dir, suffix))bgm_list = []for file in files:+ filename = os.path.basename(file)bgm_list.append({- "name": os.path.basename(file),+ "name": filename,"size": os.path.getsize(file),- "file": file,+ # 只返回文件名,避免把服务器绝对路径暴露给调用方。+ # 服务端后续会把该文件名解析回 songs 白名单目录。+ "file": filename,})response = {"files": bgm_list}@@ -263,7 +275,7 @@ def upload_bgm_file(request: Request, file: UploadFile = File(...)):# If the file already exists, it will be overwrittenfile.file.seek(0)buffer.write(file.file.read())- response = {"file": save_path}+ response = {"file": safe_filename}return utils.get_response(200, response)raise HttpException(@@ -284,11 +296,14 @@ def get_video_materials_list(request: Request):files.sort(key=lambda file_path: os.path.basename(file_path).lower())video_materials_list = []for file in files:+ filename = os.path.basename(file)video_materials_list.append({- "name": os.path.basename(file),+ "name": filename,"size": os.path.getsize(file),- "file": file,+ # 与 BGM 一样,只返回文件名;创建任务时再在 local_videos+ # 白名单目录内解析,避免 API 泄露宿主机绝对路径。+ "file": filename,})response = {"files": video_materials_list}@@ -315,7 +330,7 @@ def upload_video_material_file(request: Request, file: UploadFile = File(...)):# If the file already exists, it will be overwrittenfile.file.seek(0)buffer.write(file.file.read())- response = {"file": save_path}+ response = {"file": safe_filename}return utils.get_response(200, response)raise HttpException(diff --git a/app/services/video.py b/app/services/video.pyindex 0c1d0989..ae41ca07 100644--- a/app/services/video.py+++ b/app/services/video.py@@ -31,7 +31,7 @@ from app.models.schema import (VideoTransitionMode,)from app.services.utils import video_effects-from app.utils import utils+from app.utils import file_security, utilsclass SubClippedVideoClip:def __init__(self, file_path, start_time=None, end_time=None, width=None, height=None, duration=None):@@ -55,6 +55,7 @@ audio_codec = "aac"audio_bitrate = "192k"video_codec = "libx264"fps = 30+_BGM_EXTENSIONS = (".mp3",)def get_ffmpeg_binary():@@ -237,8 +238,26 @@ def get_bgm_file(bgm_type: str = "random", bgm_file: str = ""):if not bgm_type:return ""- if bgm_file and os.path.exists(bgm_file):- return bgm_file+ if bgm_file:+ song_dir = utils.song_dir()+ try:+ resolved_bgm_file = file_security.resolve_path_within_directory(+ song_dir, bgm_file+ )+ except ValueError as exc:+ # API 请求里的 bgm_file 来自用户输入,不能直接把任意绝对路径交给+ # MoviePy 打开。这里强制限制到 resource/songs 目录,阻止读取+ # /etc/passwd、配置文件、密钥等非背景音乐文件。+ logger.warning(+ f"reject unsafe bgm file: {bgm_file}, song_dir: {song_dir}, error: {str(exc)}"+ )+ return ""++ if not resolved_bgm_file.lower().endswith(_BGM_EXTENSIONS):+ logger.warning(f"reject unsupported bgm file extension: {resolved_bgm_file}")+ return ""++ return resolved_bgm_fileif bgm_type == "random":suffix = "*.mp3"@@ -639,23 +658,41 @@ def preprocess_video(materials: List[MaterialInfo], clip_duration=4):# 仅返回通过预处理校验的素材,避免低分辨率图片继续进入后续的视频合成流程。valid_materials = []+ local_videos_dir = utils.storage_dir("local_videos", create=True)for material in materials:if not material.url:continue- ext = utils.parse_extension(material.url)- material_source_path = material.url+ try:+ material_source_path = file_security.resolve_path_within_directory(+ local_videos_dir, material.url+ )+ except ValueError as exc:+ # local video_source 的素材路径来自 API 参数,必须限制在专用素材目录。+ # 允许用户传文件名,也兼容历史返回的绝对路径,但不允许逃逸到系统+ # 其他目录,避免任意文件读取或通过 MoviePy 探测本地敏感文件。+ logger.warning(+ f"skip unsafe local material: {material.url}, "+ f"local_videos_dir: {local_videos_dir}, error: {str(exc)}"+ )+ continue++ ext = utils.parse_extension(material_source_path)try:# 图片素材直接按图片方式读取,避免先走 VideoFileClip 误判后触发不稳定的回退分支。if ext in const.FILE_TYPE_IMAGES:- clip, material_source_path = _open_image_clip_with_fallback(material.url)+ clip, material_source_path = _open_image_clip_with_fallback(+ material_source_path+ )else:- clip = _open_video_clip_quietly(material.url)+ clip = _open_video_clip_quietly(material_source_path)except Exception:# 非标准扩展名或探测失败时再回退到图片模式,兼容历史上直接传本地图片路径的情况。try:- clip, material_source_path = _open_image_clip_with_fallback(material.url)+ clip, material_source_path = _open_image_clip_with_fallback(+ material_source_path+ )except Exception as exc:logger.warning(f"skip unreadable local material: {material.url}, error: {str(exc)}"diff --git a/app/utils/file_security.py b/app/utils/file_security.pynew file mode 100644index 00000000..00366fcd--- /dev/null+++ b/app/utils/file_security.py@@ -0,0 +1,35 @@+import os+++def resolve_path_within_directory(+ base_dir: str,+ unsafe_path: str,+ *,+ require_file: bool = True,+) -> str:+ # 用户传入的路径可能是文件名、相对路径、绝对路径,也可能夹带 `../`。+ # 这里统一解析成真实路径,并用 commonpath 判断它是否仍在允许目录内。+ # 这样比简单判断字符串前缀可靠,可以覆盖符号链接、重复分隔符、相对路径+ # 等场景,适用于上传目录、素材目录、任务产物目录这类白名单目录。+ if not unsafe_path:+ raise ValueError("empty path is not allowed")++ base_dir_real = os.path.realpath(base_dir)+ candidate_path = unsafe_path+ if not os.path.isabs(candidate_path):+ candidate_path = os.path.join(base_dir_real, candidate_path)++ resolved_path = os.path.realpath(candidate_path)+ try:+ common_path = os.path.commonpath([base_dir_real, resolved_path])+ except ValueError as exc:+ # Windows 下不同盘符会触发 ValueError,这类路径一定不属于允许目录。+ raise ValueError("path is outside the allowed directory") from exc++ if common_path != base_dir_real:+ raise ValueError("path is outside the allowed directory")++ if require_file and not os.path.isfile(resolved_path):+ raise ValueError("file does not exist")++ return resolved_pathdiff --git a/config.example.toml b/config.example.tomlindex 39f8ed20..60ca7873 100644--- a/config.example.toml+++ b/config.example.toml@@ -223,6 +223,9 @@ redis_password = ""# 文生视频时的最大并发任务数max_concurrent_tasks = 5+# 并发任务已满后的最大排队任务数,超过后接口返回 429,避免匿名请求无限堆积+max_queued_tasks = 100+[whisper]# Only effective when subtitle_provider is "whisper"diff --git a/test/services/test_video.py b/test/services/test_video.pyindex 60add022..5031c98e 100644--- a/test/services/test_video.py+++ b/test/services/test_video.py@@ -1,7 +1,9 @@import unittestimport os+import shutilimport sys+import tempfileimport typesfrom contextlib import redirect_stdoutfrom io import StringIO@@ -12,12 +14,71 @@ from moviepy import ()# add project root to python pathsys.path.insert(0, str(Path(__file__).parent.parent.parent))+from app.config import config+from app.controllers.manager.base_manager import TaskQueueFullError+from app.controllers.manager.memory_manager import InMemoryTaskManager+from app.controllers.v1 import video as video_controller+from app.models import constfrom app.models.schema import MaterialInfo+from app.services import state as smfrom app.services import video as vdfrom app.utils import utilsresources_dir = os.path.join(os.path.dirname(os.path.dirname(__file__)), "resources")++class _FakeRequest:+ def __init__(self):+ self.headers = {"x-task-id": "test-request"}+++class TestSecurityControls(unittest.TestCase):+ def setUp(self):+ self.original_app_config = dict(config.app)++ def tearDown(self):+ config.app.clear()+ config.app.update(self.original_app_config)++ def test_task_query_returns_relative_task_url_without_mutating_state(self):+ """+ endpoint 未显式配置时,任务查询接口不能使用 Host 派生绝对 URL,+ 也不能把展示 URL 回写到任务状态里,否则不同 Host 查询会污染结果。+ """+ task_id = "security-task-url"+ task_dir = utils.task_dir(task_id)+ video_path = os.path.join(task_dir, "final-1.mp4")+ Path(video_path).write_bytes(b"fake-video")+ config.app["endpoint"] = ""++ try:+ sm.state.update_task(+ task_id,+ state=const.TASK_STATE_COMPLETE,+ videos=[video_path],+ combined_videos=[video_path],+ )++ response = video_controller.get_task(_FakeRequest(), task_id=task_id)++ self.assertEqual(response["data"]["videos"], [f"/tasks/{task_id}/final-1.mp4"])+ self.assertEqual(sm.state.get_task(task_id)["videos"], [video_path])+ finally:+ sm.state.delete_task(task_id)+ shutil.rmtree(task_dir, ignore_errors=True)++ def test_in_memory_task_manager_rejects_when_queue_is_full(self):+ """+ 并发数用尽后,等待队列必须有硬上限。这里用 max_concurrent_tasks=0+ 强制任务进入队列,验证超过 max_queued_tasks 时会拒绝继续入队。+ """+ manager = InMemoryTaskManager(max_concurrent_tasks=0, max_queued_tasks=1)++ manager.add_task(lambda: None)++ with self.assertRaises(TaskQueueFullError):+ manager.add_task(lambda: None)+class TestVideoService(unittest.TestCase):def setUp(self):self.test_img_path = os.path.join(resources_dir, "1.png")@@ -28,28 +89,70 @@ class TestVideoService(unittest.TestCase):def test_preprocess_video(self):if not os.path.exists(self.test_img_path):self.fail(f"test image not found: {self.test_img_path}")-++ local_videos_dir = utils.storage_dir("local_videos", create=True)+ safe_img_path = os.path.join(local_videos_dir, "test-preprocess-1.png")+ shutil.copy2(self.test_img_path, safe_img_path)+# test preprocess_video functionm = MaterialInfo()- m.url = self.test_img_path+ m.url = os.path.basename(safe_img_path)m.provider = "local"print(m)-++ try:+ materials = vd.preprocess_video([m], clip_duration=4)+ print(materials)++ # verify result+ self.assertIsNotNone(materials)+ self.assertEqual(len(materials), 1)+ self.assertTrue(materials[0].url.endswith(".mp4"))++ # moviepy get video info+ clip = VideoFileClip(materials[0].url)+ print(clip)++ # clean generated test video file+ if os.path.exists(materials[0].url):+ os.remove(materials[0].url)+ finally:+ if os.path.exists(safe_img_path):+ os.remove(safe_img_path)++ def test_preprocess_video_rejects_material_outside_local_videos(self):+ """+ local 素材路径来自 API 参数,不能允许任意绝对路径进入 MoviePy。+ 这里验证非 local_videos 白名单目录内的路径会被跳过,避免任意文件读取。+ """+ m = MaterialInfo(provider="local", url=self.test_img_path)+materials = vd.preprocess_video([m], clip_duration=4)- print(materials)-- # verify result- self.assertIsNotNone(materials)- self.assertEqual(len(materials), 1)- self.assertTrue(materials[0].url.endswith(".mp4"))-- # moviepy get video info- clip = VideoFileClip(materials[0].url)- print(clip)-- # clean generated test video file- if os.path.exists(materials[0].url):- os.remove(materials[0].url)++ self.assertEqual(materials, [])++ def test_get_bgm_file_accepts_song_directory_filename(self):+ """+ BGM 列表接口现在只暴露文件名;生成视频时应能把文件名安全解析回+ resource/songs 白名单目录,保持正常使用路径可用。+ """+ song_dir = utils.song_dir()+ bgm_path = os.path.join(song_dir, "test-safe-bgm.mp3")+ Path(bgm_path).write_bytes(b"fake-mp3")++ try:+ self.assertEqual(vd.get_bgm_file(bgm_file="test-safe-bgm.mp3"), bgm_path)+ finally:+ if os.path.exists(bgm_path):+ os.remove(bgm_path)++ def test_get_bgm_file_rejects_path_outside_song_directory(self):+ """+ 用户传入的 bgm_file 不能直接作为本地路径打开,否则可能读取系统文件。+ 即使外部文件存在,也必须因为不在 songs 目录内被拒绝。+ """+ with tempfile.NamedTemporaryFile(suffix=".mp3") as temp_bgm:+ self.assertEqual(vd.get_bgm_file(bgm_file=temp_bgm.name), "")def test_get_ffmpeg_binary_uses_configured_env_path(self):"""配置中显式指定 ffmpeg 时,应优先使用该路径。"""