Skip to content

利用 AI 大模型和自动化工作流,根据主题或关键词一键生成高清短视频。Generate HD short videos from a topic or keyword with an automated AI workflow.

fix(material): improve Pixabay error diagnostics

harry07035f73c6d0aec9
diff --git a/app/services/material.py b/app/services/material.py
index 3d432e74..fec110a7 100644
--- a/app/services/material.py
+++ b/app/services/material.py
@@ -2,7 +2,7 @@ import os
import random
import threading
from typing import List
-from urllib.parse import urlencode
+from urllib.parse import quote_plus, urlencode
import requests
from loguru import logger
@@ -52,6 +52,45 @@ def get_api_key(cfg_key: str):
return api_keys[_api_key_counter % len(api_keys)]
+def _redact_secret(message: str, secret: str) -> str:
+ """
+ 对即将写入日志的异常文本做最小范围脱敏。
+
+ requests 的连接异常可能包含完整请求 URL,而 Pixabay API Key 通过查询
+ 参数传递。这里同时替换原始值和 URL 编码值,既保留网络错误信息用于排查,
+ 又避免密钥进入日志文件。
+ """
+ safe_message = str(message)
+ if not secret:
+ return safe_message
+
+ safe_message = safe_message.replace(secret, "***")
+ encoded_secret = quote_plus(secret)
+ if encoded_secret != secret:
+ safe_message = safe_message.replace(encoded_secret, "***")
+ return safe_message
+
+
+def _is_cloudflare_challenge(response: requests.Response) -> bool:
+ """
+ 识别 Cloudflare 返回的 HTML Challenge,而不是把它当成 Pixabay JSON。
+
+ Cloudflare 通常会设置 `cf-mitigated: challenge`;部分部署只返回带有
+ "Just a moment" 或 challenge-platform 的 HTML,因此保留内容特征兜底。
+ 响应正文仅在内存中判断,不写入日志,避免记录无价值的大段 HTML。
+ """
+ headers = getattr(response, "headers", {}) or {}
+ if str(headers.get("cf-mitigated", "")).lower() == "challenge":
+ return True
+
+ content_type = str(headers.get("content-type", "")).lower()
+ if "text/html" not in content_type:
+ return False
+
+ body = str(getattr(response, "text", "")).lower()
+ return "just a moment" in body or "/cdn-cgi/challenge-platform/" in body
+
+
def search_videos_pexels(
search_term: str,
minimum_duration: int,
@@ -129,14 +168,50 @@ def search_videos_pixabay(
query_url = f"https://pixabay.com/api/videos/?{urlencode(params)}"
logger.info(
f"searching videos on pixabay: term={search_term!r}, "
- f"with proxies: {config.proxy}"
+ f"proxy_enabled={bool(config.proxy)}"
)
try:
r = requests.get(
query_url, proxies=config.proxy, verify=_get_tls_verify(), timeout=(30, 60)
)
- response = r.json()
+ status_code = int(getattr(r, "status_code", 200))
+ headers = getattr(r, "headers", {}) or {}
+ content_type = str(headers.get("content-type", ""))
+ retry_after = headers.get("retry-after")
+ cf_ray = headers.get("cf-ray")
+
+ if _is_cloudflare_challenge(r):
+ logger.error(
+ "pixabay search was blocked by a Cloudflare challenge: "
+ f"status={status_code}, cf_ray={cf_ray or 'unknown'}. "
+ "Check the server network or proxy, or use Pexels/Coverr instead."
+ )
+ return []
+
+ if status_code == 429:
+ logger.error(
+ "pixabay API rate limit exceeded: "
+ f"status=429, retry_after={retry_after or 'unknown'}"
+ )
+ return []
+
+ if status_code >= 400:
+ logger.error(
+ "pixabay search request failed: "
+ f"status={status_code}, content_type={content_type or 'unknown'}"
+ )
+ return []
+
+ try:
+ response = r.json()
+ except ValueError:
+ logger.error(
+ "pixabay returned an unexpected non-JSON response: "
+ f"status={status_code}, content_type={content_type or 'unknown'}"
+ )
+ return []
+
video_items = []
if "hits" not in response:
logger.error(f"search videos failed: {response}")
@@ -163,7 +238,16 @@ def search_videos_pixabay(
break
return video_items
except Exception as e:
- logger.error(f"search videos failed: {str(e)}")
+ error_message = _redact_secret(str(e), api_key)
+ # ProxyError 等异常可能回显完整代理 URL,其中可能包含用户名和密码。
+ # 仅替换用户实际配置的代理值,不对普通错误文本做宽泛正则处理,
+ # 避免误删 DNS、超时、证书等真正有助于排查的信息。
+ for proxy_url in config.proxy.values():
+ error_message = _redact_secret(error_message, str(proxy_url))
+ logger.error(
+ "pixabay search request failed: "
+ f"error={type(e).__name__}, detail={error_message}"
+ )
return []
diff --git a/test/services/test_material.py b/test/services/test_material.py
index f4123e55..d4a3388a 100644
--- a/test/services/test_material.py
+++ b/test/services/test_material.py
@@ -67,6 +67,9 @@ class TestMaterialTlsVerification(unittest.TestCase):
config.proxy.clear()
fake_response = SimpleNamespace(
+ status_code=200,
+ headers={"content-type": "application/json"},
+ text="",
json=lambda: {
"hits": [
{
@@ -92,7 +95,12 @@ class TestMaterialTlsVerification(unittest.TestCase):
config.app["pixabay_api_keys"] = ["pixabay-secret-key"]
config.proxy.clear()
- fake_response = SimpleNamespace(json=lambda: {"hits": []})
+ fake_response = SimpleNamespace(
+ status_code=200,
+ headers={"content-type": "application/json"},
+ text="",
+ json=lambda: {"hits": []},
+ )
with patch(
"app.services.material.requests.get", return_value=fake_response
@@ -102,6 +110,139 @@ class TestMaterialTlsVerification(unittest.TestCase):
logged_messages = " ".join(str(call.args[0]) for call in log.call_args_list)
self.assertNotIn("pixabay-secret-key", logged_messages)
+ def test_search_pixabay_reports_cloudflare_challenge(self):
+ """
+ Cloudflare Challenge 返回的是 HTML,不是 Pixabay API 的 JSON。
+ 应直接说明服务端拦截原因,避免用户只看到没有上下文的 JSON 解析错误。
+ """
+ config.app["pixabay_api_keys"] = ["pixabay-secret-key"]
+ config.proxy.clear()
+
+ fake_response = SimpleNamespace(
+ status_code=429,
+ headers={
+ "content-type": "text/html; charset=UTF-8",
+ "cf-mitigated": "challenge",
+ "cf-ray": "test-ray",
+ },
+ text="<html><title>Just a moment...</title></html>",
+ )
+
+ with patch(
+ "app.services.material.requests.get", return_value=fake_response
+ ), patch("app.services.material.logger.error") as log:
+ results = material.search_videos_pixabay("nature", minimum_duration=1)
+
+ logged_messages = " ".join(str(call.args[0]) for call in log.call_args_list)
+ self.assertEqual(results, [])
+ self.assertIn("Cloudflare challenge", logged_messages)
+ self.assertIn("cf_ray=test-ray", logged_messages)
+ self.assertNotIn("pixabay-secret-key", logged_messages)
+ self.assertNotIn("Just a moment", logged_messages)
+
+ def test_search_pixabay_reports_api_rate_limit(self):
+ """
+ Pixabay 自身的 429 限流与 Cloudflare HTML Challenge 是不同问题。
+ 保留 Retry-After 可以帮助用户判断何时重试,同时不记录响应正文。
+ """
+ config.app["pixabay_api_keys"] = ["pixabay-key"]
+ config.proxy.clear()
+
+ fake_response = SimpleNamespace(
+ status_code=429,
+ headers={
+ "content-type": "text/plain; charset=UTF-8",
+ "retry-after": "60",
+ },
+ text="API rate limit exceeded",
+ )
+
+ with patch(
+ "app.services.material.requests.get", return_value=fake_response
+ ), patch("app.services.material.logger.error") as log:
+ results = material.search_videos_pixabay("nature", minimum_duration=1)
+
+ logged_messages = " ".join(str(call.args[0]) for call in log.call_args_list)
+ self.assertEqual(results, [])
+ self.assertIn("API rate limit exceeded", logged_messages)
+ self.assertIn("retry_after=60", logged_messages)
+
+ def test_search_pixabay_reports_non_json_response(self):
+ """
+ 即使状态码为 200,上游代理也可能返回登录页或其他非 JSON 内容。
+ 该场景应记录响应类型,而不是向外暴露底层 JSONDecodeError。
+ """
+ config.app["pixabay_api_keys"] = ["pixabay-key"]
+ config.proxy.clear()
+
+ def raise_invalid_json():
+ raise ValueError("Expecting value: line 1 column 1")
+
+ fake_response = SimpleNamespace(
+ status_code=200,
+ headers={"content-type": "text/plain"},
+ text="unexpected response",
+ json=raise_invalid_json,
+ )
+
+ with patch(
+ "app.services.material.requests.get", return_value=fake_response
+ ), patch("app.services.material.logger.error") as log:
+ results = material.search_videos_pixabay("nature", minimum_duration=1)
+
+ logged_messages = " ".join(str(call.args[0]) for call in log.call_args_list)
+ self.assertEqual(results, [])
+ self.assertIn("unexpected non-JSON response", logged_messages)
+ self.assertNotIn("Expecting value", logged_messages)
+
+ def test_search_pixabay_redacts_api_key_from_network_error(self):
+ """
+ requests 的连接异常可能回显完整请求 URL。异常详情仍应保留用于排查,
+ 但 URL 查询参数中的 Pixabay API Key 必须在写入日志前脱敏。
+ """
+ api_key = "pixabay-secret-key"
+ config.app["pixabay_api_keys"] = [api_key]
+ config.proxy.clear()
+ error = requests.ConnectionError(
+ "request failed for "
+ f"https://pixabay.com/api/videos/?q=nature&key={api_key}"
+ )
+
+ with patch(
+ "app.services.material.requests.get", side_effect=error
+ ), patch("app.services.material.logger.error") as log:
+ results = material.search_videos_pixabay("nature", minimum_duration=1)
+
+ logged_messages = " ".join(str(call.args[0]) for call in log.call_args_list)
+ self.assertEqual(results, [])
+ self.assertIn("ConnectionError", logged_messages)
+ self.assertIn("key=***", logged_messages)
+ self.assertNotIn(api_key, logged_messages)
+
+ def test_search_pixabay_redacts_proxy_credentials_from_network_error(self):
+ """
+ 代理连接异常可能回显含认证信息的完整代理 URL。日志应保留异常类型,
+ 但不能把代理用户名和密码持久化到日志文件。
+ """
+ proxy_url = "http://proxy-user:[email protected]:8080"
+ config.app["pixabay_api_keys"] = ["pixabay-key"]
+ config.proxy.clear()
+ config.proxy["http"] = proxy_url
+ error = requests.exceptions.ProxyError(
+ f"failed to connect to proxy {proxy_url}"
+ )
+
+ with patch(
+ "app.services.material.requests.get", side_effect=error
+ ), patch("app.services.material.logger.error") as log:
+ results = material.search_videos_pixabay("nature", minimum_duration=1)
+
+ logged_messages = " ".join(str(call.args[0]) for call in log.call_args_list)
+ self.assertEqual(results, [])
+ self.assertIn("ProxyError", logged_messages)
+ self.assertNotIn("proxy-user", logged_messages)
+ self.assertNotIn("proxy-password", logged_messages)
+
def test_save_video_uses_tls_verification_by_default(self):
config.app.pop("tls_verify", None)
config.proxy.clear()