Skip to content

利用 AI 大模型和自动化工作流,根据主题或关键词一键生成高清短视频。Generate HD short videos from a topic or keyword with an automated AI workflow.

fix(llm): remove emoji marks from social hashtags

harry0703f063a2b95e0f
diff --git a/app/services/llm.py b/app/services/llm.py
index bd935d34..c661243c 100644
--- a/app/services/llm.py
+++ b/app/services/llm.py
@@ -1241,16 +1241,24 @@ def _normalize_hashtags(raw, count: int) -> List[str]:
seen = set()
result: List[str] = []
for item in candidates:
- # \w does not match combining marks, so keep them explicitly: they are
- # the vowel signs and tone marks of Devanagari, Bengali, Tamil and Thai
- # tags, and dropping them misspells the tag ("#भारत" became "#भरत").
- tag = "".join(
- character
- for character in item
- if character.isalnum()
- or character == "_"
- or unicodedata.category(character).startswith("M")
- )
+ # 印地语、泰语等依赖组合符,不能像 \w 那样直接删除。但组合符必须
+ # 附着于保留的字符;emoji 被删除后,其变体选择符也应丢弃,否则
+ # #☕️ 会留下只有不可见字符的标签,#coffee☕️ 也无法与 #coffee 去重。
+ # 数字本身会保留,因此还需清理 emoji 展示选择符和键帽组合符,
+ # 让 #1️⃣ 与 #1 一致;其它语言组合符和 CJK 字形变体仍按附着规则保留。
+ characters = []
+ can_attach_mark = False
+ for character in item:
+ if character in {"\ufe0e", "\ufe0f", "\u20e3"}:
+ can_attach_mark = False
+ elif character.isalnum() or character == "_":
+ characters.append(character)
+ can_attach_mark = True
+ elif can_attach_mark and unicodedata.category(character).startswith("M"):
+ characters.append(character)
+ else:
+ can_attach_mark = False
+ tag = "".join(characters)
if not tag:
continue
key = tag.lower()
diff --git a/test/services/test_social_hashtag_marks.py b/test/services/test_social_hashtag_marks.py
new file mode 100644
index 00000000..d9644385
--- /dev/null
+++ b/test/services/test_social_hashtag_marks.py
@@ -0,0 +1,128 @@
+"""组合符必须附着于保留的字符,不能由已删除的 emoji 留下空标签。"""
+
+import json
+import unicodedata
+from unittest.mock import patch
+
+from fastapi.testclient import TestClient
+import pytest
+
+from app.asgi import app
+from app.services import llm
+
+
+ "tag",
+ [
+ "#भारत",
+ "#বাংলা",
+ "#தமிழ்",
+ "#ท่องเที่ยว",
+ "#سَلَام",
+ "#שָׁלוֹם",
+ unicodedata.normalize("NFD", "#việtnam"),
+ "#邊\ufe00",
+ "#邊\U000e0100",
+ "#ᠠ\u180b",
+ ],
+)
+def test_hashtag_keeps_marks_attached_to_retained_letters(tag):
+ """保留真实语言的元音、声调及 CJK 字形变体,不全局删除 Mark 字符。"""
+ assert llm._normalize_hashtags([tag], count=5) == [tag]
+
+
+ "tag", ["#☕️", "#*️⃣", "#👩‍💻️", "#\u0301", "#\u093e", "#\ufe0f", "#\U000e0100"]
+)
+def test_hashtag_drops_marks_without_a_retained_base(tag):
+ assert llm._normalize_hashtags([tag], count=5) == []
+
+
+ "decorated", ["#coffee☕️", "#☕️coffee", "#co☕️ffee", "#coffee!\u0301"]
+)
+def test_removed_characters_do_not_leave_marks_that_break_deduplication(decorated):
+ assert llm._normalize_hashtags([decorated, "#coffee"], count=3) == ["#coffee"]
+
+
+def test_emoji_only_hashtags_do_not_consume_the_result_limit():
+ assert llm._normalize_hashtags(
+ ["#☕️", "#Travel", "#Coffee", "#Shorts"], count=3
+ ) == ["#Travel", "#Coffee", "#Shorts"]
+
+
+ "decorated,plain",
+ [
+ ("#1️⃣", "#1"),
+ ("#top1️⃣", "#top1"),
+ ("#1\u20e3", "#1"),
+ ("#1\ufe0e", "#1"),
+ ("#ℹ️", "#ℹ"),
+ ],
+)
+def test_emoji_marks_after_retained_digits_or_letters_do_not_break_deduplication(
+ decorated, plain
+):
+ assert llm._normalize_hashtags([decorated, plain], count=3) == [plain]
+
+
+def test_string_hashtags_keep_language_marks_after_discarding_emoji():
+ assert llm._normalize_hashtags("#☕️ #भारत, #বাংলা #தமிழ் #ท่องเที่ยว", count=5) == [
+ "#भारत",
+ "#বাংলা",
+ "#தமிழ்",
+ "#ท่องเที่ยว",
+ ]
+
+
[email protected]("platform", list(llm.SOCIAL_PLATFORMS))
+def test_social_metadata_api_uses_cleaned_tags_without_displacing_valid_ones(platform):
+ """固定模型输出,真实运行 API 的解析、清洗、去重和数量限制。"""
+ response_text = json.dumps(
+ {
+ "title": "旅行",
+ "caption": "清晨旅行。",
+ "hashtags": [
+ "#☕️",
+ "#coffee☕️",
+ "#coffee",
+ "#भारत",
+ "#বাংলা",
+ "#தமிழ்",
+ "#ท่องเที่ยว",
+ ],
+ },
+ ensure_ascii=False,
+ )
+ with patch.object(llm, "_generate_response", return_value=response_text):
+ response = TestClient(app).post(
+ "/api/v1/social-metadata",
+ json={"video_subject": "旅行", "platform": platform},
+ )
+
+ assert response.status_code == 200
+ expected = ["#coffee", "#भारत", "#বাংলা", "#தமிழ்", "#ท่องเที่ยว"]
+ assert (
+ response.json()["data"]["hashtags"]
+ == expected[: llm.SOCIAL_PLATFORMS[platform]["hashtag_count"]]
+ )
+
+
+def test_keycap_tags_do_not_displace_other_youtube_tags():
+ response_text = json.dumps(
+ {
+ "title": "旅行",
+ "caption": "收藏旅行技巧。",
+ "hashtags": ["#1️⃣", "#1", "#Travel", "#Coffee"],
+ },
+ ensure_ascii=False,
+ )
+ with patch.object(llm, "_generate_response", return_value=response_text):
+ response = TestClient(app).post(
+ "/api/v1/social-metadata",
+ json={"video_subject": "旅行", "platform": "youtube_shorts"},
+ )
+
+ assert response.status_code == 200
+ assert response.json()["data"]["hashtags"] == ["#1", "#Travel", "#Coffee"]