Skip to content

⭐AI-driven public opinion & trend monitor with multi-platform aggregation, RSS, and smart alerts.🎯 告别信息过载,你的 AI 舆情监控助手与热点筛选工具!聚合多平台热点 + RSS 订阅,支持关键词精准筛选。AI 智能筛选新闻 + AI 翻译 + AI 分析简报直推手机,也支持接入 MCP 架构,赋能 AI 自然语言对话分析、情感洞察与趋势预测等。支持 Docker ,数据本地/云端自持。集成微信/飞书/钉钉/Telegram/邮件/ntfy/bark/slack 等渠道智能推送。

fix(notification): 修复消息分批截断丢弃内容的问题

sansan3194a7a7e849
diff --git a/trendradar/notification/batch.py b/trendradar/notification/batch.py
index 889b394b52..d00672fdc9 100644
--- a/trendradar/notification/batch.py
+++ b/trendradar/notification/batch.py
@@ -61,24 +61,125 @@ def truncate_to_bytes(text: str, max_bytes: int) -> str:
if len(text_bytes) <= max_bytes:
return text
- # 截断到指定字节数
truncated = text_bytes[:max_bytes]
-
- # 处理可能的不完整 UTF-8 字符
for i in range(min(4, len(truncated))):
try:
return truncated[: len(truncated) - i].decode("utf-8")
except UnicodeDecodeError:
continue
-
- # 极端情况:返回空字符串
return ""
+def truncate_at_line_boundary(text: str, max_bytes: int) -> str:
+ """在行边界处截断,确保不在标题或内容中间断开
+
+ 先按字节截断,再回退到最近的换行符位置,保证每一行都完整。
+
+ Args:
+ text: 要截断的文本
+ max_bytes: 最大字节数
+
+ Returns:
+ 在最后一个完整行处结束的截断文本
+ """
+ if len(text.encode("utf-8")) <= max_bytes:
+ return text
+
+ rough_cut = truncate_to_bytes(text, max_bytes)
+ last_newline = rough_cut.rfind("\n")
+ if last_newline > 0:
+ return rough_cut[:last_newline]
+ return rough_cut
+
+
+def truncate_preserving_footer(content: str, max_bytes: int) -> str:
+ """截断内容,优先保留尾部 footer(更新时间等),正文在行边界处截断
+
+ 识别内容末尾的 footer 区域(更新时间、版本提示等),
+ 对 footer 之前的正文部分在行边界处截断,再拼接完整 footer。
+
+ Args:
+ content: 完整内容(正文 + footer)
+ max_bytes: 最大字节数
+
+ Returns:
+ 截断后的内容,footer 完整保留,正文在行边界处截断
+ """
+ if len(content.encode("utf-8")) <= max_bytes:
+ return content
+
+ # 各平台 footer 的常见开头模式
+ footer_markers = ["\n\n\n> ", "\n\n> ", "\n\n<font", "\n\n_", "\n\n更新时间"]
+ footer_start = -1
+ for marker in footer_markers:
+ pos = content.rfind(marker)
+ if pos > 0:
+ footer_start = pos
+ break
+
+ if footer_start <= 0:
+ return truncate_at_line_boundary(content, max_bytes)
+
+ footer = content[footer_start:]
+ body = content[:footer_start]
+ footer_size = len(footer.encode("utf-8"))
+
+ if footer_size >= max_bytes:
+ return truncate_at_line_boundary(content, max_bytes)
+
+ truncated_body = truncate_at_line_boundary(body, max_bytes - footer_size)
+ return truncated_body + footer
+
+
+def _split_oversized_batch(content: str, max_content_bytes: int) -> List[str]:
+ """将超限批次按行边界拆分成多个子批次(保留 footer)
+
+ Args:
+ content: 超限的批次内容(含 footer)
+ max_content_bytes: 每个子批次的最大字节数
+
+ Returns:
+ 拆分后的子批次列表
+ """
+ # 识别 footer
+ footer_markers = ["\n\n\n> ", "\n\n> ", "\n\n<font", "\n\n_", "\n\n更新时间"]
+ footer = ""
+ body = content
+ for marker in footer_markers:
+ pos = content.rfind(marker)
+ if pos > 0:
+ footer = content[pos:]
+ body = content[:pos]
+ break
+
+ footer_size = len(footer.encode("utf-8"))
+ available = max_content_bytes - footer_size
+ if available <= 0:
+ return [truncate_at_line_boundary(content, max_content_bytes)]
+
+ # 按行拆分 body
+ lines = body.split("\n")
+ sub_batches = []
+ current = ""
+
+ for line in lines:
+ candidate = current + line + "\n"
+ if len(candidate.encode("utf-8")) > available and current.strip():
+ sub_batches.append(current + footer)
+ current = line + "\n"
+ else:
+ current = candidate
+
+ if current.strip():
+ sub_batches.append(current + footer)
+
+ return sub_batches if sub_batches else [content]
+
+
def add_batch_headers(
batches: List[str], format_type: str, max_bytes: int
) -> List[str]:
- """为批次添加头部,动态计算确保总大小不超过限制
+ """为批次添加头部,超限时拆分成多个子批次(不丢弃内容)
Args:
batches: 原始批次列表
@@ -91,24 +192,29 @@ def add_batch_headers(
if len(batches) <= 1:
return batches
- total = len(batches)
- result = []
+ # 第一遍:拆分超限批次
+ expanded = []
+ max_header_size = get_max_batch_header_size(format_type)
+ for content in batches:
+ if len(content.encode("utf-8")) + max_header_size > max_bytes:
+ expanded.extend(_split_oversized_batch(content, max_bytes - max_header_size))
+ else:
+ expanded.append(content)
+
+ # 第二遍:添加头部
+ if len(expanded) <= 1:
+ return expanded
- for i, content in enumerate(batches, 1):
- # 生成批次头部
+ total = len(expanded)
+ result = []
+ for i, content in enumerate(expanded, 1):
header = get_batch_header(format_type, i, total)
header_size = len(header.encode("utf-8"))
-
- # 动态计算允许的最大内容大小
max_content_size = max_bytes - header_size
- content_size = len(content.encode("utf-8"))
-
- # 如果超出,截断到安全大小
- if content_size > max_content_size:
- print(
- f"警告:{format_type} 第 {i}/{total} 批次内容({content_size}字节) + 头部({header_size}字节) 超出限制({max_bytes}字节),截断到 {max_content_size} 字节"
- )
- content = truncate_to_bytes(content, max_content_size)
+
+ if len(content.encode("utf-8")) > max_content_size:
+ # 仍超限(极端情况:单行过长),行边界截断
+ content = truncate_preserving_footer(content, max_content_size)
result.append(header + content)
diff --git a/trendradar/notification/splitter.py b/trendradar/notification/splitter.py
index db0ce65921..d4b4202782 100644
--- a/trendradar/notification/splitter.py
+++ b/trendradar/notification/splitter.py
@@ -11,6 +11,113 @@ from typing import Dict, List, Optional, Callable
from trendradar.report.formatter import format_title_for_platform
from trendradar.report.helpers import format_rank_display
from trendradar.utils.time import DEFAULT_TIMEZONE, format_iso_time_friendly, convert_time_for_display
+from trendradar.notification.batch import truncate_at_line_boundary
+
+
+# === 分批安全辅助函数 ===
+
+def _split_content_by_lines(
+ content: str, footer: str, max_bytes: int, base_header: str
+) -> List[str]:
+ """将超长内容按行边界拆分成多个完整批次(每个批次带 footer)
+
+ 不会丢弃任何内容,溢出部分自动分配到后续批次。
+
+ Args:
+ content: 正文内容(不含 footer,可能含 base_header)
+ footer: 尾部内容(更新时间等)
+ max_bytes: 单批次最大字节数
+ base_header: 后续批次的头部
+
+ Returns:
+ 完整批次列表(每个元素 = 正文 + footer,大小 ≤ max_bytes)
+ """
+ footer_size = len(footer.encode("utf-8"))
+ result_batches = []
+ lines = content.split("\n")
+
+ current = ""
+ for line in lines:
+ candidate = current + line + "\n"
+ if len(candidate.encode("utf-8")) + footer_size > max_bytes and current.strip():
+ result_batches.append(current + footer)
+ current = base_header + line + "\n"
+ else:
+ current = candidate
+
+ if current.strip():
+ result_batches.append(current + footer)
+
+ return result_batches
+
+
+def _safe_append_batch(
+ batches: List[str], content: str, footer: str, max_bytes: int,
+ base_header: str = ""
+) -> None:
+ """安全追加批次,超限时按行拆分成多个批次(不丢弃内容)
+
+ Args:
+ batches: 批次列表(原地修改)
+ content: 正文内容(不含 footer)
+ footer: 尾部内容(更新时间等)
+ max_bytes: 最大字节数
+ base_header: 溢出时后续批次的头部
+ """
+ full = content + footer
+ if len(full.encode("utf-8")) <= max_bytes:
+ batches.append(full)
+ return
+
+ split_batches = _split_content_by_lines(content, footer, max_bytes, base_header)
+ if split_batches:
+ batches.extend(split_batches)
+ else:
+ # 极端情况:单行就超限,强制截断
+ batches.append(truncate_at_line_boundary(full, max_bytes))
+
+
+def _safe_new_batch(
+ new_content: str, footer: str, max_bytes: int, base_header: str,
+ batches: List[str] = None
+) -> str:
+ """安全创建新批次,超限时将溢出内容拆分到 batches 中,返回最后一段作为 current_batch
+
+ Args:
+ new_content: 新批次完整内容(含 base_header + section_header + ...)
+ footer: 尾部内容
+ max_bytes: 最大字节数
+ base_header: 基础头部
+ batches: 批次列表,溢出部分追加到此(可选)
+
+ Returns:
+ 可安全继续追加内容的 current_batch(大小 + footer ≤ max_bytes)
+ """
+ if len((new_content + footer).encode("utf-8")) <= max_bytes:
+ return new_content
+
+ if batches is None:
+ # 无法拆分到 batches,退回行边界截断
+ footer_size = len(footer.encode("utf-8"))
+ available = max_bytes - footer_size
+ header_size = len(base_header.encode("utf-8"))
+ if available <= header_size:
+ return base_header
+ return truncate_at_line_boundary(new_content, available)
+
+ # 拆分:前面的部分存入 batches,最后一段作为 current_batch 返回
+ split_batches = _split_content_by_lines(new_content, footer, max_bytes, base_header)
+ if len(split_batches) <= 1:
+ # 无法再拆,直接返回(由后续 _safe_append_batch 兜底)
+ return new_content
+
+ # 前 N-1 个批次存入 batches
+ batches.extend(split_batches[:-1])
+ # 最后一个批次去掉 footer 作为 current_batch(后续还会追加内容)
+ last = split_batches[-1]
+ if last.endswith(footer):
+ return last[: -len(footer)]
+ return last
# 默认批次大小配置
@@ -271,9 +378,10 @@ def split_content_into_batches(
current_batch_has_content = True
else:
if current_batch_has_content:
- batches.append(current_batch + base_footer)
- # 新批次开头不需要分割线,使用原始 stats_header
- current_batch = base_header + stats_header
+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)
+ current_batch = _safe_new_batch(
+ base_header + stats_header, base_footer, max_bytes, base_header, batches
+ )
current_batch_has_content = True
# 逐个处理词组(确保词组标题+第一条新闻的原子性)
@@ -389,10 +497,12 @@ def split_content_into_batches(
len(test_content.encode("utf-8")) + len(base_footer.encode("utf-8"))
>= max_bytes
):
- # 当前批次容纳不下,开启新批次
if current_batch_has_content:
- batches.append(current_batch + base_footer)
- current_batch = base_header + stats_header + word_with_first_news
+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)
+ current_batch = _safe_new_batch(
+ base_header + stats_header + word_with_first_news,
+ base_footer, max_bytes, base_header, batches
+ )
current_batch_has_content = True
start_index = 1
else:
@@ -440,8 +550,11 @@ def split_content_into_batches(
>= max_bytes
):
if current_batch_has_content:
- batches.append(current_batch + base_footer)
- current_batch = base_header + stats_header + word_header + news_line
+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)
+ current_batch = _safe_new_batch(
+ base_header + stats_header + word_header + news_line,
+ base_footer, max_bytes, base_header, batches
+ )
current_batch_has_content = True
else:
current_batch = test_content
@@ -517,8 +630,10 @@ def split_content_into_batches(
>= max_bytes
):
if current_batch_has_content:
- batches.append(current_batch + base_footer)
- current_batch = base_header + new_header
+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)
+ current_batch = _safe_new_batch(
+ base_header + new_header, base_footer, max_bytes, base_header, batches
+ )
current_batch_has_content = True
else:
current_batch = test_content
@@ -581,8 +696,11 @@ def split_content_into_batches(
>= max_bytes
):
if current_batch_has_content:
- batches.append(current_batch + base_footer)
- current_batch = base_header + new_header + source_with_first_news
+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)
+ current_batch = _safe_new_batch(
+ base_header + new_header + source_with_first_news,
+ base_footer, max_bytes, base_header, batches
+ )
current_batch_has_content = True
start_index = 1
else:
@@ -627,8 +745,11 @@ def split_content_into_batches(
>= max_bytes
):
if current_batch_has_content:
- batches.append(current_batch + base_footer)
- current_batch = base_header + new_header + source_header + news_line
+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)
+ current_batch = _safe_new_batch(
+ base_header + new_header + source_header + news_line,
+ base_footer, max_bytes, base_header, batches
+ )
current_batch_has_content = True
else:
current_batch = test_content
@@ -668,13 +789,27 @@ def split_content_into_batches(
current_batch = test_content
current_batch_has_content = True
else:
- # 当前批次容纳不下,开启新批次
if current_batch_has_content:
- batches.append(current_batch + base_footer)
- # AI 内容可能很长,需要考虑是否需要进一步分割
- ai_with_header = base_header + ai_content
- current_batch = ai_with_header
- current_batch_has_content = True
+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)
+
+ # AI 内容可能很长,按行拆分成多个批次
+ footer_size = len(base_footer.encode("utf-8"))
+ header_size = len(base_header.encode("utf-8"))
+ available = max_bytes - footer_size - header_size
+
+ ai_lines = ai_content.split("\n")
+ current_batch = base_header
+ current_batch_has_content = False
+
+ for line in ai_lines:
+ test_line = line + "\n" if not line.endswith("\n") else line
+ test_content = current_batch + test_line
+ if len(test_content.encode("utf-8")) + footer_size >= max_bytes and current_batch_has_content:
+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)
+ current_batch = base_header + test_line
+ else:
+ current_batch = test_content
+ current_batch_has_content = True
return current_batch, current_batch_has_content, batches
@@ -789,8 +924,10 @@ def split_content_into_batches(
>= max_bytes
):
if current_batch_has_content:
- batches.append(current_batch + base_footer)
- current_batch = base_header + failed_header
+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)
+ current_batch = _safe_new_batch(
+ base_header + failed_header, base_footer, max_bytes, base_header, batches
+ )
current_batch_has_content = True
else:
current_batch = test_content
@@ -810,8 +947,11 @@ def split_content_into_batches(
>= max_bytes
):
if current_batch_has_content:
- batches.append(current_batch + base_footer)
- current_batch = base_header + failed_header + failed_line
+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)
+ current_batch = _safe_new_batch(
+ base_header + failed_header + failed_line,
+ base_footer, max_bytes, base_header, batches
+ )
current_batch_has_content = True
else:
current_batch = test_content
@@ -819,7 +959,7 @@ def split_content_into_batches(
# 完成最后批次
if current_batch_has_content:
- batches.append(current_batch + base_footer)
+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)
return batches
@@ -899,8 +1039,10 @@ def _process_rss_stats_section(
current_batch_has_content = True
else:
if current_batch_has_content:
- batches.append(current_batch + base_footer)
- current_batch = base_header + rss_header
+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)
+ current_batch = _safe_new_batch(
+ base_header + rss_header, base_footer, max_bytes, base_header, batches
+ )
current_batch_has_content = True
# 逐个处理关键词组(与热榜一致)
@@ -983,8 +1125,11 @@ def _process_rss_stats_section(
if len(test_content.encode("utf-8")) + len(base_footer.encode("utf-8")) >= max_bytes:
if current_batch_has_content:
- batches.append(current_batch + base_footer)
- current_batch = base_header + rss_header + word_with_first_news
+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)
+ current_batch = _safe_new_batch(
+ base_header + rss_header + word_with_first_news,
+ base_footer, max_bytes, base_header, batches
+ )
current_batch_has_content = True
start_index = 1
else:
@@ -1017,8 +1162,11 @@ def _process_rss_stats_section(
test_content = current_batch + news_line
if len(test_content.encode("utf-8")) + len(base_footer.encode("utf-8")) >= max_bytes:
if current_batch_has_content:
- batches.append(current_batch + base_footer)
- current_batch = base_header + rss_header + word_header + news_line
+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)
+ current_batch = _safe_new_batch(
+ base_header + rss_header + word_header + news_line,
+ base_footer, max_bytes, base_header, batches
+ )
current_batch_has_content = True
else:
current_batch = test_content
@@ -1132,8 +1280,10 @@ def _process_rss_new_titles_section(
test_content = current_batch + new_header
if len(test_content.encode("utf-8")) + len(base_footer.encode("utf-8")) >= max_bytes:
if current_batch_has_content:
- batches.append(current_batch + base_footer)
- current_batch = base_header + new_header
+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)
+ current_batch = _safe_new_batch(
+ base_header + new_header, base_footer, max_bytes, base_header, batches
+ )
current_batch_has_content = True
else:
current_batch = test_content
@@ -1187,8 +1337,11 @@ def _process_rss_new_titles_section(
if len(test_content.encode("utf-8")) + len(base_footer.encode("utf-8")) >= max_bytes:
if current_batch_has_content:
- batches.append(current_batch + base_footer)
- current_batch = base_header + new_header + source_with_first_news
+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)
+ current_batch = _safe_new_batch(
+ base_header + new_header + source_with_first_news,
+ base_footer, max_bytes, base_header, batches
+ )
current_batch_has_content = True
start_index = 1
else:
@@ -1220,8 +1373,11 @@ def _process_rss_new_titles_section(
test_content = current_batch + news_line
if len(test_content.encode("utf-8")) + len(base_footer.encode("utf-8")) >= max_bytes:
if current_batch_has_content:
- batches.append(current_batch + base_footer)
- current_batch = base_header + new_header + source_header + news_line
+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)
+ current_batch = _safe_new_batch(
+ base_header + new_header + source_header + news_line,
+ base_footer, max_bytes, base_header, batches
+ )
current_batch_has_content = True
else:
current_batch = test_content
@@ -1377,8 +1533,10 @@ def _process_standalone_section(
current_batch_has_content = True
else:
if current_batch_has_content:
- batches.append(current_batch + base_footer)
- current_batch = base_header + section_header
+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)
+ current_batch = _safe_new_batch(
+ base_header + section_header, base_footer, max_bytes, base_header, batches
+ )
current_batch_has_content = True
# 处理热榜平台
@@ -1414,8 +1572,11 @@ def _process_standalone_section(
if len(test_content.encode("utf-8")) + len(base_footer.encode("utf-8")) >= max_bytes:
if current_batch_has_content:
- batches.append(current_batch + base_footer)
- current_batch = base_header + section_header + platform_with_first
+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)
+ current_batch = _safe_new_batch(
+ base_header + section_header + platform_with_first,
+ base_footer, max_bytes, base_header, batches
+ )
current_batch_has_content = True
start_index = 1
else:
@@ -1430,8 +1591,11 @@ def _process_standalone_section(
test_content = current_batch + item_line
if len(test_content.encode("utf-8")) + len(base_footer.encode("utf-8")) >= max_bytes:
if current_batch_has_content:
- batches.append(current_batch + base_footer)
- current_batch = base_header + section_header + platform_header + item_line
+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)
+ current_batch = _safe_new_batch(
+ base_header + section_header + platform_header + item_line,
+ base_footer, max_bytes, base_header, batches
+ )
current_batch_has_content = True
else:
current_batch = test_content
@@ -1472,8 +1636,11 @@ def _process_standalone_section(
if len(test_content.encode("utf-8")) + len(base_footer.encode("utf-8")) >= max_bytes:
if current_batch_has_content:
- batches.append(current_batch + base_footer)
- current_batch = base_header + section_header + feed_with_first
+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)
+ current_batch = _safe_new_batch(
+ base_header + section_header + feed_with_first,
+ base_footer, max_bytes, base_header, batches
+ )
current_batch_has_content = True
start_index = 1
else:
@@ -1488,8 +1655,11 @@ def _process_standalone_section(
test_content = current_batch + item_line
if len(test_content.encode("utf-8")) + len(base_footer.encode("utf-8")) >= max_bytes:
if current_batch_has_content:
- batches.append(current_batch + base_footer)
- current_batch = base_header + section_header + feed_header + item_line
+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)
+ current_batch = _safe_new_batch(
+ base_header + section_header + feed_header + item_line,
+ base_footer, max_bytes, base_header, batches
+ )
current_batch_has_content = True
else:
current_batch = test_content