⭐AI-driven public opinion & trend monitor with multi-platform aggregation, RSS, and smart alerts.🎯 告别信息过载,你的 AI 舆情监控助手与热点筛选工具!聚合多平台热点 + RSS 订阅,支持关键词精准筛选。AI 智能筛选新闻 + AI 翻译 + AI 分析简报直推手机,也支持接入 MCP 架构,赋能 AI 自然语言对话分析、情感洞察与趋势预测等。支持 Docker ,数据本地/云端自持。集成微信/飞书/钉钉/Telegram/邮件/ntfy/bark/slack 等渠道智能推送。
Python aibarkdata-analysisdockerhot-newsllmmailmcpmcp-servernewsntfypythonrsstrending-topicswechatwework
fix(notification): 修复消息分批截断丢弃内容的问题
sansan3194a7a7e849
diff --git a/trendradar/notification/batch.py b/trendradar/notification/batch.pyindex 889b394b52..d00672fdc9 100644--- a/trendradar/notification/batch.py+++ b/trendradar/notification/batch.py@@ -61,24 +61,125 @@ def truncate_to_bytes(text: str, max_bytes: int) -> str:if len(text_bytes) <= max_bytes:return text- # 截断到指定字节数truncated = text_bytes[:max_bytes]-- # 处理可能的不完整 UTF-8 字符for i in range(min(4, len(truncated))):try:return truncated[: len(truncated) - i].decode("utf-8")except UnicodeDecodeError:continue-- # 极端情况:返回空字符串return ""+def truncate_at_line_boundary(text: str, max_bytes: int) -> str:+ """在行边界处截断,确保不在标题或内容中间断开++ 先按字节截断,再回退到最近的换行符位置,保证每一行都完整。++ Args:+ text: 要截断的文本+ max_bytes: 最大字节数++ Returns:+ 在最后一个完整行处结束的截断文本+ """+ if len(text.encode("utf-8")) <= max_bytes:+ return text++ rough_cut = truncate_to_bytes(text, max_bytes)+ last_newline = rough_cut.rfind("\n")+ if last_newline > 0:+ return rough_cut[:last_newline]+ return rough_cut+++def truncate_preserving_footer(content: str, max_bytes: int) -> str:+ """截断内容,优先保留尾部 footer(更新时间等),正文在行边界处截断++ 识别内容末尾的 footer 区域(更新时间、版本提示等),+ 对 footer 之前的正文部分在行边界处截断,再拼接完整 footer。++ Args:+ content: 完整内容(正文 + footer)+ max_bytes: 最大字节数++ Returns:+ 截断后的内容,footer 完整保留,正文在行边界处截断+ """+ if len(content.encode("utf-8")) <= max_bytes:+ return content++ # 各平台 footer 的常见开头模式+ footer_markers = ["\n\n\n> ", "\n\n> ", "\n\n<font", "\n\n_", "\n\n更新时间"]+ footer_start = -1+ for marker in footer_markers:+ pos = content.rfind(marker)+ if pos > 0:+ footer_start = pos+ break++ if footer_start <= 0:+ return truncate_at_line_boundary(content, max_bytes)++ footer = content[footer_start:]+ body = content[:footer_start]+ footer_size = len(footer.encode("utf-8"))++ if footer_size >= max_bytes:+ return truncate_at_line_boundary(content, max_bytes)++ truncated_body = truncate_at_line_boundary(body, max_bytes - footer_size)+ return truncated_body + footer+++def _split_oversized_batch(content: str, max_content_bytes: int) -> List[str]:+ """将超限批次按行边界拆分成多个子批次(保留 footer)++ Args:+ content: 超限的批次内容(含 footer)+ max_content_bytes: 每个子批次的最大字节数++ Returns:+ 拆分后的子批次列表+ """+ # 识别 footer+ footer_markers = ["\n\n\n> ", "\n\n> ", "\n\n<font", "\n\n_", "\n\n更新时间"]+ footer = ""+ body = content+ for marker in footer_markers:+ pos = content.rfind(marker)+ if pos > 0:+ footer = content[pos:]+ body = content[:pos]+ break++ footer_size = len(footer.encode("utf-8"))+ available = max_content_bytes - footer_size+ if available <= 0:+ return [truncate_at_line_boundary(content, max_content_bytes)]++ # 按行拆分 body+ lines = body.split("\n")+ sub_batches = []+ current = ""++ for line in lines:+ candidate = current + line + "\n"+ if len(candidate.encode("utf-8")) > available and current.strip():+ sub_batches.append(current + footer)+ current = line + "\n"+ else:+ current = candidate++ if current.strip():+ sub_batches.append(current + footer)++ return sub_batches if sub_batches else [content]++def add_batch_headers(batches: List[str], format_type: str, max_bytes: int) -> List[str]:- """为批次添加头部,动态计算确保总大小不超过限制+ """为批次添加头部,超限时拆分成多个子批次(不丢弃内容)Args:batches: 原始批次列表@@ -91,24 +192,29 @@ def add_batch_headers(if len(batches) <= 1:return batches- total = len(batches)- result = []+ # 第一遍:拆分超限批次+ expanded = []+ max_header_size = get_max_batch_header_size(format_type)+ for content in batches:+ if len(content.encode("utf-8")) + max_header_size > max_bytes:+ expanded.extend(_split_oversized_batch(content, max_bytes - max_header_size))+ else:+ expanded.append(content)++ # 第二遍:添加头部+ if len(expanded) <= 1:+ return expanded- for i, content in enumerate(batches, 1):- # 生成批次头部+ total = len(expanded)+ result = []+ for i, content in enumerate(expanded, 1):header = get_batch_header(format_type, i, total)header_size = len(header.encode("utf-8"))-- # 动态计算允许的最大内容大小max_content_size = max_bytes - header_size- content_size = len(content.encode("utf-8"))-- # 如果超出,截断到安全大小- if content_size > max_content_size:- print(- f"警告:{format_type} 第 {i}/{total} 批次内容({content_size}字节) + 头部({header_size}字节) 超出限制({max_bytes}字节),截断到 {max_content_size} 字节"- )- content = truncate_to_bytes(content, max_content_size)++ if len(content.encode("utf-8")) > max_content_size:+ # 仍超限(极端情况:单行过长),行边界截断+ content = truncate_preserving_footer(content, max_content_size)result.append(header + content)diff --git a/trendradar/notification/splitter.py b/trendradar/notification/splitter.pyindex db0ce65921..d4b4202782 100644--- a/trendradar/notification/splitter.py+++ b/trendradar/notification/splitter.py@@ -11,6 +11,113 @@ from typing import Dict, List, Optional, Callablefrom trendradar.report.formatter import format_title_for_platformfrom trendradar.report.helpers import format_rank_displayfrom trendradar.utils.time import DEFAULT_TIMEZONE, format_iso_time_friendly, convert_time_for_display+from trendradar.notification.batch import truncate_at_line_boundary+++# === 分批安全辅助函数 ===++def _split_content_by_lines(+ content: str, footer: str, max_bytes: int, base_header: str+) -> List[str]:+ """将超长内容按行边界拆分成多个完整批次(每个批次带 footer)++ 不会丢弃任何内容,溢出部分自动分配到后续批次。++ Args:+ content: 正文内容(不含 footer,可能含 base_header)+ footer: 尾部内容(更新时间等)+ max_bytes: 单批次最大字节数+ base_header: 后续批次的头部++ Returns:+ 完整批次列表(每个元素 = 正文 + footer,大小 ≤ max_bytes)+ """+ footer_size = len(footer.encode("utf-8"))+ result_batches = []+ lines = content.split("\n")++ current = ""+ for line in lines:+ candidate = current + line + "\n"+ if len(candidate.encode("utf-8")) + footer_size > max_bytes and current.strip():+ result_batches.append(current + footer)+ current = base_header + line + "\n"+ else:+ current = candidate++ if current.strip():+ result_batches.append(current + footer)++ return result_batches+++def _safe_append_batch(+ batches: List[str], content: str, footer: str, max_bytes: int,+ base_header: str = ""+) -> None:+ """安全追加批次,超限时按行拆分成多个批次(不丢弃内容)++ Args:+ batches: 批次列表(原地修改)+ content: 正文内容(不含 footer)+ footer: 尾部内容(更新时间等)+ max_bytes: 最大字节数+ base_header: 溢出时后续批次的头部+ """+ full = content + footer+ if len(full.encode("utf-8")) <= max_bytes:+ batches.append(full)+ return++ split_batches = _split_content_by_lines(content, footer, max_bytes, base_header)+ if split_batches:+ batches.extend(split_batches)+ else:+ # 极端情况:单行就超限,强制截断+ batches.append(truncate_at_line_boundary(full, max_bytes))+++def _safe_new_batch(+ new_content: str, footer: str, max_bytes: int, base_header: str,+ batches: List[str] = None+) -> str:+ """安全创建新批次,超限时将溢出内容拆分到 batches 中,返回最后一段作为 current_batch++ Args:+ new_content: 新批次完整内容(含 base_header + section_header + ...)+ footer: 尾部内容+ max_bytes: 最大字节数+ base_header: 基础头部+ batches: 批次列表,溢出部分追加到此(可选)++ Returns:+ 可安全继续追加内容的 current_batch(大小 + footer ≤ max_bytes)+ """+ if len((new_content + footer).encode("utf-8")) <= max_bytes:+ return new_content++ if batches is None:+ # 无法拆分到 batches,退回行边界截断+ footer_size = len(footer.encode("utf-8"))+ available = max_bytes - footer_size+ header_size = len(base_header.encode("utf-8"))+ if available <= header_size:+ return base_header+ return truncate_at_line_boundary(new_content, available)++ # 拆分:前面的部分存入 batches,最后一段作为 current_batch 返回+ split_batches = _split_content_by_lines(new_content, footer, max_bytes, base_header)+ if len(split_batches) <= 1:+ # 无法再拆,直接返回(由后续 _safe_append_batch 兜底)+ return new_content++ # 前 N-1 个批次存入 batches+ batches.extend(split_batches[:-1])+ # 最后一个批次去掉 footer 作为 current_batch(后续还会追加内容)+ last = split_batches[-1]+ if last.endswith(footer):+ return last[: -len(footer)]+ return last# 默认批次大小配置@@ -271,9 +378,10 @@ def split_content_into_batches(current_batch_has_content = Trueelse:if current_batch_has_content:- batches.append(current_batch + base_footer)- # 新批次开头不需要分割线,使用原始 stats_header- current_batch = base_header + stats_header+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)+ current_batch = _safe_new_batch(+ base_header + stats_header, base_footer, max_bytes, base_header, batches+ )current_batch_has_content = True# 逐个处理词组(确保词组标题+第一条新闻的原子性)@@ -389,10 +497,12 @@ def split_content_into_batches(len(test_content.encode("utf-8")) + len(base_footer.encode("utf-8"))>= max_bytes):- # 当前批次容纳不下,开启新批次if current_batch_has_content:- batches.append(current_batch + base_footer)- current_batch = base_header + stats_header + word_with_first_news+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)+ current_batch = _safe_new_batch(+ base_header + stats_header + word_with_first_news,+ base_footer, max_bytes, base_header, batches+ )current_batch_has_content = Truestart_index = 1else:@@ -440,8 +550,11 @@ def split_content_into_batches(>= max_bytes):if current_batch_has_content:- batches.append(current_batch + base_footer)- current_batch = base_header + stats_header + word_header + news_line+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)+ current_batch = _safe_new_batch(+ base_header + stats_header + word_header + news_line,+ base_footer, max_bytes, base_header, batches+ )current_batch_has_content = Trueelse:current_batch = test_content@@ -517,8 +630,10 @@ def split_content_into_batches(>= max_bytes):if current_batch_has_content:- batches.append(current_batch + base_footer)- current_batch = base_header + new_header+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)+ current_batch = _safe_new_batch(+ base_header + new_header, base_footer, max_bytes, base_header, batches+ )current_batch_has_content = Trueelse:current_batch = test_content@@ -581,8 +696,11 @@ def split_content_into_batches(>= max_bytes):if current_batch_has_content:- batches.append(current_batch + base_footer)- current_batch = base_header + new_header + source_with_first_news+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)+ current_batch = _safe_new_batch(+ base_header + new_header + source_with_first_news,+ base_footer, max_bytes, base_header, batches+ )current_batch_has_content = Truestart_index = 1else:@@ -627,8 +745,11 @@ def split_content_into_batches(>= max_bytes):if current_batch_has_content:- batches.append(current_batch + base_footer)- current_batch = base_header + new_header + source_header + news_line+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)+ current_batch = _safe_new_batch(+ base_header + new_header + source_header + news_line,+ base_footer, max_bytes, base_header, batches+ )current_batch_has_content = Trueelse:current_batch = test_content@@ -668,13 +789,27 @@ def split_content_into_batches(current_batch = test_contentcurrent_batch_has_content = Trueelse:- # 当前批次容纳不下,开启新批次if current_batch_has_content:- batches.append(current_batch + base_footer)- # AI 内容可能很长,需要考虑是否需要进一步分割- ai_with_header = base_header + ai_content- current_batch = ai_with_header- current_batch_has_content = True+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)++ # AI 内容可能很长,按行拆分成多个批次+ footer_size = len(base_footer.encode("utf-8"))+ header_size = len(base_header.encode("utf-8"))+ available = max_bytes - footer_size - header_size++ ai_lines = ai_content.split("\n")+ current_batch = base_header+ current_batch_has_content = False++ for line in ai_lines:+ test_line = line + "\n" if not line.endswith("\n") else line+ test_content = current_batch + test_line+ if len(test_content.encode("utf-8")) + footer_size >= max_bytes and current_batch_has_content:+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)+ current_batch = base_header + test_line+ else:+ current_batch = test_content+ current_batch_has_content = Truereturn current_batch, current_batch_has_content, batches@@ -789,8 +924,10 @@ def split_content_into_batches(>= max_bytes):if current_batch_has_content:- batches.append(current_batch + base_footer)- current_batch = base_header + failed_header+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)+ current_batch = _safe_new_batch(+ base_header + failed_header, base_footer, max_bytes, base_header, batches+ )current_batch_has_content = Trueelse:current_batch = test_content@@ -810,8 +947,11 @@ def split_content_into_batches(>= max_bytes):if current_batch_has_content:- batches.append(current_batch + base_footer)- current_batch = base_header + failed_header + failed_line+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)+ current_batch = _safe_new_batch(+ base_header + failed_header + failed_line,+ base_footer, max_bytes, base_header, batches+ )current_batch_has_content = Trueelse:current_batch = test_content@@ -819,7 +959,7 @@ def split_content_into_batches(# 完成最后批次if current_batch_has_content:- batches.append(current_batch + base_footer)+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)return batches@@ -899,8 +1039,10 @@ def _process_rss_stats_section(current_batch_has_content = Trueelse:if current_batch_has_content:- batches.append(current_batch + base_footer)- current_batch = base_header + rss_header+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)+ current_batch = _safe_new_batch(+ base_header + rss_header, base_footer, max_bytes, base_header, batches+ )current_batch_has_content = True# 逐个处理关键词组(与热榜一致)@@ -983,8 +1125,11 @@ def _process_rss_stats_section(if len(test_content.encode("utf-8")) + len(base_footer.encode("utf-8")) >= max_bytes:if current_batch_has_content:- batches.append(current_batch + base_footer)- current_batch = base_header + rss_header + word_with_first_news+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)+ current_batch = _safe_new_batch(+ base_header + rss_header + word_with_first_news,+ base_footer, max_bytes, base_header, batches+ )current_batch_has_content = Truestart_index = 1else:@@ -1017,8 +1162,11 @@ def _process_rss_stats_section(test_content = current_batch + news_lineif len(test_content.encode("utf-8")) + len(base_footer.encode("utf-8")) >= max_bytes:if current_batch_has_content:- batches.append(current_batch + base_footer)- current_batch = base_header + rss_header + word_header + news_line+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)+ current_batch = _safe_new_batch(+ base_header + rss_header + word_header + news_line,+ base_footer, max_bytes, base_header, batches+ )current_batch_has_content = Trueelse:current_batch = test_content@@ -1132,8 +1280,10 @@ def _process_rss_new_titles_section(test_content = current_batch + new_headerif len(test_content.encode("utf-8")) + len(base_footer.encode("utf-8")) >= max_bytes:if current_batch_has_content:- batches.append(current_batch + base_footer)- current_batch = base_header + new_header+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)+ current_batch = _safe_new_batch(+ base_header + new_header, base_footer, max_bytes, base_header, batches+ )current_batch_has_content = Trueelse:current_batch = test_content@@ -1187,8 +1337,11 @@ def _process_rss_new_titles_section(if len(test_content.encode("utf-8")) + len(base_footer.encode("utf-8")) >= max_bytes:if current_batch_has_content:- batches.append(current_batch + base_footer)- current_batch = base_header + new_header + source_with_first_news+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)+ current_batch = _safe_new_batch(+ base_header + new_header + source_with_first_news,+ base_footer, max_bytes, base_header, batches+ )current_batch_has_content = Truestart_index = 1else:@@ -1220,8 +1373,11 @@ def _process_rss_new_titles_section(test_content = current_batch + news_lineif len(test_content.encode("utf-8")) + len(base_footer.encode("utf-8")) >= max_bytes:if current_batch_has_content:- batches.append(current_batch + base_footer)- current_batch = base_header + new_header + source_header + news_line+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)+ current_batch = _safe_new_batch(+ base_header + new_header + source_header + news_line,+ base_footer, max_bytes, base_header, batches+ )current_batch_has_content = Trueelse:current_batch = test_content@@ -1377,8 +1533,10 @@ def _process_standalone_section(current_batch_has_content = Trueelse:if current_batch_has_content:- batches.append(current_batch + base_footer)- current_batch = base_header + section_header+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)+ current_batch = _safe_new_batch(+ base_header + section_header, base_footer, max_bytes, base_header, batches+ )current_batch_has_content = True# 处理热榜平台@@ -1414,8 +1572,11 @@ def _process_standalone_section(if len(test_content.encode("utf-8")) + len(base_footer.encode("utf-8")) >= max_bytes:if current_batch_has_content:- batches.append(current_batch + base_footer)- current_batch = base_header + section_header + platform_with_first+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)+ current_batch = _safe_new_batch(+ base_header + section_header + platform_with_first,+ base_footer, max_bytes, base_header, batches+ )current_batch_has_content = Truestart_index = 1else:@@ -1430,8 +1591,11 @@ def _process_standalone_section(test_content = current_batch + item_lineif len(test_content.encode("utf-8")) + len(base_footer.encode("utf-8")) >= max_bytes:if current_batch_has_content:- batches.append(current_batch + base_footer)- current_batch = base_header + section_header + platform_header + item_line+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)+ current_batch = _safe_new_batch(+ base_header + section_header + platform_header + item_line,+ base_footer, max_bytes, base_header, batches+ )current_batch_has_content = Trueelse:current_batch = test_content@@ -1472,8 +1636,11 @@ def _process_standalone_section(if len(test_content.encode("utf-8")) + len(base_footer.encode("utf-8")) >= max_bytes:if current_batch_has_content:- batches.append(current_batch + base_footer)- current_batch = base_header + section_header + feed_with_first+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)+ current_batch = _safe_new_batch(+ base_header + section_header + feed_with_first,+ base_footer, max_bytes, base_header, batches+ )current_batch_has_content = Truestart_index = 1else:@@ -1488,8 +1655,11 @@ def _process_standalone_section(test_content = current_batch + item_lineif len(test_content.encode("utf-8")) + len(base_footer.encode("utf-8")) >= max_bytes:if current_batch_has_content:- batches.append(current_batch + base_footer)- current_batch = base_header + section_header + feed_header + item_line+ _safe_append_batch(batches, current_batch, base_footer, max_bytes, base_header)+ current_batch = _safe_new_batch(+ base_header + section_header + feed_header + item_line,+ base_footer, max_bytes, base_header, batches+ )current_batch_has_content = Trueelse:current_batch = test_content