AI turns documents or topics into real, native PowerPoint decks—with native shapes, transitions and animations, data-backed charts and tables on demand, audio narration from speaker notes, and support for your own .pptx templates. · by Hugo He
feat(tools): 优化 pdf_to_md 工具以适配大模型处理
hugohe3ada8311610eb
diff --git a/tools/pdf_to_md.py b/tools/pdf_to_md.pyindex 3fbf14ed..01bc99bc 100644--- a/tools/pdf_to_md.py+++ b/tools/pdf_to_md.py@@ -48,15 +48,75 @@ def analyze_font_sizes(doc) -> dict:return size_map-def get_heading_level(size: float, size_map: dict) -> int:- """根据字体大小返回标题级别,0 表示正文"""+def get_heading_level(size: float, size_map: dict, text: str = "",+ flags: int = 0, strict: bool = True) -> int:+ """+ 综合判断标题级别++ Args:+ size: 字体大小+ size_map: 字体大小映射+ text: 文本内容(用于辅助判断)+ flags: 字体标志(bit 4 = 粗体)+ strict: 严格模式,需要满足更多条件++ Returns:+ 标题级别(0 表示正文,1-3 表示 H1-H3)+ """+ # 基于字体大小的初步判断+ level = 0if "h1" in size_map and size >= size_map["h1"] - 0.5:- return 1- if "h2" in size_map and size >= size_map["h2"] - 0.5:- return 2- if "h3" in size_map and size >= size_map["h3"] - 0.5:- return 3- return 0+ level = 1+ elif "h2" in size_map and size >= size_map["h2"] - 0.5:+ level = 2+ elif "h3" in size_map and size >= size_map["h3"] - 0.5:+ level = 3++ if level == 0:+ return 0++ # 非严格模式直接返回(向后兼容)+ if not strict or not text:+ return level++ # 严格模式:额外验证条件+ text = text.strip()++ # 排除条件:太长的文本不太可能是标题+ if len(text) > 80:+ return 0++ # 排除条件:以句号等结尾的完整句子+ sentence_endings = '.。!!??'+ if text and text[-1] in sentence_endings:+ # 但保留编号标题如 "1. 概述" 或 "第一章."+ if not re.match(r'^[\d第]+[.、章节]', text):+ return 0++ # 加分条件:粗体文本更可能是标题+ is_bold = flags & 16+ if not is_bold and level >= 2:+ # 非粗体的小标题需要更严格的字体大小差异+ body_size = size_map.get("body", 12)+ if size < body_size + 2:+ return 0++ return level++def is_monospace_font(font_name: str) -> bool:+ """+ 判断是否为等宽字体(通常用于代码)+ """+ if not font_name:+ return False+ font_lower = font_name.lower()+ mono_fonts = [+ 'courier', 'consolas', 'monaco', 'menlo', 'monospace',+ 'source code', 'fira code', 'jetbrains', 'inconsolata',+ 'dejavu sans mono', 'liberation mono', 'ubuntu mono',+ 'roboto mono', 'robotomono', 'sf mono', 'cascadia', 'hack'+ ]+ return any(f in font_lower for f in mono_fonts)def format_span_text(text: str, flags: int) -> str:@@ -100,6 +160,137 @@ def detect_list_item(text: str) -> tuple:return (False, None, text)+def remove_page_footer(text: str) -> str:+ """+ 移除页脚中的页码模式,如 'November 2025 8' 或 '2025年11月 8'+ """+ # 英文月份 + 年份 + 页码+ months_en = r'(?:January|February|March|April|May|June|July|August|September|October|November|December)'+ pattern_en = rf'\s*{months_en}\s+\d{{4}}\s+\d{{1,3}}\s*$'+ text = re.sub(pattern_en, '', text, flags=re.IGNORECASE)++ # 中文格式:2025年11月 8+ pattern_cn = r'\s*\d{4}年\d{1,2}月\s+\d{1,3}\s*$'+ text = re.sub(pattern_cn, '', text)++ return text.rstrip()+++def detect_headers_footers(doc, threshold_ratio: float = 0.6) -> set:+ """+ 统计检测页眉和页脚++ 原理:页眉和页脚通常在每一页的固定位置(顶部或底部)出现相同的内容。+ 我们收集所有页面的顶部和底部文本,如果某个文本出现的频率超过阈值,则视为噪声。+ """+ if len(doc) < 3:+ return set()++ headers = []+ footers = []++ # 采样前 20 页和后 20 页(避免处理太慢)+ pages_to_scan = list(range(len(doc)))+ if len(doc) > 40:+ pages_to_scan = pages_to_scan[:20] + pages_to_scan[-20:]++ for i in pages_to_scan:+ page = doc[i]+ rect = page.rect+ h = rect.height++ # 定义顶部和底部区域 (各 15%)+ top_rect = fitz.Rect(0, 0, rect.width, h * 0.15)+ bottom_rect = fitz.Rect(0, h * 0.85, rect.width, h)++ # 提取文本块+ blocks = page.get_text("blocks")+ for b in blocks:+ b_rect = fitz.Rect(b[:4])+ text = b[4].strip()+ if not text:+ continue++ # 使用简单的空间判定+ if b_rect.intersects(top_rect):+ headers.append(text)+ elif b_rect.intersects(bottom_rect):+ footers.append(text)++ # 统计频率+ noise_texts = set()+ total_scanned = len(pages_to_scan)++ for collection in [headers, footers]:+ counter = Counter(collection)+ for text, count in counter.items():+ # if text appears in > 60% of scanned pages, mark as noise+ if count / total_scanned > threshold_ratio:+ noise_texts.add(text)++ return noise_texts+++def merge_adjacent_headings(elements: list) -> list:+ """+ 合并相邻的同级短标题+ 例如: '# Agent Tools &' + '# Interoperability' → '# Agent Tools & Interoperability'+ """+ if not elements:+ return elements++ merged = []+ i = 0++ while i < len(elements):+ el = elements[i]++ # 只处理标题元素+ if el.get("type") != 0 or not el.get("is_heading"):+ merged.append(el)+ i += 1+ continue++ content = el["content"]+ # 提取标题级别+ match = re.match(r'^(#{1,6})\s+(.+)$', content)+ if not match:+ merged.append(el)+ i += 1+ continue++ level = match.group(1)+ title_text = match.group(2)++ # 如果标题较短且下一个也是同级标题,尝试合并+ j = i + 1+ while j < len(elements) and len(title_text) < 60:+ next_el = elements[j]+ if next_el.get("type") != 0 or not next_el.get("is_heading"):+ break++ next_match = re.match(r'^(#{1,6})\s+(.+)$', next_el["content"])+ if not next_match or next_match.group(1) != level:+ break++ next_text = next_match.group(2)+ # 只合并短标题片段+ if len(next_text) > 40:+ break++ # 合并+ title_text += " " + next_text+ j += 1++ # 创建合并后的元素+ merged_el = el.copy()+ merged_el["content"] = f"{level} {title_text}"+ merged.append(merged_el)+ i = j++ return merged++def clean_text(text: str) -> str:"""清理提取的文本"""lines = text.split('\n')@@ -167,6 +358,13 @@ def extract_pdf_to_markdown(pdf_path: str, output_path: str = None) -> str:print(f" 字体大小映射: 正文={size_map.get('body', 'N/A')}, " +f"H1={size_map.get('h1', 'N/A')}, H2={size_map.get('h2', 'N/A')}, H3={size_map.get('h3', 'N/A')}")+ print(f"🧹 检测重复页眉/页脚...")+ noise_texts = detect_headers_footers(doc)+ if noise_texts:+ print(f" 发现 {len(noise_texts)} 个重复噪声文本 (将被移除):")+ for t in list(noise_texts)[:3]:+ print(f" - {t[:30]}...")+markdown_content = f"# {title}\n\n"img_dir = None@@ -180,6 +378,10 @@ def extract_pdf_to_markdown(pdf_path: str, output_path: str = None) -> str:img_count = 0for page_num, page in enumerate(doc, 1):+ if page_num > 1:+ # 添加分页符,方便 LLM 理解上下文切分+ markdown_content += f"\n\n<!-- Page {page_num} -->\n\n"+try:tabs = page.find_tables()except Exception:@@ -202,6 +404,7 @@ def extract_pdf_to_markdown(pdf_path: str, output_path: str = None) -> str:for block in blocks:block_rect = fitz.Rect(block["bbox"])+ # 检查是否为表格内容is_in_table = Falsefor tab_rect in tab_rects:intersect = block_rect & tab_rect@@ -213,11 +416,17 @@ def extract_pdf_to_markdown(pdf_path: str, output_path: str = None) -> str:continueif block["type"] == 0:+ # 检查是否为需要过滤的噪声文本 (Whole block match)+ block_text_full = "".join([span["text"] for line in block["lines"] for span in line["spans"]]).strip()+ if block_text_full in noise_texts:+ continue+for line in block["lines"]:line_text = ""line_size = 0line_flags = 0span_count = 0+ is_code_line = Falseformatted_spans = []for span in line["spans"]:@@ -234,8 +443,14 @@ def extract_pdf_to_markdown(pdf_path: str, output_path: str = None) -> str:line_flags |= span_flagsspan_count += 1- heading_level = get_heading_level(span_size, size_map)- if heading_level > 0:+ heading_level = get_heading_level(span_size, size_map, span_text, span_flags)++ # 检测代码字体+ font_name = span.get("font", "")+ if is_monospace_font(font_name):+ is_code_line = True+ formatted_spans.append(span_text) # 代码不加格式+ elif heading_level > 0:formatted_spans.append(span_text.strip())else:formatted_spans.append(format_span_text(span_text, span_flags))@@ -244,9 +459,13 @@ def extract_pdf_to_markdown(pdf_path: str, output_path: str = None) -> str:if not line_text:continue+ # 二次检查:行级别是否匹配噪声 (有时 block 被切分了)+ if line_text in noise_texts:+ continue+line_text = merge_adjacent_formatting(line_text)- heading_level = get_heading_level(line_size, size_map)+ heading_level = get_heading_level(line_size, size_map, line_text, line_flags)is_list, list_type, list_content = detect_list_item(line_text)@@ -264,7 +483,8 @@ def extract_pdf_to_markdown(pdf_path: str, output_path: str = None) -> str:"type": 0,"content": final_text,"is_heading": heading_level > 0,- "is_list": is_list+ "is_list": is_list,+ "is_code": is_code_line})elif block["type"] == 1:@@ -276,6 +496,9 @@ def extract_pdf_to_markdown(pdf_path: str, output_path: str = None) -> str:page_elements.sort(key=lambda x: x["y0"])+ # 合并相邻的同级短标题+ page_elements = merge_adjacent_headings(page_elements)+merged_elements = []i = 0while i < len(page_elements):@@ -293,7 +516,7 @@ def extract_pdf_to_markdown(pdf_path: str, output_path: str = None) -> str:j += 1merged_elements.append({"type": 0,- "content": merged_content,+ "content": remove_page_footer(merged_content),"is_heading": False,"is_list": False})@@ -303,32 +526,64 @@ def extract_pdf_to_markdown(pdf_path: str, output_path: str = None) -> str:i += 1prev_was_list = False+ prev_was_code = False+ code_block_lines = []++ def flush_code_block():+ """输出累积的代码块"""+ nonlocal code_block_lines, markdown_content+ if code_block_lines:+ markdown_content += "```\n"+ markdown_content += "\n".join(code_block_lines) + "\n"+ markdown_content += "```\n\n"+ code_block_lines = []+for el in merged_elements:if el["type"] == 0:is_list = el.get("is_list", False)is_heading = el.get("is_heading", False)+ is_code = el.get("is_code", False)- if is_heading:+ if is_code:+ # 累积代码行if prev_was_list:markdown_content += "\n"- markdown_content += el["content"] + "\n\n"- prev_was_list = False- elif is_list:- markdown_content += el["content"] + "\n"- prev_was_list = True+ prev_was_list = False+ code_block_lines.append(el["content"])+ prev_was_code = Trueelse:- if prev_was_list:- markdown_content += "\n"- markdown_content += el["content"] + "\n\n"- prev_was_list = False+ # 非代码行,先输出累积的代码块+ if prev_was_code:+ flush_code_block()+ prev_was_code = False++ if is_heading:+ if prev_was_list:+ markdown_content += "\n"+ markdown_content += el["content"] + "\n\n"+ prev_was_list = False+ elif is_list:+ markdown_content += el["content"] + "\n"+ prev_was_list = True+ else:+ if prev_was_list:+ markdown_content += "\n"+ markdown_content += el["content"] + "\n\n"+ prev_was_list = Falseelif el["type"] == 2:+ if prev_was_code:+ flush_code_block()+ prev_was_code = Falseif prev_was_list:markdown_content += "\n"markdown_content += el["content"] + "\n\n"prev_was_list = Falseelif el["type"] == 1:+ if prev_was_code:+ flush_code_block()+ prev_was_code = Falseif img_dir:block = el["content"]ext = block["ext"]@@ -349,6 +604,10 @@ def extract_pdf_to_markdown(pdf_path: str, output_path: str = None) -> str:print(f" ✓ 提取图片: {image_name}")except Exception as e:print(f" ⚠️ 图片保存失败: {e}")++ # 页面末尾刷新代码块+ if prev_was_code:+ flush_code_block()doc.close()@@ -385,7 +644,7 @@ def process_directory(input_dir: str, output_dir: str = None):def main():parser = argparse.ArgumentParser(- description='PDF 转 Markdown 工具(支持结构识别)',+ description='PDF 转 Markdown 工具(支持结构识别与 LLM 优化)',formatter_class=argparse.RawDescriptionHelpFormatter,epilog='''示例:@@ -398,7 +657,9 @@ def main():- 自动识别标题层级(基于字体大小)- 识别粗体和斜体文本- 识别有序和无序列表- - 提取表格并转为 Markdown 格式+ - 提取表格并转为 Markdown 格式 (自动去重)+ - [新] 智能检测并移除页面重复的页眉/页脚+ - [新] 添加 <!-- Page N --> 分页符,辅助 LLM 理解''')