Skip to content

AI turns documents or topics into real, native PowerPoint decks—with native shapes, transitions and animations, data-backed charts and tables on demand, audio narration from speaker notes, and support for your own .pptx templates. · by Hugo He

feat(tools): 增强 pdf_to_md 结构识别能力" -m "- 基于字体大小自动识别标题层级 (H1/H2/H3) - 识别粗体、斜体文本并转换为 Markdown 格式 - 识别有序/无序列表 - 智能合并段落(避免 PDF 行断裂问题)

hugohe37ee2f60799f9
diff --git a/tools/pdf_to_md.py b/tools/pdf_to_md.py
index bf6d26d6..3fbf14ed 100644
--- a/tools/pdf_to_md.py
+++ b/tools/pdf_to_md.py
@@ -2,6 +2,7 @@
"""
PDF 转 Markdown 工具
使用 PyMuPDF 提取 PDF 文本内容并转换为 Markdown 格式
+支持标题层级、粗体、斜体、列表识别
"""
import fitz # PyMuPDF
@@ -9,11 +10,98 @@ import argparse
import os
import re
from pathlib import Path
+from collections import Counter
+
+
+def analyze_font_sizes(doc) -> dict:
+ """分析文档中的字体大小分布,用于推断标题层级"""
+ size_counter = Counter()
+
+ for page in doc:
+ blocks = page.get_text("dict")["blocks"]
+ for block in blocks:
+ if block["type"] == 0:
+ for line in block["lines"]:
+ for span in line["spans"]:
+ size = round(span["size"], 1)
+ text = span["text"].strip()
+ if text:
+ size_counter[size] += len(text)
+
+ if not size_counter:
+ return {"body": 12, "h1": 24, "h2": 18, "h3": 14}
+
+ sorted_sizes = sorted(size_counter.items(), key=lambda x: x[1], reverse=True)
+ body_size = sorted_sizes[0][0]
+
+ all_sizes = sorted(size_counter.keys(), reverse=True)
+ larger_sizes = [s for s in all_sizes if s > body_size + 1]
+
+ size_map = {"body": body_size}
+ if len(larger_sizes) >= 1:
+ size_map["h1"] = larger_sizes[0]
+ if len(larger_sizes) >= 2:
+ size_map["h2"] = larger_sizes[1]
+ if len(larger_sizes) >= 3:
+ size_map["h3"] = larger_sizes[2]
+
+ return size_map
+
+
+def get_heading_level(size: float, size_map: dict) -> int:
+ """根据字体大小返回标题级别,0 表示正文"""
+ if "h1" in size_map and size >= size_map["h1"] - 0.5:
+ return 1
+ if "h2" in size_map and size >= size_map["h2"] - 0.5:
+ return 2
+ if "h3" in size_map and size >= size_map["h3"] - 0.5:
+ return 3
+ return 0
+
+
+def format_span_text(text: str, flags: int) -> str:
+ """根据字体标志格式化文本(粗体、斜体)"""
+ text = text.strip()
+ if not text:
+ return ""
+
+ is_bold = flags & 16
+ is_italic = flags & 2
+
+ if is_bold and is_italic:
+ return f"***{text}***"
+ elif is_bold:
+ return f"**{text}**"
+ elif is_italic:
+ return f"*{text}*"
+ return text
+
+
+def detect_list_item(text: str) -> tuple:
+ """检测是否为列表项,返回 (是否列表, 列表类型, 内容)"""
+ text = text.strip()
+
+ ul_patterns = [
+ (r'^[•●○◦▪▸►]\s*', '-'),
+ (r'^[-–—]\s+', '-'),
+ (r'^\*\s+', '-'),
+ ]
+ for pattern, marker in ul_patterns:
+ match = re.match(pattern, text)
+ if match:
+ return (True, 'ul', marker + ' ' + text[match.end():])
+
+ ol_pattern = r'^(\d+)[.、)]\s*'
+ match = re.match(ol_pattern, text)
+ if match:
+ num = match.group(1)
+ return (True, 'ol', f"{num}. " + text[match.end():])
+
+ return (False, None, text)
def clean_text(text: str) -> str:
"""清理提取的文本"""
- # 移除多余的空白行
lines = text.split('\n')
cleaned_lines = []
prev_empty = False
@@ -33,6 +121,34 @@ def clean_text(text: str) -> str:
return '\n'.join(cleaned_lines)
+def merge_adjacent_formatting(text: str) -> str:
+ """合并相邻的格式标记,如 **text1****text2** → **text1 text2**"""
+ text = re.sub(r'\*\*\s*\*\*', ' ', text)
+ text = re.sub(r'\*\s*\*', ' ', text)
+ text = re.sub(r'\*\*\*\s*\*\*\*', ' ', text)
+ return text
+
+
+def is_sentence_end(text: str) -> bool:
+ """判断文本是否以句末标点结尾"""
+ text = text.rstrip()
+ if not text:
+ return True
+ end_puncts = '.。!!??::;;'
+ return text[-1] in end_puncts
+
+
+def should_merge_lines(current: dict, next_line: dict) -> bool:
+ """判断两行是否应该合并为同一段落"""
+ if current.get("is_heading") or next_line.get("is_heading"):
+ return False
+ if current.get("is_list") or next_line.get("is_list"):
+ return False
+ if is_sentence_end(current.get("content", "")):
+ return False
+ return True
+
+
def extract_pdf_to_markdown(pdf_path: str, output_path: str = None) -> str:
"""
从 PDF 提取文本、图片和表格并转换为 Markdown
@@ -46,9 +162,13 @@ def extract_pdf_to_markdown(pdf_path: str, output_path: str = None) -> str:
filename = Path(pdf_path).stem
title = re.sub(r'^\d+-', '', filename).strip()
+ print(f"📄 分析文档结构...")
+ size_map = analyze_font_sizes(doc)
+ print(f" 字体大小映射: 正文={size_map.get('body', 'N/A')}, " +
+ f"H1={size_map.get('h1', 'N/A')}, H2={size_map.get('h2', 'N/A')}, H3={size_map.get('h3', 'N/A')}")
+
markdown_content = f"# {title}\n\n"
- # 准备图片保存目录
img_dir = None
rel_img_dir = "images"
if output_path:
@@ -60,7 +180,6 @@ def extract_pdf_to_markdown(pdf_path: str, output_path: str = None) -> str:
img_count = 0
for page_num, page in enumerate(doc, 1):
- # 1. 查找表格
try:
tabs = page.find_tables()
except Exception:
@@ -68,11 +187,8 @@ def extract_pdf_to_markdown(pdf_path: str, output_path: str = None) -> str:
tab_rects = [fitz.Rect(t.bbox) for t in tabs]
- # 收集页面素有元素: (y0, type, content)
- # type: 0=text, 1=image, 2=table
page_elements = []
- # 添加表格到列表
for tab in tabs:
page_elements.append({
"y0": tab.bbox[1],
@@ -81,17 +197,13 @@ def extract_pdf_to_markdown(pdf_path: str, output_path: str = None) -> str:
})
print(f" ✓ 发现表格: P{page_num}")
- # 获取所有文本和图片块
blocks = page.get_text("dict")["blocks"]
for block in blocks:
block_rect = fitz.Rect(block["bbox"])
- # 检查块是否在某个表格区域内
is_in_table = False
for tab_rect in tab_rects:
- # 如果重叠面积超过 60%,认为该块属于表格,跳过(避免内容重复)
- # 使用 intersect 计算交集
intersect = block_rect & tab_rect
if intersect.get_area() > 0.6 * block_rect.get_area():
is_in_table = True
@@ -100,40 +212,123 @@ def extract_pdf_to_markdown(pdf_path: str, output_path: str = None) -> str:
if is_in_table:
continue
- if block["type"] == 0: # 文本块
- text = ""
+ if block["type"] == 0:
for line in block["lines"]:
+ line_text = ""
+ line_size = 0
+ line_flags = 0
+ span_count = 0
+
+ formatted_spans = []
for span in line["spans"]:
- text += span["text"]
- text += "\n"
-
- cleaned = clean_text(text)
- if cleaned:
+ span_text = span["text"]
+ if not span_text.strip():
+ if span_text:
+ formatted_spans.append(span_text)
+ continue
+
+ span_size = span["size"]
+ span_flags = span["flags"]
+
+ line_size = max(line_size, span_size)
+ line_flags |= span_flags
+ span_count += 1
+
+ heading_level = get_heading_level(span_size, size_map)
+ if heading_level > 0:
+ formatted_spans.append(span_text.strip())
+ else:
+ formatted_spans.append(format_span_text(span_text, span_flags))
+
+ line_text = ''.join(formatted_spans).strip()
+ if not line_text:
+ continue
+
+ line_text = merge_adjacent_formatting(line_text)
+
+ heading_level = get_heading_level(line_size, size_map)
+
+ is_list, list_type, list_content = detect_list_item(line_text)
+
+ if heading_level > 0:
+ prefix = '#' * heading_level + ' '
+ clean_line = re.sub(r'\*+([^*]+)\*+', r'\1', line_text)
+ final_text = prefix + clean_line
+ elif is_list:
+ final_text = list_content
+ else:
+ final_text = line_text
+
page_elements.append({
- "y0": block["bbox"][1],
+ "y0": line["bbox"][1],
"type": 0,
- "content": cleaned
+ "content": final_text,
+ "is_heading": heading_level > 0,
+ "is_list": is_list
})
- elif block["type"] == 1: # 图片块
+ elif block["type"] == 1:
page_elements.append({
"y0": block["bbox"][1],
"type": 1,
- "content": block # 暂存 block 数据后续处理
+ "content": block
})
- # 按垂直位置(y0)排序,还原页面阅读顺序
page_elements.sort(key=lambda x: x["y0"])
- # 生成 Markdown 内容
- for el in page_elements:
- if el["type"] == 0: # Text
- markdown_content += el["content"] + "\n\n"
+ merged_elements = []
+ i = 0
+ while i < len(page_elements):
+ el = page_elements[i]
+ if el["type"] == 0 and not el.get("is_heading") and not el.get("is_list"):
+ merged_content = el["content"]
+ j = i + 1
+ while j < len(page_elements):
+ next_el = page_elements[j]
+ if next_el["type"] != 0:
+ break
+ if not should_merge_lines({"content": merged_content, "is_heading": False, "is_list": False}, next_el):
+ break
+ merged_content += " " + next_el["content"]
+ j += 1
+ merged_elements.append({
+ "type": 0,
+ "content": merged_content,
+ "is_heading": False,
+ "is_list": False
+ })
+ i = j
+ else:
+ merged_elements.append(el)
+ i += 1
+
+ prev_was_list = False
+ for el in merged_elements:
+ if el["type"] == 0:
+ is_list = el.get("is_list", False)
+ is_heading = el.get("is_heading", False)
- elif el["type"] == 2: # Table
+ if is_heading:
+ if prev_was_list:
+ markdown_content += "\n"
+ markdown_content += el["content"] + "\n\n"
+ prev_was_list = False
+ elif is_list:
+ markdown_content += el["content"] + "\n"
+ prev_was_list = True
+ else:
+ if prev_was_list:
+ markdown_content += "\n"
+ markdown_content += el["content"] + "\n\n"
+ prev_was_list = False
+
+ elif el["type"] == 2:
+ if prev_was_list:
+ markdown_content += "\n"
markdown_content += el["content"] + "\n\n"
+ prev_was_list = False
- elif el["type"] == 1: # Image
+ elif el["type"] == 1:
if img_dir:
block = el["content"]
ext = block["ext"]
@@ -146,15 +341,18 @@ def extract_pdf_to_markdown(pdf_path: str, output_path: str = None) -> str:
with open(image_path, "wb") as f:
f.write(image_data)
+ if prev_was_list:
+ markdown_content += "\n"
markdown_content += f"![{image_name}]({rel_img_dir}/{image_name})\n\n"
img_count += 1
+ prev_was_list = False
print(f" ✓ 提取图片: {image_name}")
except Exception as e:
print(f" ⚠️ 图片保存失败: {e}")
doc.close()
- # 移除末尾多余的换行
+ markdown_content = re.sub(r'\n{3,}', '\n\n', markdown_content)
markdown_content = markdown_content.strip() + "\n"
if output_path:
@@ -187,7 +385,7 @@ def process_directory(input_dir: str, output_dir: str = None):
def main():
parser = argparse.ArgumentParser(
- description='PDF 转 Markdown 工具',
+ description='PDF 转 Markdown 工具(支持结构识别)',
formatter_class=argparse.RawDescriptionHelpFormatter,
epilog='''
示例:
@@ -195,6 +393,12 @@ def main():
python pdf_to_md.py book.pdf -o output.md # 指定输出文件
python pdf_to_md.py ./pdfs # 转换目录下所有 PDF
python pdf_to_md.py ./pdfs -o ./markdown # 指定输出目录
+
+结构识别功能:
+ - 自动识别标题层级(基于字体大小)
+ - 识别粗体和斜体文本
+ - 识别有序和无序列表
+ - 提取表格并转为 Markdown 格式
'''
)