World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
//! MinerU 读出来的版面(`content_list`)→ 一份正文 + 按页的出处段(0040 第二刀)。
//!
//! MinerU 先认版面、再逐区域认字,交回一串按阅读顺序排好的区域:正文、标题、表、公式、
//! 图注,每个带 `page_idx` 和 `bbox`(映射到 0–1000)。这里把它们拼成分块器认得的
//! Markdown——标题还是标题,表还是表,于是扫描件的块跟原生文档按同一套规矩切:表头
//! 跟着续块走、标题贴着下一段。
//!
//! **段按页分,不按区域分。**一块只装一种出处(决定 2),而每个区域的框都不一样;按区域
//! 分段,一段话一块,块就碎到抽取看不见上下文。页是人核对时翻到的那个单位,所以一页一
//! 段,块切好之后再把它盖住的那几个区域的框并成一个,写进锚点。
//!
//! 页码从 1 数(`page_idx` + 1):锚点是给人翻页用的,查看器的 `#page=` 也从 1 数。
use crate::provenance::{Origin, Provenance, Segment};
use crate::reading::{Place, Reading, Region};
use serde_json::{json, Value};
/// 版面上的辅助区域:页眉、页脚、页码。每页重复一遍,进了正文只会让每块都多一行噪声。
/// 页脚注和旁注不在这里——合同的脚注里写着条款
const CHROME: &[&str] = &["header", "footer", "page_number"];
/// 读 `content_list`。`model` 记在出处上:哪个版本、哪个后端读的
pub fn reading(content_list: &Value, model: &str) -> Reading {
let mut text = String::new();
let mut regions: Vec<Region> = Vec::new();
for item in content_list.as_array().map(Vec::as_slice).unwrap_or(&[]) {
let kind = item["type"].as_str().unwrap_or("text");
if CHROME.contains(&kind) {
continue;
}
let Some(body) = render(kind, item) else {
continue;
};
let body = without_nul(&body);
let body = body.trim();
if body.is_empty() {
continue;
}
if !text.is_empty() {
text.push_str("\n\n");
}
let start = text.len();
text.push_str(body);
regions.push(Region {
range: start..text.len(),
place: Place::Page {
page: item["page_idx"].as_u64().unwrap_or(0) + 1,
bbox: bbox(&item["bbox"]),
},
});
}
// 一页一段,段首尾相接盖满整份正文:段从这一页第一个区域开始,到下一页第一个区域为止
let mut segments: Vec<Segment> = Vec::new();
for r in ®ions {
let Place::Page { page, .. } = r.place else {
continue;
};
if segments
.last()
.is_some_and(|s| s.provenance.anchor.as_ref().map(|a| &a["page"]) == Some(&json!(page)))
{
continue;
}
if let Some(prev) = segments.last_mut() {
prev.range.end = r.range.start;
}
segments.push(Segment {
range: r.range.start..text.len(),
provenance: Provenance {
origin: Origin::Ocr,
model: Some(model.to_string()),
anchor: Some(json!({ "page": page })),
},
});
}
if let Some(first) = segments.first_mut() {
first.range.start = 0;
}
Reading {
text,
segments,
regions,
}
}
/// 一个区域写成 Markdown。认不出的类型有字就照字收,没字就跳过
fn render(kind: &str, item: &Value) -> Option<String> {
let s = |key: &str| item[key].as_str().map(str::trim).filter(|t| !t.is_empty());
let lines = |key: &str| -> Vec<String> {
item[key]
.as_array()
.map(|a| {
a.iter()
.filter_map(|v| v.as_str().map(str::trim).filter(|t| !t.is_empty()))
.map(String::from)
.collect()
})
.unwrap_or_default()
};
let with_notes = |caption: &str, body: Option<String>, footnote: &str| -> Option<String> {
let parts: Vec<String> = lines(caption)
.into_iter()
.chain(body)
.chain(lines(footnote))
.collect();
(!parts.is_empty()).then(|| parts.join("\n\n"))
};
match kind {
"text" => {
let t = s("text")?;
match item["text_level"].as_u64().unwrap_or(0) {
0 => Some(t.to_string()),
// 标题要单独一行才认得出;认字认出来的换行并成空格
n => Some(format!(
"{} {}",
"#".repeat(n.min(6) as usize),
t.split_whitespace().collect::<Vec<_>>().join(" ")
)),
}
}
"table" => {
let body = s("table_body").map(|html| {
crate::html::fragment_to_markdown(html)
.map(|md| md.trim().to_string())
.unwrap_or_else(|_| html.to_string())
});
with_notes("table_caption", body, "table_footnote")
}
"image" | "chart" => with_notes("image_caption", None, "image_footnote"),
"code" => with_notes(
"code_caption",
s("code_body").map(|c| format!("```\n{c}\n```")),
"code_footnote",
),
"list" => {
let items = lines("list_items");
(!items.is_empty()).then(|| items.join("\n"))
}
_ => s("text").map(String::from),
}
}
fn bbox(v: &Value) -> Option<[f64; 4]> {
let a = v.as_array()?;
if a.len() != 4 {
return None;
}
let mut out = [0.0; 4];
for (o, x) in out.iter_mut().zip(a) {
*o = x.as_f64()?;
}
Some(out)
}
/// 认字、转写出来的 NUL 跟 PDF 文本层里的一样要剥(#611)
pub(crate) fn without_nul(s: &str) -> String {
s.replace('\0', "")
}
#[cfg(test)]
mod tests {
use super::*;
fn scan() -> Value {
json!([
{ "type": "header", "text": "ACME CONFIDENTIAL", "page_idx": 0, "bbox": [0, 0, 1000, 30] },
{ "type": "text", "text": "Lease Agreement", "text_level": 1, "page_idx": 0, "bbox": [100, 50, 900, 90] },
{ "type": "text", "text": "The tenant is Beta Robotics.", "page_idx": 0, "bbox": [100, 100, 900, 140] },
{ "type": "text", "text": "Rent is paid monthly.", "page_idx": 0, "bbox": [100, 150, 800, 190] },
{ "type": "page_number", "text": "1", "page_idx": 0, "bbox": [480, 970, 520, 990] },
{ "type": "table", "table_caption": ["Schedule of rent"],
"table_body": "<table><tr><td>Year</td><td>Rent</td></tr><tr><td>2026</td><td>1,000</td></tr></table>",
"table_footnote": [], "page_idx": 1, "bbox": [100, 60, 900, 300] },
{ "type": "image", "img_path": "images/stamp.png", "image_caption": [], "page_idx": 1, "bbox": [600, 800, 900, 950] },
{ "type": "text", "text": "Signed by both parties.", "page_idx": 1, "bbox": [100, 320, 900, 360] }
])
}
#[test]
fn a_scan_reads_as_markdown_without_page_chrome() {
let r = reading(&scan(), "mineru 2.5.4 vlm");
assert!(r
.text
.starts_with("# Lease Agreement\n\nThe tenant is Beta Robotics."));
assert!(
!r.text.contains("ACME CONFIDENTIAL"),
"the header repeats on every page"
);
assert!(!r.text.contains("\n\n1\n\n"), "the page number is not text");
assert!(
r.text.contains("Schedule of rent\n\n| Year | Rent |"),
"{}",
r.text
);
assert!(r.text.contains("| 2026 | 1,000 |"));
assert!(
!r.text.contains("stamp.png"),
"an image without a caption says nothing yet"
);
}
#[test]
fn pages_are_segments_that_cover_the_text() {
let r = reading(&scan(), "mineru 2.5.4 vlm");
assert_eq!(r.segments.len(), 2);
assert_eq!(r.segments[0].range.start, 0);
assert_eq!(r.segments[0].range.end, r.segments[1].range.start);
assert_eq!(r.segments[1].range.end, r.text.len());
assert!(r.text[r.segments[1].range.clone()].starts_with("Schedule of rent"));
assert_eq!(r.segments[1].provenance.anchor, Some(json!({ "page": 2 })));
}
#[test]
fn a_chunk_stays_on_its_page_and_boxes_what_it_covers() {
let r = reading(&scan(), "mineru 2.5.4 vlm");
let pieces = r.chunk(300);
assert_eq!(pieces.len(), 2, "{pieces:#?}");
let first = &pieces[0];
assert_eq!(first.provenance.origin, Origin::Ocr);
assert_eq!(first.provenance.model.as_deref(), Some("mineru 2.5.4 vlm"));
assert_eq!(
first.provenance.anchor,
Some(json!({ "page": 1, "bbox": [100.0, 50.0, 900.0, 190.0] }))
);
assert!(first.text.contains("Rent is paid monthly."));
let second = &pieces[1];
assert_eq!(second.provenance.anchor.as_ref().unwrap()["page"], json!(2));
assert!(second.text.contains("Signed by both parties."));
assert_eq!(
second.heading.as_deref(),
Some("Lease Agreement"),
"the breadcrumb crosses the page"
);
for p in &pieces {
assert!(p
.text
.ends_with(&r.text[p.char_start as usize..p.char_end as usize]));
}
}
#[test]
fn an_empty_or_odd_list_reads_as_nothing() {
assert!(reading(&json!([]), "m").text.is_empty());
assert!(reading(&json!({"not": "a list"}), "m").segments.is_empty());
let r = reading(
&json!([{ "type": "equation", "text": "$$E = mc^2$$", "page_idx": 3 }]),
"m",
);
assert_eq!(r.text, "$$E = mc^2$$");
assert_eq!(
r.chunk(300)[0].provenance.anchor,
Some(json!({ "page": 4 }))
);
}
}