Skip to content

World's first open-source enterprise world model.

A chunk says where its words came from (#719)

WaylandYang50fe8f4b1c34
diff --git a/Cargo.lock b/Cargo.lock
index 921b95b8..bf685952 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -5438,6 +5438,7 @@ dependencies = [
"percent-encoding",
"pulldown-cmark",
"quick-xml 0.42.0",
+ "serde_json",
"text-splitter",
"thiserror 2.0.20",
"tiktoken-rs",
diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rs
index ae1277f8..54c44cec 100644
--- a/crates/utopia-core/src/models.rs
+++ b/crates/utopia-core/src/models.rs
@@ -111,6 +111,9 @@ pub struct Document {
pub deleted_at: Option<DateTime<Utc>>,
/// 真删(#268 下半):内容已抹掉,回不来。行留作墓碑
pub purged_at: Option<DateTime<Utc>>,
+ /// 这份文件的字要靠哪一种模型读,而那种模型还没配:`ocr` / `transcribe`(0040)。
+ /// 文档此时是 failed;配上之后按它重新排进处理队列
+ pub reader_needed: Option<String>,
pub created_at: DateTime<Utc>,
pub updated_at: DateTime<Utc>,
}
@@ -875,6 +878,8 @@ pub struct GraphChange {
pub document_id: Option<Uuid>,
pub filename: Option<String>,
pub quote: Option<String>,
+ /// 这条引文从哪来(0040):stated / ocr / transcribed / described;没有证据为空
+ pub quote_origin: Option<String>,
}
/// 消解审核项的一侧实体摘要。
@@ -960,6 +965,13 @@ pub struct EvidenceView {
pub stale: bool,
/// 这条证据所在的文档已被删除(#268)。事实若还活着,是因为它另有出处
pub document_deleted: bool,
+ /// 引文从哪来(0040):`stated` 文件里写的、`ocr` 扫描页上认出来的、`transcribed`
+ /// 录音转写、`described` 模型对一张图的描述——最后一种没有原话可对
+ pub origin: String,
+ /// 读出这段文字的引擎或模型;原文为空
+ pub origin_model: Option<String>,
+ /// 指回原文件的位置:页码(和框)、录音起止毫秒与说话人、图在哪一页
+ pub anchor: Option<serde_json::Value>,
}
/// 这个库走到哪一步了(#313):四个页面的空状态共用同一个判断。
diff --git a/crates/utopia-ingest/Cargo.toml b/crates/utopia-ingest/Cargo.toml
index 7b620f68..8bb639a0 100644
--- a/crates/utopia-ingest/Cargo.toml
+++ b/crates/utopia-ingest/Cargo.toml
@@ -25,6 +25,7 @@ oxttl.workspace = true
oxrdfxml.workspace = true
oxrdf.workspace = true
tracing.workspace = true
+serde_json.workspace = true
[dev-dependencies]
# 只给 against_real_packs 用:包是 gz 存的,测试要解开它们
diff --git a/crates/utopia-ingest/src/chunker.rs b/crates/utopia-ingest/src/chunker.rs
index 405bad8c..e6a58795 100644
--- a/crates/utopia-ingest/src/chunker.rs
+++ b/crates/utopia-ingest/src/chunker.rs
@@ -17,6 +17,7 @@
//! 切片,只是从别处抄来的。`char_start` / `char_end` 指正文那一段,不含前缀。
use crate::blocks::{blocks, Block, Kind};
+use crate::provenance::{Provenance, Segment};
use std::ops::Range;
use std::sync::OnceLock;
use text_splitter::{ChunkConfig, TextSplitter};
@@ -30,6 +31,8 @@ pub struct ChunkPiece {
pub char_end: i32,
/// 这块所在的章节路径,`›` 分隔;没有标题的文档为 None
pub heading: Option<String>,
+ /// 这块文字从哪来(0040)。一块只有一种出处
+ pub provenance: Provenance,
}
/// 一块的预算(cl100k token)。
@@ -206,9 +209,29 @@ struct Packer<'a> {
/// 正在攒的块:前缀(章节标题)与正文
prefix: Vec<Range<usize>>,
body: Vec<Piece>,
+ /// 正文按出处分成的段;正在攒的块属于 `current` 那一段
+ segments: &'a [Segment],
+ current: usize,
}
impl<'a> Packer<'a> {
+ /// 位置 `at` 落在哪一段(段按位置排好序)
+ fn segment_of(&self, at: usize) -> usize {
+ self.segments
+ .iter()
+ .rposition(|s| s.range.start <= at)
+ .unwrap_or(0)
+ }
+
+ /// 下一项要进 `seg` 那一段:跟正在攒的块不是同一段,先把那块收掉。
+ /// 标题不跟着收——它属于接下来的正文,面包屑也照样往下传
+ fn enter_segment(&mut self, seg: usize) {
+ if seg != self.current {
+ self.flush();
+ self.current = seg;
+ }
+ }
+
fn render(&self, prefix: &[Range<usize>], body: &[Piece]) -> String {
let mut parts: Vec<String> = prefix
.iter()
@@ -270,8 +293,10 @@ impl<'a> Packer<'a> {
self.body.push(piece);
return None;
}
- // 上一块已经发出去了(长段落走退路切分会立刻 flush):接到它尾上
- if let Some(last) = self.out.last_mut() {
+ // 上一块已经发出去了(长段落走退路切分会立刻 flush):接到它尾上。
+ // 只接同一种出处的——一行转写不能混进原文那一块里借它的可信度
+ let here = &self.segments[self.current].provenance;
+ if let Some(last) = self.out.last_mut().filter(|l| &l.provenance == here) {
last.text.push_str(
"
@@ -324,6 +349,7 @@ impl<'a> Packer<'a> {
char_start: start as i32,
char_end: end as i32,
heading,
+ provenance: self.segments[self.current].provenance.clone(),
});
}
@@ -432,6 +458,28 @@ impl<'a> Packer<'a> {
}
pub fn chunk_with_budget(text: &str, budget: usize) -> Vec<ChunkPiece> {
+ let whole = [Segment {
+ range: 0..text.len(),
+ provenance: Provenance::stated(),
+ }];
+ chunk_segments(text, &whole, budget)
+}
+
+/// 按出处分段的正文分块:块不跨段(0040 决定 2),面包屑跨段照传。
+///
+/// 读扫描件的引擎一页一段、转写一句一段地交回文字;它们拼成一份正文、按同一套规矩切,
+/// 只是切点多了「出处变了」这一条。`segments` 按位置排好序、覆盖整份正文;
+/// 空的时候当作整份都是原文
+pub fn chunk_segments(text: &str, segments: &[Segment], budget: usize) -> Vec<ChunkPiece> {
+ let whole = [Segment {
+ range: 0..text.len(),
+ provenance: Provenance::stated(),
+ }];
+ let segments = if segments.is_empty() {
+ &whole[..]
+ } else {
+ segments
+ };
let mut p = Packer {
text,
budget,
@@ -440,8 +488,18 @@ pub fn chunk_with_budget(text: &str, budget: usize) -> Vec<ChunkPiece> {
pending: Vec::new(),
prefix: Vec::new(),
body: Vec::new(),
+ segments,
+ current: 0,
};
for unit in units(text, blocks(text)) {
+ let start = match &unit {
+ Unit::Heading { range, .. } | Unit::Text(range) => range.start,
+ Unit::Table { caption, head, .. } => caption.as_ref().map_or(head.start, |c| c.start),
+ };
+ // 标题不切段:它贴着下一项走,由下一项决定进哪一段
+ if !matches!(unit, Unit::Heading { .. }) {
+ p.enter_segment(p.segment_of(start));
+ }
match unit {
Unit::Heading { level, range } => p.heading(level, range),
Unit::Text(range) => p.text_unit(range),
@@ -465,6 +523,59 @@ pub fn chunk_with_budget(text: &str, budget: usize) -> Vec<ChunkPiece> {
mod tests {
use super::*;
+ /// 一块只装一种出处(0040 决定 2):原文与扫描页挨着,也分成两块;
+ /// 面包屑跨段照传;页末那几个 token 的尾巴不接到别的出处那一块上
+ #[test]
+ fn a_chunk_holds_one_provenance_and_the_breadcrumb_crosses_it() {
+ use crate::provenance::{Origin, Provenance, Segment};
+ let stated = "# Lease
+
+The tenant pays rent monthly.
+
+";
+ let page1 = "The rent is 1,000 per month.
+
+";
+ let page2 = "4
+
+";
+ let text = format!("{stated}{page1}{page2}");
+ let ocr = |page: i64| Provenance {
+ origin: Origin::Ocr,
+ model: Some("mineru".into()),
+ anchor: Some(serde_json::json!({ "page": page })),
+ };
+ let a = stated.len();
+ let b = a + page1.len();
+ let segments = vec![
+ Segment {
+ range: 0..a,
+ provenance: Provenance::stated(),
+ },
+ Segment {
+ range: a..b,
+ provenance: ocr(1),
+ },
+ Segment {
+ range: b..text.len(),
+ provenance: ocr(2),
+ },
+ ];
+ let chunks = chunk_segments(&text, &segments, BUDGET_TOKENS);
+ let origins: Vec<_> = chunks.iter().map(|c| c.provenance.clone()).collect();
+ assert_eq!(chunks.len(), 3, "{chunks:#?}");
+ assert_eq!(origins[0], Provenance::stated());
+ assert_eq!(origins[1], ocr(1));
+ assert_eq!(origins[2], ocr(2), "the page number stays on its own page");
+ assert!(chunks[1].text.contains("1,000"));
+ assert!(!chunks[0].text.contains("1,000"));
+ assert_eq!(chunks[1].heading.as_deref(), Some("Lease"));
+ // 不分段的老路:整份都是原文
+ assert!(chunk_with_budget(&text, BUDGET_TOKENS)
+ .iter()
+ .all(|c| c.provenance == Provenance::stated()));
+ }
+
fn table(n_rows: usize) -> String {
let mut s = String::from("The results of the voting were as follows:\n\n| Nominee | For | Against |\n| --- | --- | --- |\n");
for i in 0..n_rows {
diff --git a/crates/utopia-ingest/src/lib.rs b/crates/utopia-ingest/src/lib.rs
index c05b9715..b162cc61 100644
--- a/crates/utopia-ingest/src/lib.rs
+++ b/crates/utopia-ingest/src/lib.rs
@@ -105,15 +105,73 @@ mod html_tests {
}
}
+#[cfg(test)]
+mod reader_tests {
+ use super::*;
+
+ const PNG: &[u8] = &[
+ 0x89, b'P', b'N', b'G', 0x0D, 0x0A, 0x1A, 0x0A, 0, 0, 0, 0x0D,
+ ];
+ const MP3: &[u8] = &[b'I', b'D', b'3', 4, 0, 0, 0, 0, 0, 0];
+ const WAV: &[u8] = &[
+ b'R', b'I', b'F', b'F', 0x24, 0, 0, 0, b'W', b'A', b'V', b'E', b'f', b'm', b't', b' ',
+ ];
+
+ fn needs(filename: &str, bytes: &[u8]) -> Option<Reader> {
+ parse(filename, bytes)
+ .err()
+ .and_then(|e| e.downcast_ref::<NeedsReader>().map(|n| n.reader))
+ }
+
+ fn unreadable(filename: &str, bytes: &[u8]) -> bool {
+ parse(filename, bytes)
+ .err()
+ .is_some_and(|e| e.downcast_ref::<Unreadable>().is_some())
+ }
+
+ #[test]
+ fn an_image_or_a_recording_waits_for_its_reader() {
+ assert_eq!(needs("scan.png", PNG), Some(Reader::Ocr));
+ // 扩展名撒谎:文件头说了算
+ assert_eq!(needs("notes.txt", PNG), Some(Reader::Ocr));
+ assert_eq!(needs("meeting.mp3", MP3), Some(Reader::Transcribe));
+ assert_eq!(needs("meeting.wav", WAV), Some(Reader::Transcribe));
+ // 认不出文件头时看扩展名
+ assert_eq!(needs("photo.heic", &[0, 1, 2, 3]), Some(Reader::Ocr));
+ assert_eq!(needs("call.amr", &[0, 1, 2, 3]), Some(Reader::Transcribe));
+ }
+
+ #[test]
+ fn a_binary_is_not_decoded_as_text() {
+ // 老式 .doc 这类:开头夹着 NUL,从前会解成乱码进库
+ assert!(unreadable(
+ "old.doc",
+ &[0xD0, 0xCF, 0x11, 0xE0, 0, 0, 0, 0, b'x']
+ ));
+ assert!(unreadable(
+ "clip.mp4",
+ &[0, 0, 0, 0x18, b'f', b't', b'y', b'p', b'm', b'p', b'4', b'2']
+ ));
+ assert!(unreadable("empty.txt", b" \n\n "));
+ // 真正的文本照读,UTF-16 带字节序标记的不算二进制
+ assert!(parse("a.txt", "你好,世界".as_bytes()).is_ok());
+ assert!(!looks_binary(&[0xFF, 0xFE, b'h', 0, b'i', 0]));
+ // PDF 文本层抄来的正文夹着 NUL 但是合法 UTF-8:照读(#611)
+ assert!(parse("layer.md", "Revenue grew\0 twelve percent.".as_bytes()).is_ok());
+ }
+}
+
mod blocks;
mod chunker;
pub mod html;
pub mod ontology_rdf;
mod parsers;
+pub mod provenance;
-pub use chunker::{chunk_text, chunk_with_budget, ChunkPiece, BUDGET_TOKENS};
+pub use chunker::{chunk_segments, chunk_text, chunk_with_budget, ChunkPiece, BUDGET_TOKENS};
/// Decode fetched text with the same encoding detection as file ingestion.
pub use parsers::plain_text as decode_text;
+pub use provenance::{Origin, Provenance, Segment};
/// 解析产物:纯文本 + 可选结构信息。
#[derive(Debug)]
@@ -121,7 +179,111 @@ pub struct ParsedDoc {
pub text: String,
}
-/// 支持的格式(P1):pdf / docx / xlsx·xls·ods / pptx / md / txt / html / csv / json / yaml / xml / log
+/// 读出字要靠哪一种模型(0040)
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Reader {
+ /// 扫描件、图片:版面识别(MinerU 这类服务)
+ Ocr,
+ /// 录音:带说话人的转写
+ Transcribe,
+}
+
+impl Reader {
+ pub fn as_str(self) -> &'static str {
+ match self {
+ Reader::Ocr => "ocr",
+ Reader::Transcribe => "transcribe",
+ }
+ }
+
+ /// 报错里说缺的是什么
+ fn wanted(self) -> &'static str {
+ match self {
+ Reader::Ocr => "a document-reading (OCR) service",
+ Reader::Transcribe => "a transcription model that labels speakers",
+ }
+ }
+}
+
+/// 这份文件没有可以直接读的文字,要靠 [`Reader`] 那一种模型读。
+///
+/// **不是解析失败**:文件没坏,只是这一步轮不到文本解析器。从前图片和录音会掉进
+/// 「按文本解码」那一支,解出一堆乱码、照样分块、嵌入、抽取;扫描件则报一句笼统的
+/// 「没抽出文字」,重试三次。现在它们停在这里,交给配了模型的读取器,没配就降级并告警
+#[derive(Debug, Clone, thiserror::Error)]
+#[error("This {what} has no text layer; reading it needs {}", reader.wanted())]
+pub struct NeedsReader {
+ pub reader: Reader,
+ /// 给人看的是什么文件:scanned PDF / image / recording
+ pub what: &'static str,
+}
+
+/// 这份文件读不了,换什么模型也读不了(视频、可执行文件、老式二进制格式)。重试没用
+#[derive(Debug, Clone, thiserror::Error)]
+#[error("{0}")]
+pub struct Unreadable(pub String);
+
+const IMAGE_EXTS: &[&str] = &[
+ "png", "jpg", "jpeg", "gif", "bmp", "tif", "tiff", "webp", "heic", "heif",
+];
+const AUDIO_EXTS: &[&str] = &[
+ "mp3", "wav", "m4a", "flac", "ogg", "oga", "opus", "aac", "amr", "wma",
+];
+const VIDEO_EXTS: &[&str] = &["mp4", "mov", "mkv", "avi", "webm", "m4v", "wmv"];
+
+/// 按文件头(认得出来的话)或扩展名判断这是不是要模型读的媒体文件。
+///
+/// 文件头优先:扩展名可能撒谎。认不出文件头的才看扩展名
+fn media_kind(bytes: &[u8], ext: &str) -> Option<Result<NeedsReader, Unreadable>> {
+ use infer::MatcherType;
+ let by_header = infer::get(bytes).map(|t| t.matcher_type());
+ let is = |m: MatcherType, exts: &[&str]| match by_header {
+ Some(h) => h == m,
+ None => exts.contains(&ext),
+ };
+ if is(MatcherType::Image, IMAGE_EXTS) {
+ return Some(Ok(NeedsReader {
+ reader: Reader::Ocr,
+ what: "image",
+ }));
+ }
+ if is(MatcherType::Audio, AUDIO_EXTS) {
+ return Some(Ok(NeedsReader {
+ reader: Reader::Transcribe,
+ what: "recording",
+ }));
+ }
+ if is(MatcherType::Video, VIDEO_EXTS) {
+ return Some(Err(Unreadable(
+ "Video files are not read yet; upload its audio track or a transcript".into(),
+ )));
+ }
+ None
+}
+
+/// 按文本解码之前看一眼是不是二进制:开头夹着 NUL、**而且**不是合法的 UTF-8。
+///
+/// 两条都要:PDF 文本层抄出来的正文会夹 NUL,但它是合法的 UTF-8(#611,照读、剥掉 NUL);
+/// GBK 这类中文编码不是合法的 UTF-8,但不夹 NUL。老式 .doc、压缩包两条都占。
+/// 带 UTF-16 字节序标记的不算
+fn looks_binary(bytes: &[u8]) -> bool {
+ if bytes.starts_with(&[0xFF, 0xFE]) || bytes.starts_with(&[0xFE, 0xFF]) {
+ return false;
+ }
+ let head = &bytes[..bytes.len().min(8192)];
+ if !head.contains(&0) {
+ return false;
+ }
+ // 截断可能切在一个多字节字符中间:只看完整的那一段是否合法
+ match std::str::from_utf8(head) {
+ Ok(_) => false,
+ Err(e) => e.error_len().is_some(),
+ }
+}
+
+/// 支持的格式(P1):pdf / docx / xlsx·xls·ods / pptx / md / txt / html / csv / json / yaml / xml / log。
+///
+/// 要模型读的(扫描件、图片、录音)返回挂着 [`NeedsReader`] 的错误;读不了的挂 [`Unreadable`]
pub fn parse(filename: &str, bytes: &[u8]) -> anyhow::Result<ParsedDoc> {
let ext = filename
.rsplit('.')
@@ -129,23 +291,47 @@ pub fn parse(filename: &str, bytes: &[u8]) -> anyhow::Result<ParsedDoc> {
.map(|s| s.to_ascii_lowercase())
.unwrap_or_default();
+ match media_kind(bytes, &ext) {
+ Some(Ok(needs)) => return Err(needs.into()),
+ Some(Err(unreadable)) => return Err(unreadable.into()),
+ None => {}
+ }
+
// 魔数探测优先于扩展名(扩展名可能撒谎)
let kind = infer::get(bytes).map(|t| t.extension()).unwrap_or("");
let text = match (kind, ext.as_str()) {
- ("pdf", _) | (_, "pdf") => parsers::pdf(bytes)?,
+ ("pdf", _) | (_, "pdf") => {
+ let text = parsers::pdf(bytes)?;
+ // 文本层是空的:扫描件,字在图里
+ if text.trim().is_empty() {
+ return Err(NeedsReader {
+ reader: Reader::Ocr,
+ what: "scanned PDF",
+ }
+ .into());
+ }
+ text
+ }
("docx", _) | (_, "docx") => parsers::docx(bytes)?,
("xlsx", _) | (_, "xlsx") | (_, "xls") | (_, "ods") => parsers::spreadsheet(bytes)?,
("pptx", _) | (_, "pptx") => parsers::pptx(bytes)?,
(_, "html") | (_, "htm") => parsers::html(bytes)?,
(_, "csv") | (_, "tsv") => parsers::csv_text(bytes, ext == "tsv")?,
- // md/json/yaml/xml/log/txt 及一切未识别格式:按文本解码(编码探测覆盖 GBK 等)
+ // md/json/yaml/xml/log/txt 及一切未识别格式:按文本解码(编码探测覆盖 GBK 等)。
+ // 解码之前先看是不是二进制:老式 .doc、压缩包、可执行文件解出来是乱码,
+ // 从前照样分块、嵌入、抽取
+ _ if looks_binary(bytes) => {
+ return Err(
+ Unreadable("This file is not in a format that can be read as text".into()).into(),
+ )
+ }
_ => parsers::plain_text(bytes),
};
let text = normalize(&text);
if text.trim().is_empty() {
- anyhow::bail!("No text could be extracted (possibly a scanned or empty file)");
+ return Err(Unreadable("No text could be extracted from this file".into()).into());
}
Ok(ParsedDoc { text })
}
diff --git a/crates/utopia-ingest/src/provenance.rs b/crates/utopia-ingest/src/provenance.rs
new file mode 100644
index 00000000..1df81e20
--- /dev/null
+++ b/crates/utopia-ingest/src/provenance.rs
@@ -0,0 +1,72 @@
+//! 一段文字从哪来(0040):原文写的、从扫描页上认出来的、从录音里转出来的、还是模型看图说的。
+//!
+//! 分块之后的一切只吃文字,分不出一句话是写在文件里的,还是模型转述的。这四种不是同等的证据:
+//! 前三种是「原话,照读」,人能对着页面或录音核对;第四种是模型对一张图的转述,没有原话可对。
+//! 所以来源跟着分块走,不跟着证据行走——证据引用的就是分块,抽取读的也是分块。
+
+use std::ops::Range;
+
+/// 文字的来源。数据库里存 [`Origin::as_str`] 那个词
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum Origin {
+ /// 文件里写着的
+ Stated,
+ /// 从扫描页、截图上认出来的字(可能认错一个数字)
+ Ocr,
+ /// 录音转写(名字最容易听错,而名字正是抽取要抽的)
+ Transcribed,
+ /// 模型对一张图的描述:没有人写过、说过这句话
+ Described,
+}
+
+impl Origin {
+ pub fn as_str(self) -> &'static str {
+ match self {
+ Origin::Stated => "stated",
+ Origin::Ocr => "ocr",
+ Origin::Transcribed => "transcribed",
+ Origin::Described => "described",
+ }
+ }
+
+ pub fn parse(s: &str) -> Option<Self> {
+ match s {
+ "stated" => Some(Origin::Stated),
+ "ocr" => Some(Origin::Ocr),
+ "transcribed" => Some(Origin::Transcribed),
+ "described" => Some(Origin::Described),
+ _ => None,
+ }
+ }
+}
+
+/// 一段文字的出处:来源、读出它的引擎或模型、以及指回原文件字节的锚点。
+///
+/// 锚点的形状由来源决定(数据库上有 CHECK):原文没有锚点(`char_start` / `char_end`
+/// 已经给出位置);扫描页是 `{"page": n}`,引擎给了框就带 `"bbox"`;录音是
+/// `{"start_ms", "end_ms", "speaker"}`;图片描述指到页和图
+#[derive(Debug, Clone, PartialEq)]
+pub struct Provenance {
+ pub origin: Origin,
+ pub model: Option<String>,
+ pub anchor: Option<serde_json::Value>,
+}
+
+impl Provenance {
+ pub fn stated() -> Self {
+ Provenance {
+ origin: Origin::Stated,
+ model: None,
+ anchor: None,
+ }
+ }
+}
+
+/// 正文里的一段,连同它的出处。读扫描件、录音的引擎一页、一段地交回文字,拼成一份正文
+/// 交给分块;分块按段切开,**一块只装一种出处**(0040 决定 2)
+#[derive(Debug, Clone)]
+pub struct Segment {
+ /// 在拼好的正文里的字节区间
+ pub range: Range<usize>,
+ pub provenance: Provenance,
+}
diff --git a/crates/utopia-server/src/alerting.rs b/crates/utopia-server/src/alerting.rs
index 3327f342..377470b3 100644
--- a/crates/utopia-server/src/alerting.rs
+++ b/crates/utopia-server/src/alerting.rs
@@ -56,6 +56,39 @@ pub async fn observe_job_failure(
state.emit_alert();
}
+/// 一份文件的字要靠没配的那种模型读(0040):报一条库级告警,名字存进告警里——
+/// 文件删了告警还读得懂
+pub async fn observe_document_needs_reader(
+ state: &AppState,
+ kb_id: uuid::Uuid,
+ document_id: uuid::Uuid,
+ filename: &str,
+ needs: &utopia_ingest::NeedsReader,
+) {
+ if let Err(e) = alerts::raise(
+ &state.pool,
+ alerts::NewAlert {
+ kb_id: Some(kb_id),
+ severity: "warning",
+ kind: alerts::kind::DOCUMENT_NEEDS_READER,
+ min_role: Role::Editor,
+ subject_type: Some("document"),
+ subject_id: Some(document_id),
+ detail: serde_json::json!({
+ "name": filename,
+ "reader": needs.reader.as_str(),
+ "error": needs.to_string(),
+ }),
+ },
+ )
+ .await
+ {
+ tracing::warn!(error = %e, "上报告警失败");
+ return;
+ }
+ state.emit_alert();
+}
+
/// 这次失败还值不值得重试。
///
/// **只有欠费是没救的。** 限流会自己恢复,端点不可达可能是重启中的服务,
diff --git a/crates/utopia-server/src/api/tools.rs b/crates/utopia-server/src/api/tools.rs
index a09e2093..f34b83fd 100644
--- a/crates/utopia-server/src/api/tools.rs
+++ b/crates/utopia-server/src/api/tools.rs
@@ -545,6 +545,7 @@ pub async fn changes(ctx: &ToolCtx<'_>, args: &serde_json::Value) -> ToolResult
"valid_from_precision": r.valid_from_precision,
"valid_to_precision": r.valid_to_precision,
"document_id": r.document_id, "filename": r.filename, "quote": r.quote,
+ "quote_origin": r.quote_origin,
})).collect::<Vec<_>>()
}))
}
@@ -1074,6 +1075,7 @@ mod tests {
document_id: None,
filename: None,
quote: None,
+ quote_origin: None,
}
}
diff --git a/crates/utopia-server/src/extraction.rs b/crates/utopia-server/src/extraction.rs
index 98827774..59a028a0 100644
--- a/crates/utopia-server/src/extraction.rs
+++ b/crates/utopia-server/src/extraction.rs
@@ -81,6 +81,21 @@ async fn chat_retrying_rate_limits(
const MIN_CONFIDENCE: f32 = 0.6;
+/// 模型看图描述出来的事实,置信度压到这里(0040 决定 4):低于时态引擎自动关闭的门槛,
+/// 它能进图、能被搜到和引用,却不能单独把一段正确的旧值关掉——看柱状图读错一个数字是常事,
+/// 按普通事实入库的话,错的数会关掉它反驳的那个对的数,而库里没有一行说这次关闭靠的是一张图。
+/// 真要关,时态引擎记一条低置信冲突,交给人或治理去判
+const DESCRIBED_CEILING: f32 = utopia_store::temporal::AUTO_CLOSE_MIN_CONFIDENCE - 0.05;
+
+/// 这块文字的来源给事实置信度设的上限
+fn origin_ceiling(origin: &str, confidence: f32) -> f32 {
+ if origin == "described" {
+ confidence.min(DESCRIBED_CEILING)
+ } else {
+ confidence
+ }
+}
+
/// 这串字**是不是一个东西的名字**。
///
/// 判据是**词数**不是字符数。字符数分不开真假:
@@ -1585,6 +1600,7 @@ async fn run(state: &AppState, document_id: Uuid, proposer: Proposer) -> anyhow:
.await;
continue;
}
+ let confidence = origin_ceiling(&chunk.origin, confidence);
let validity =
validity_of(f.valid_from.as_deref(), f.valid_to.as_deref(), doc.doc_time);
@@ -3390,6 +3406,22 @@ fn unit_for(
}
}
+#[cfg(test)]
+mod origin_ceiling_tests {
+ use super::*;
+
+ #[test]
+ fn a_described_fact_cannot_close_a_value_by_itself() {
+ let ceiling = origin_ceiling("described", 0.95);
+ assert!(ceiling < utopia_store::temporal::AUTO_CLOSE_MIN_CONFIDENCE);
+ assert!(ceiling >= MIN_CONFIDENCE, "it still enters the graph");
+ assert_eq!(origin_ceiling("described", 0.62), 0.62);
+ for origin in ["stated", "ocr", "transcribed"] {
+ assert_eq!(origin_ceiling(origin, 0.95), 0.95);
+ }
+ }
+}
+
#[cfg(test)]
mod unit_for_tests {
use super::unit_for;
diff --git a/crates/utopia-server/src/pipeline.rs b/crates/utopia-server/src/pipeline.rs
index 75bfd053..0a4f9ee4 100644
--- a/crates/utopia-server/src/pipeline.rs
+++ b/crates/utopia-server/src/pipeline.rs
@@ -37,11 +37,42 @@ pub async fn process_document(state: &AppState, document_id: Uuid) -> anyhow::Re
match run(state, document_id).await {
Ok(()) => Ok(()),
Err(e) => {
+ let doc = utopia_store::documents::get(&state.pool, document_id)
+ .await
+ .ok();
+ // 字要靠模型读、而那种模型没配(0040):**降级**。文件留着,文档停在 failed
+ // 并记下缺哪一种,报一条库级告警;配上模型会自己重新处理。重试没用——
+ // 模型不会在两分钟里自己配上,所以挂 Terminal
+ if let Some(needs) = e.downcast_ref::<utopia_ingest::NeedsReader>() {
+ let _ = utopia_store::documents::set_needs_reader(
+ &state.pool,
+ document_id,
+ needs.reader.as_str(),
+ &needs.to_string(),
+ )
+ .await;
+ if let Some(doc) = &doc {
+ crate::alerting::observe_document_needs_reader(
+ state,
+ doc.kb_id,
+ document_id,
+ &doc.filename,
+ needs,
+ )
+ .await;
+ state.emit_document(doc.kb_id, document_id);
+ }
+ return Err(e.context(utopia_core::Terminal));
+ }
let _ =
utopia_store::documents::set_failed(&state.pool, document_id, &e.to_string()).await;
- if let Ok(doc) = utopia_store::documents::get(&state.pool, document_id).await {
+ if let Some(doc) = &doc {
state.emit_document(doc.kb_id, document_id);
}
+ // 读不了的格式(视频、二进制、空文件)换多少次也一样
+ if e.downcast_ref::<utopia_ingest::Unreadable>().is_some() {
+ return Err(e.context(utopia_core::Terminal));
+ }
Err(e)
}
}
diff --git a/crates/utopia-server/src/pipeline_tests.rs b/crates/utopia-server/src/pipeline_tests.rs
index 333a13c5..ed728ff0 100644
--- a/crates/utopia-server/src/pipeline_tests.rs
+++ b/crates/utopia-server/src/pipeline_tests.rs
@@ -186,18 +186,23 @@ impl Fx {
/// 同上,正文由调用方给
async fn document_with_text(&self, text: &str) -> anyhow::Result<Uuid> {
+ self.document_with_bytes("long.md", text.as_bytes()).await
+ }
+
+ /// 任意字节、任意文件名:图片、录音、二进制都从这里进
+ async fn document_with_bytes(&self, filename: &str, bytes: &[u8]) -> anyhow::Result<Uuid> {
use sha2::{Digest, Sha256};
- let sha: String = Sha256::digest(text.as_bytes())
+ let sha: String = Sha256::digest(bytes)
.iter()
.map(|b| format!("{b:02x}"))
.collect();
- self.state.blob.put(&sha, text.as_bytes()).await?;
+ self.state.blob.put(&sha, bytes).await?;
Ok(utopia_store::documents::create(
&self.pool,
self.kb,
- "long.md",
- "text/markdown",
- text.len() as i64,
+ filename,
+ "application/octet-stream",
+ bytes.len() as i64,
&sha,
None,
None,
@@ -207,6 +212,16 @@ impl Fx {
.id)
}
+ async fn alerts(&self, kind: &str) -> anyhow::Result<Vec<serde_json::Value>> {
+ Ok(sqlx::query_scalar(
+ "SELECT detail FROM alerts WHERE kb_id = $1 AND kind = $2 ORDER BY created_at",
+ )
+ .bind(self.kb)
+ .bind(kind)
+ .fetch_all(&self.pool)
+ .await?)
+ }
+
async fn embed(&self, doc: Uuid) -> anyhow::Result<usize> {
let settings = utopia_store::settings::get(&self.pool, self.ws)
.await?
@@ -367,6 +382,66 @@ async fn a_nul_byte_does_not_fail_the_whole_document() -> anyhow::Result<()> {
f.cleanup().await
}
+/// 0040:图片、扫描件、录音的字要靠模型读。没配那种模型时**降级**:文件留着、文档停在
+/// failed 并记下缺哪一种、报一条库级告警;不重试,也不再解出一堆乱码去分块、嵌入、抽取
+#[tokio::test]
+async fn a_file_that_needs_a_reader_waits_and_says_so() -> anyhow::Result<()> {
+ let Some(f) = fixture(FakeEmbed::new(Duration::from_millis(5))).await? else {
+ return Ok(());
+ };
+ let png = [
+ 0x89, b'P', b'N', b'G', 0x0D, 0x0A, 0x1A, 0x0A, 0, 0, 0, 0x0D, b'I', b'H',
+ ];
+ let mp3 = [b'I', b'D', b'3', 4, 0, 0, 0, 0, 0, 0, 0xFF, 0xFB];
+ let image = f.document_with_bytes("contract-scan.png", &png).await?;
+ let recording = f.document_with_bytes("board-meeting.mp3", &mp3).await?;
+
+ for (doc, reader) in [(image, "ocr"), (recording, "transcribe")] {
+ let err = super::process_document(&f.state, doc)
+ .await
+ .expect_err("nothing can be read without the model");
+ assert!(
+ utopia_core::is_terminal(&err),
+ "retrying cannot configure a model"
+ );
+ let row = utopia_store::documents::get(&f.pool, doc).await?;
+ assert_eq!(row.status, "failed");
+ assert_eq!(row.reader_needed.as_deref(), Some(reader));
+ assert!(row.error.is_some(), "the document says why");
+ assert!(f.stored(doc).await?.is_empty(), "no garbage chunks");
+ }
+ let alerts = f.alerts("document.needs_reader").await?;
+ assert_eq!(alerts.len(), 2);
+ assert_eq!(alerts[0]["name"], "contract-scan.png");
+ assert_eq!(alerts[0]["reader"], "ocr");
+ assert_eq!(alerts[1]["reader"], "transcribe");
+ f.cleanup().await
+}
+
+/// 读不了的格式(老式 .doc、压缩包):一次失败、不重试,也不进库成乱码;它不缺模型,不报那条告警
+#[tokio::test]
+async fn a_binary_file_fails_once_instead_of_becoming_text() -> anyhow::Result<()> {
+ let Some(f) = fixture(FakeEmbed::new(Duration::from_millis(5))).await? else {
+ return Ok(());
+ };
+ let doc = f
+ .document_with_bytes(
+ "minutes.doc",
+ &[0xD0, 0xCF, 0x11, 0xE0, 0xA1, 0xB1, 0, 0, 0, 0],
+ )
+ .await?;
+ let err = super::process_document(&f.state, doc)
+ .await
+ .expect_err("not text");
+ assert!(utopia_core::is_terminal(&err));
+ let row = utopia_store::documents::get(&f.pool, doc).await?;
+ assert_eq!(row.status, "failed");
+ assert_eq!(row.reader_needed, None);
+ assert!(f.stored(doc).await?.is_empty());
+ assert!(f.alerts("document.needs_reader").await?.is_empty());
+ f.cleanup().await
+}
+
#[test]
fn without_nul_borrows_when_there_is_nothing_to_strip() {
use std::borrow::Cow;
diff --git a/crates/utopia-server/src/rdf.rs b/crates/utopia-server/src/rdf.rs
index 512c5558..8be69b9b 100644
--- a/crates/utopia-server/src/rdf.rs
+++ b/crates/utopia-server/src/rdf.rs
@@ -500,6 +500,10 @@ pub fn emit_fact(
for quote in &f.quotes {
sink.l(&stmt, &utopia("quote"), &text(quote.clone()))?;
}
+ // 引文从哪来(0040):审计的人要不问我们就分得清一句原文和一段看图描述(0020)
+ for origin in &f.quote_origins {
+ sink.l(&stmt, &utopia("evidenceOrigin"), &text(origin.clone()))?;
+ }
// 边上的属性(0037):陈述节点上各多一行,谓词是属性的 IRI,字面量按它的 datatype
for q in &f.qualifiers {
let Some(p) = vocab.relation(q.qualifier_type_id) else {
@@ -731,6 +735,7 @@ mod tests {
supersedes: None,
documents: vec![],
quotes: vec![],
+ quote_origins: vec![],
}
}
diff --git a/crates/utopia-store/src/alerts.rs b/crates/utopia-store/src/alerts.rs
index dfa110eb..53358007 100644
--- a/crates/utopia-store/src/alerts.rs
+++ b/crates/utopia-store/src/alerts.rs
@@ -58,6 +58,11 @@ pub mod kind {
/// (0025 决定 9)。`severity = warning`,`min_role = editor`:裁重复项的人就是
/// 该去看 Agent 队列、决定要不要再打开的人
pub const GOVERNANCE_TRIPPED: &str = "governance.tripped";
+ /// 库级:一份文件的字要靠模型读(扫描件、图片要版面识别,录音要转写),而那种模型
+ /// 没配(0040)。`severity = warning`,`min_role = editor`:文件留着、文档停在 failed,
+ /// 在管理页「模型」里配上之后会自己重新处理。没有这一条,传上来的扫描件只是悄悄地
+ /// 什么都没有——0005 里一直空着的 `document.no_text_layer` 就是它
+ pub const DOCUMENT_NEEDS_READER: &str = "document.needs_reader";
}
/// 一次故障。打包成结构体不只是为了参数个数——调用点写 `severity: "error"`
diff --git a/crates/utopia-store/src/documents.rs b/crates/utopia-store/src/documents.rs
index e0651030..f5fb30a0 100644
--- a/crates/utopia-store/src/documents.rs
+++ b/crates/utopia-store/src/documents.rs
@@ -749,11 +749,15 @@ pub async fn list_missing(pool: &PgPool, source_id: Uuid) -> AppResult<Vec<Uuid>
}
pub async fn set_status(pool: &PgPool, id: Uuid, status: &str) -> AppResult<()> {
- sqlx::query("UPDATE documents SET status = $2, error = NULL, updated_at = now() WHERE id = $1")
- .bind(id)
- .bind(status)
- .execute(pool)
- .await?;
+ // 重新开始处理:上一次缺的读取模型不再算数,读不出来会再记一次
+ sqlx::query(
+ "UPDATE documents SET status = $2, error = NULL, reader_needed = NULL, updated_at = now()
+ WHERE id = $1",
+ )
+ .bind(id)
+ .bind(status)
+ .execute(pool)
+ .await?;
Ok(())
}
@@ -768,10 +772,36 @@ pub async fn set_failed(pool: &PgPool, id: Uuid, error: &str) -> AppResult<()> {
Ok(())
}
+/// 这份文件要 `reader` 那一种模型才读得出字,而那种模型没配(0040):停在 failed,
+/// 记下缺的是哪一种,配上之后按它重新排队
+pub async fn set_needs_reader(pool: &PgPool, id: Uuid, reader: &str, error: &str) -> AppResult<()> {
+ sqlx::query(
+ "UPDATE documents SET status = 'failed', error = $3, reader_needed = $2, updated_at = now()
+ WHERE id = $1",
+ )
+ .bind(id)
+ .bind(reader)
+ .bind(error)
+ .execute(pool)
+ .await?;
+ Ok(())
+}
+
+/// 等着 `reader` 那一种模型的文档(活着的),配上模型之后重新排进处理队列用
+pub async fn waiting_for_reader(pool: &PgPool, reader: &str) -> AppResult<Vec<(Uuid, Uuid)>> {
+ Ok(sqlx::query_as(
+ "SELECT id, kb_id FROM documents
+ WHERE reader_needed = $1 AND deleted_at IS NULL ORDER BY created_at, id",
+ )
+ .bind(reader)
+ .fetch_all(pool)
+ .await?)
+}
+
pub async fn set_ready(pool: &PgPool, id: Uuid, text_len: i32, chunk_count: i32) -> AppResult<()> {
sqlx::query(
- "UPDATE documents SET status = 'ready', error = NULL, text_len = $2, chunk_count = $3,
- updated_at = now() WHERE id = $1",
+ "UPDATE documents SET status = 'ready', error = NULL, reader_needed = NULL, text_len = $2,
+ chunk_count = $3, updated_at = now() WHERE id = $1",
)
.bind(id)
.bind(text_len)
@@ -1153,16 +1183,41 @@ pub async fn replace_chunks(
.await?;
// 认领池:现行块按文本分组(同文重复块按多重集配对,各认领各的)
- let old: Vec<(Uuid, String)> = sqlx::query_as(
- "SELECT id, text FROM chunks WHERE document_id = $1 AND superseded_at IS NULL",
+ // 认领按「文本 + 出处」:同一句话从原文里读到和从扫描页上认出来,不是同一块(0040)
+ type Stored = (
+ Uuid,
+ String,
+ String,
+ Option<String>,
+ Option<serde_json::Value>,
+ );
+ let old: Vec<Stored> = sqlx::query_as(
+ "SELECT id, text, origin, origin_model, anchor FROM chunks
+ WHERE document_id = $1 AND superseded_at IS NULL",
)
.bind(document_id)
.fetch_all(&mut *tx)
.await?;
- let mut claim_pool: std::collections::HashMap<String, Vec<Uuid>> =
+ type ClaimKey = (String, String, Option<String>, Option<String>);
+ let key_of = |text: &str,
+ origin: &str,
+ model: Option<&str>,
+ anchor: Option<&serde_json::Value>|
+ -> ClaimKey {
+ (
+ text.to_string(),
+ origin.to_string(),
+ model.map(str::to_string),
+ anchor.map(|a| a.to_string()),
+ )
+ };
+ let mut claim_pool: std::collections::HashMap<ClaimKey, Vec<Uuid>> =
std::collections::HashMap::new();
- for (id, text) in old {
- claim_pool.entry(text).or_default().push(id);
+ for (id, text, origin, model, anchor) in old {
+ claim_pool
+ .entry(key_of(&text, &origin, model.as_deref(), anchor.as_ref()))
+ .or_default()
+ .push(id);
}
// 第一阶段:认领(原行更新)与待插清单——新块必须等软删跑完再插,
@@ -1171,7 +1226,15 @@ pub async fn replace_chunks(
let mut to_insert: Vec<(Uuid, &ChunkPiece)> = Vec::new();
let mut out = Vec::with_capacity(pieces.len());
for piece in pieces {
- let claimed = claim_pool.get_mut(&piece.text).and_then(|ids| ids.pop());
+ let p = &piece.provenance;
+ let claimed = claim_pool
+ .get_mut(&key_of(
+ &piece.text,
+ p.origin.as_str(),
+ p.model.as_deref(),
+ p.anchor.as_ref(),
+ ))
+ .and_then(|ids| ids.pop());
if let Some(id) = claimed {
sqlx::query(
"UPDATE chunks SET seq = $2, char_start = $3, char_end = $4, doc_version = $5,
@@ -1214,8 +1277,9 @@ pub async fn replace_chunks(
for (id, piece) in to_insert {
sqlx::query(
"INSERT INTO chunks
- (id, kb_id, document_id, seq, text, char_start, char_end, doc_version, heading)
- VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9)",
+ (id, kb_id, document_id, seq, text, char_start, char_end, doc_version, heading,
+ origin, origin_model, anchor)
+ VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12)",
)
.bind(id)
.bind(kb_id)
@@ -1226,6 +1290,9 @@ pub async fn replace_chunks(
.bind(piece.char_end)
.bind(version)
.bind(&piece.heading)
+ .bind(piece.provenance.origin.as_str())
+ .bind(&piece.provenance.model)
+ .bind(&piece.provenance.anchor)
.execute(&mut *tx)
.await?;
}
@@ -1307,6 +1374,8 @@ pub struct ChunkForExtract {
pub seq: i32,
pub text: String,
pub embedding: Option<Vector>,
+ /// 这块文字从哪来(0040):看图描述出来的事实不能单独改写账本
+ pub origin: String,
}
pub async fn chunks_for_extraction(
@@ -1315,7 +1384,7 @@ pub async fn chunks_for_extraction(
) -> AppResult<Vec<ChunkForExtract>> {
// 只取未抽取的分块:认领的未变段落携带 extracted_at 跳过(增量抽取 + 断点续抽)
let rows = sqlx::query_as(
- "SELECT id, seq, text, embedding FROM chunks
+ "SELECT id, seq, text, embedding, origin FROM chunks
WHERE document_id = $1 AND superseded_at IS NULL AND extracted_at IS NULL
ORDER BY seq",
)
diff --git a/crates/utopia-store/src/export.rs b/crates/utopia-store/src/export.rs
index 27b4b8f1..46ef2fcb 100644
--- a/crates/utopia-store/src/export.rs
+++ b/crates/utopia-store/src/export.rs
@@ -83,6 +83,8 @@ pub struct ExportFact {
pub supersedes: Option<Uuid>,
pub documents: Vec<Uuid>,
pub quotes: Vec<String>,
+ /// 证据文字的来源(0040),去重:一条陈述的引文里有没有扫描、转写、看图描述来的
+ pub quote_origins: Vec<String>,
}
#[derive(Debug, Clone, sqlx::FromRow)]
@@ -194,7 +196,11 @@ pub async fn facts_page(
AS documents,
COALESCE(ARRAY(SELECT e.quote FROM fact_evidence e
WHERE e.fact_id = f.id AND e.quote IS NOT NULL
- ORDER BY e.chunk_id), '{{}}') AS quotes
+ ORDER BY e.chunk_id), '{{}}') AS quotes,
+ COALESCE(ARRAY(SELECT DISTINCT c.origin FROM fact_evidence e
+ JOIN chunks c ON c.id = e.chunk_id
+ WHERE e.fact_id = f.id
+ ORDER BY c.origin), '{{}}') AS quote_origins
FROM facts f
WHERE f.kb_id = $1 AND f.id > COALESCE($2, '00000000-0000-0000-0000-000000000000'::uuid)
ORDER BY f.id LIMIT $3",
diff --git a/crates/utopia-store/src/graph.rs b/crates/utopia-store/src/graph.rs
index d3230cdf..8ecf57e0 100644
--- a/crates/utopia-store/src/graph.rs
+++ b/crates/utopia-store/src/graph.rs
@@ -1549,7 +1549,8 @@ pub async fn fact_evidence(pool: &PgPool, fact_id: Uuid) -> AppResult<Vec<Eviden
c.doc_version < COALESCE(
(SELECT MAX(version) FROM document_versions dv
WHERE dv.document_id = c.document_id), 1) AS stale,
- d.deleted_at IS NOT NULL AS document_deleted
+ d.deleted_at IS NOT NULL AS document_deleted,
+ c.origin, c.origin_model, c.anchor
FROM fact_evidence fe
JOIN chunks c ON c.id = fe.chunk_id
JOIN documents d ON d.id = c.document_id
@@ -1839,13 +1840,13 @@ pub async fn graph_changes(
COALESCE(r.label, fact_surface_predicate(ev.id)) AS predicate_label, o.canonical_name AS object_name,
ev.object_value, ev.valid_from, ev.valid_from_precision,
ev.valid_to, ev.valid_to_precision,
- ev.confidence, src.document_id, src.filename, src.quote
+ ev.confidence, src.document_id, src.filename, src.quote, src.quote_origin
FROM ev
LEFT JOIN relation_types r ON r.id = ev.predicate_id
JOIN entities s ON s.id = ev.subject_id
LEFT JOIN entities o ON o.id = ev.object_id
LEFT JOIN LATERAL (
- SELECT d.id AS document_id, d.filename, fe.quote
+ SELECT d.id AS document_id, d.filename, fe.quote, c.origin AS quote_origin
FROM fact_evidence fe
JOIN chunks c ON c.id = fe.chunk_id
JOIN documents d ON d.id = c.document_id
diff --git a/crates/utopia-store/src/temporal.rs b/crates/utopia-store/src/temporal.rs
index 09f29991..3a91a824 100644
--- a/crates/utopia-store/src/temporal.rs
+++ b/crates/utopia-store/src/temporal.rs
@@ -30,7 +30,7 @@ use utopia_core::AppResult;
use uuid::Uuid;
/// 低于此置信度的后任不允许自动改写前任的历史(进审)。
-const AUTO_CLOSE_MIN_CONFIDENCE: f32 = 0.75;
+pub const AUTO_CLOSE_MIN_CONFIDENCE: f32 = 0.75;
/// 证据文件**自带**的最早日期,按 `facts` 的别名 `f` 投影。只认正文(`content`)与来源
/// (`source`)给的日期:上传时刻、文件修改时间不是文档自己的日期——拿它们排序,
diff --git a/crates/utopia-store/tests/a_chunk_says_where_its_words_came_from.rs b/crates/utopia-store/tests/a_chunk_says_where_its_words_came_from.rs
new file mode 100644
index 00000000..f34471d0
--- /dev/null
+++ b/crates/utopia-store/tests/a_chunk_says_where_its_words_came_from.rs
@@ -0,0 +1,163 @@
+//! 0040:一块文字从哪来,存在分块上。
+//!
+//! 1. **出处原样落库、原样认领。** 同一份正文里一块原文、一块扫描页,读回来各是各的;
+//! 再处理一遍,出处没变的块认领原行(id 不变),同一句话换了出处算新块。
+//! 2. **锚点的形状由来源定。** 转写必须带说话人,原文不许带锚点——数据库拒收。
+//!
+//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆,绝不碰已有的库。
+
+use sqlx::PgPool;
+use utopia_ingest::{ChunkPiece, Origin, Provenance};
+use uuid::Uuid;
+
+fn piece(seq: i32, text: &str, provenance: Provenance) -> ChunkPiece {
+ ChunkPiece {
+ seq,
+ text: text.into(),
+ char_start: 0,
+ char_end: text.len() as i32,
+ heading: None,
+ provenance,
+ }
+}
+
+fn ocr(page: i64) -> Provenance {
+ Provenance {
+ origin: Origin::Ocr,
+ model: Some("mineru".into()),
+ anchor: Some(serde_json::json!({ "page": page, "bbox": [10, 20, 300, 400] })),
+ }
+}
+
+async fn seed(pool: &PgPool) -> anyhow::Result<(Uuid, Uuid, Uuid)> {
+ let (org, ws, kb, doc) = (
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ Uuid::now_v7(),
+ );
+ sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'origin-test')")
+ .bind(org)
+ .execute(pool)
+ .await?;
+ sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'origin-test')")
+ .bind(ws)
+ .bind(org)
+ .execute(pool)
+ .await?;
+ sqlx::query(
+ "INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, 'origin-test')",
+ )
+ .bind(kb)
+ .bind(ws)
+ .execute(pool)
+ .await?;
+ sqlx::query(
+ "INSERT INTO documents (id, kb_id, filename, sha256, status)
+ VALUES ($1, $2, 'scan.pdf', $1::text, 'ready')",
+ )
+ .bind(doc)
+ .bind(kb)
+ .execute(pool)
+ .await?;
+ Ok((org, kb, doc))
+}
+
+type Row = (
+ Uuid,
+ String,
+ String,
+ Option<String>,
+ Option<serde_json::Value>,
+);
+
+async fn stored(pool: &PgPool, doc: Uuid) -> anyhow::Result<Vec<Row>> {
+ Ok(sqlx::query_as(
+ "SELECT id, text, origin, origin_model, anchor FROM chunks
+ WHERE document_id = $1 AND superseded_at IS NULL ORDER BY seq",
+ )
+ .bind(doc)
+ .fetch_all(pool)
+ .await?)
+}
+
+#[tokio::test]
+async fn a_chunk_keeps_its_provenance_and_is_claimed_by_it() -> anyhow::Result<()> {
+ let Some(url) = utopia_store::test_db::url() else {
+ return Ok(());
+ };
+ let pool = PgPool::connect(&url).await?;
+ let (org, kb, doc) = seed(&pool).await?;
+ let run = async {
+ let first = vec![
+ piece(0, "Rent is paid monthly.", Provenance::stated()),
+ piece(1, "The rent is 1,000 per month.", ocr(1)),
+ ];
+ utopia_store::documents::replace_chunks(&pool, kb, doc, &first).await?;
+ let rows = stored(&pool, doc).await?;
+ assert_eq!(rows[0].2, "stated");
+ assert_eq!(rows[0].4, None);
+ assert_eq!(rows[1].2, "ocr");
+ assert_eq!(rows[1].3.as_deref(), Some("mineru"));
+ assert_eq!(
+ rows[1].4.as_ref().map(|a| a["page"].clone()),
+ Some(serde_json::json!(1))
+ );
+
+ // 再处理一遍:出处没变的认领原行;同一句话换到另一页,是新的一块
+ let second = vec![
+ piece(0, "Rent is paid monthly.", Provenance::stated()),
+ piece(1, "The rent is 1,000 per month.", ocr(2)),
+ ];
+ utopia_store::documents::replace_chunks(&pool, kb, doc, &second).await?;
+ let again = stored(&pool, doc).await?;
+ assert_eq!(again[0].0, rows[0].0, "the stated chunk is claimed");
+ assert_ne!(
+ again[1].0, rows[1].0,
+ "a different anchor is a different chunk"
+ );
+ assert_eq!(
+ again[1].4.as_ref().map(|a| a["page"].clone()),
+ Some(serde_json::json!(2))
+ );
+
+ // 锚点形状:转写必须分得出说话人;原文不带锚点
+ let no_speaker = Provenance {
+ origin: Origin::Transcribed,
+ model: Some("whisper".into()),
+ anchor: Some(serde_json::json!({ "start_ms": 0, "end_ms": 900 })),
+ };
+ assert!(utopia_store::documents::replace_chunks(
+ &pool,
+ kb,
+ doc,
+ &[piece(0, "We ship in Q3.", no_speaker)]
+ )
+ .await
+ .is_err());
+ let stated_with_anchor = Provenance {
+ origin: Origin::Stated,
+ model: None,
+ anchor: Some(serde_json::json!({ "page": 1 })),
+ };
+ assert!(utopia_store::documents::replace_chunks(
+ &pool,
+ kb,
+ doc,
+ &[piece(0, "x", stated_with_anchor)]
+ )
+ .await
+ .is_err());
+ anyhow::Ok(())
+ }
+ .await;
+ let _ = sqlx::query("DELETE FROM knowledge_bases WHERE id = $1")
+ .bind(kb)
+ .execute(&pool)
+ .await;
+ let _ = sqlx::query("DELETE FROM organizations WHERE id = $1")
+ .bind(org)
+ .execute(&pool)
+ .await;
+ run
+}
diff --git a/crates/utopia-store/tests/a_search_reads_the_base_as_it_was.rs b/crates/utopia-store/tests/a_search_reads_the_base_as_it_was.rs
index 0fcf450f..7e082603 100644
--- a/crates/utopia-store/tests/a_search_reads_the_base_as_it_was.rs
+++ b/crates/utopia-store/tests/a_search_reads_the_base_as_it_was.rs
@@ -107,6 +107,7 @@ async fn seed(pool: &PgPool) -> anyhow::Result<Fixture> {
char_start: 0,
char_end: 51,
heading: None,
+ provenance: utopia_ingest::Provenance::stated(),
}];
utopia_store::documents::replace_chunks(pool, kb, doc, &pieces).await?;
let new_chunk: (Uuid,) = sqlx::query_as(
diff --git a/docs/decisions/0040-a-chunk-says-where-its-words-came-from.md b/docs/decisions/0040-a-chunk-says-where-its-words-came-from.md
index 60061e04..60cd8e42 100644
--- a/docs/decisions/0040-a-chunk-says-where-its-words-came-from.md
+++ b/docs/decisions/0040-a-chunk-says-where-its-words-came-from.md
@@ -1,8 +1,14 @@
# 0040 · A chunk says where its words came from
-- **Status**: Proposed · nothing built · cut 1 is the ledger shape (`chunks.origin`,
- `chunks.origin_model`, `chunks.anchor`, the packer rule, the confidence ceiling, the read
- contract); the media readers follow in the order at the end
+- **Status**: Cut 1 implemented (2026-09-15) · `chunks.origin` / `origin_model` / `anchor`
+ with the anchor shape checked (migration 0058), the packer never mixes provenances
+ (`chunk_segments`), the ceiling on described facts, origin in the evidence API, the MCP
+ changes feed and the RDF export (`utopia:evidenceOrigin`) · a file that needs a reader no
+ longer becomes garbage text: images and recordings are recognised by header or extension, a
+ PDF with an empty text layer is a scan, and without the model the document fails once with
+ `documents.reader_needed` and a `document.needs_reader` alert · revised 2026-09-15: scans and
+ images are read by a MinerU service instead of a Docling sidecar, and a transcript must label
+ speakers · readers are cuts 2 and 3
- **Written**: 2026-09-13 (conventions in the [README](README.md))
- **Related**: [0039](0039-a-chunk-is-what-extraction-sees.md) (#633, not merged yet) puts Docling
behind the block model as its cut 2 and leaves "evidence that points at a table cell or an image
@@ -126,6 +132,25 @@ is not decided here.
`/v1/audio/transcriptions` asking for segment timestamps, vision through a chat request with
image parts. OCR is the Docling sidecar of 0039's cut 2.
+*Revised 2026-09-15.* Two changes to this decision, made by the maintainer before cut 2:
+
+- **OCR is a MinerU service, not a Docling sidecar.** MinerU detects the page layout first
+ and then recognises each region, and returns Markdown plus a content list in reading order
+ with `page_idx` and a `bbox` per block. That is exactly the `{"page", "bbox"}` anchor above.
+ Its VLM is not interchangeable with a generic vision model: the model behind `mineru-api`
+ speaks MinerU's own two-step protocol, and a generic model asked to "transcribe this page"
+ gives neither the layout nor the boxes. So the setting is `ocr_*` (the service URL and key),
+ not a vision chat model; a generic vision model stays the reader for cut 4, descriptions.
+- **A transcript must say who spoke** (the first open question below, now decided). A
+ transcription endpoint that returns segments without speaker labels is treated like a
+ missing one: the recording is not read, and the alert says why. The database refuses a
+ `transcribed` anchor without `speaker`.
+
+Where a setting is empty, "fails with an alert" means **degrades**. The file is kept, and the
+document stops at `failed` with `documents.reader_needed` set to the missing reader. It raises
+one `document.needs_reader` alert and is not retried. Saving the setting queues those documents
+again.
+
These are not a reuse of the chat model. A recording of a board meeting or a scanned contract is
more sensitive than a paragraph of text, and a deployment will reasonably keep transcription and
OCR on local models while chat goes to a hosted one. One setting would force the most sensitive
@@ -170,7 +195,8 @@ the recording at `start_ms` — is a separate cut after the capability, not part
## Open
-- **Who said it.** Whisper-compatible endpoints do not separate speakers. Without speakers, "Zhang
+- ~~**Who said it.**~~ *Decided 2026-09-15: a transcript without speaker labels is refused (decision 5).*
+ Whisper-compatible endpoints do not separate speakers. Without speakers, "Zhang
San said he would deliver in Q3" and "Li Si said Zhang San would deliver in Q3" can be the same
transcript line, and a meeting ingested that way attributes commitments to the wrong people —
worse than extracting nothing. Cut 3 does not ship until this is decided: require an endpoint
diff --git a/docs/decisions/README.md b/docs/decisions/README.md
index 5f372dc8..bb2ff4ff 100644
--- a/docs/decisions/README.md
+++ b/docs/decisions/README.md
@@ -63,7 +63,7 @@ The test for writing one: if someone (including us) looks at a piece of code in
| 0037 | [A relation carries its own attributes](0037-a-relation-carries-its-own-attributes.md) | Written · cut 1 in progress: qualifier tables (0049), declaration, extraction, panel, export · entity-valued qualifier reserved, conflict-as-two-rows and the canvas label next |
| 0038 | [The interface has a light side](0038-the-interface-has-a-light-side.md) | Implemented (#599) · `data-theme` on `<html>`, chosen by the reader and stored in the browser, `system` resolved to one of the two before first paint · a second token block in `styles.css` and not one rule rewritten: alpha stays, the triplet swaps · the canvas reads its colours from the tokens through one reader file and rebuilds on a switch; edges are flattened over the ground on light because the WebGL shader can only brighten · a `raw-colour` guard rule over `.ts` and `.tsx` · the entity palette is the same in both themes; a paper-tuned set is open |
| 0039 | [A chunk is what extraction sees](0039-a-chunk-is-what-extraction-sees.md) | Implemented, cut 1 · the parser's Markdown is read into blocks and packed by a token budget (1000, cl100k) · a table travels with its caption and header and a continuation repeats both · headings become a breadcrumb and fill `chunks.heading` · a page-break rule is not a boundary and a table it split is joined back · not measured on its own: the budget · cut 2: an external parser behind the block model, evidence that names a cell or a region |
-| 0040 | [A chunk says where its words came from](0040-a-chunk-says-where-its-words-came-from.md) | Proposed · nothing built · a chunk carries an **origin** (`stated`, `ocr`, `transcribed`, `described`) and the model that produced it, and an **anchor** back into the original bytes (a page and region, a time range, an image inside the file) — decided before any media reader, because a transcript stored without its times can never be tied to the recording again · the packer never mixes origins in a chunk · facts from a description enter below `AUTO_CLOSE_MIN_CONFIDENCE`, so a misread chart opens a conflict instead of closing a correct fact · per-modality model settings, media reading as a resumable job, origin and anchor in the API, MCP and RDF · cuts: ledger shape, scans via Docling, recordings, descriptions, video |
+| 0040 | [A chunk says where its words came from](0040-a-chunk-says-where-its-words-came-from.md) | Cut 1 implemented (the ledger shape, and a file that needs a reader degrades with an alert instead of becoming garbage text) · scans and images through a MinerU service, recordings only with speaker labels (revised 2026-09-15) · a chunk carries an **origin** (`stated`, `ocr`, `transcribed`, `described`) and the model that produced it, and an **anchor** back into the original bytes (a page and region, a time range, an image inside the file) — decided before any media reader, because a transcript stored without its times can never be tied to the recording again · the packer never mixes origins in a chunk · facts from a description enter below `AUTO_CLOSE_MIN_CONFIDENCE`, so a misread chart opens a conflict instead of closing a correct fact · per-modality model settings, media reading as a resumable job, origin and anchor in the API, MCP and RDF · cuts: ledger shape, scans via Docling, recordings, descriptions, video |
| 0041 | [A name is a claim about an entity](0041-a-name-is-a-claim-about-an-entity.md) | Cut 0 built (identity bench) · cut 1 implemented (#670): names are value facts on `known_as`, the extractor reports other names, a shared name goes to the adjudicator; forward/reverse F1 0.43/0.54 → 0.68/0.68 · cuts 2–4 (name vectors and neighbours, evidence decides, re-evaluation) not started |
| 0042 | [The chat loop is a runner with hooks](0042-the-chat-loop-is-a-runner-with-hooks.md) | Implemented (#548) · the loop is rig's runner and every policy is a hook with a typed result · the wire stays `LlmClient` behind `RigModel` · a turn cannot end before a tool has run, `no_evidence_needed` is the exit for questions not about the base · RAG fallback only on a 400/422 to the first request with tools · an empty reply is asked again once · the skip rate is the model's (DeepSeek-V3 1–3 of 12, Qwen2.5-72B 0) and recorded, not prevented |
diff --git a/migrations/0058_a_chunk_says_where_its_words_came_from.sql b/migrations/0058_a_chunk_says_where_its_words_came_from.sql
new file mode 100644
index 00000000..fd285932
--- /dev/null
+++ b/migrations/0058_a_chunk_says_where_its_words_came_from.sql
@@ -0,0 +1,32 @@
+-- 一块文字从哪来(0040):原文写的、扫描页上认出来的、录音里转出来的、模型看图说的。
+--
+-- 分块之后的一切只吃 `chunks.text`,而一段转写、一段对柱状图的描述,与文件里的一句话进库的
+-- 样子一模一样,事后没有任何一列能把它们分开。来源放在分块上:引用这一块的证据共享它的来源,
+-- 抽取读的也是这一块。现有的块都是原文,这是真的,默认值就是它。
+ALTER TABLE chunks
+ ADD COLUMN origin TEXT NOT NULL DEFAULT 'stated'
+ CHECK (origin IN ('stated', 'ocr', 'transcribed', 'described')),
+ -- 读出这段文字的引擎或模型;原文为空。留着它,好在更好的模型出来时重读,
+ -- 也好按模型数误读
+ ADD COLUMN origin_model TEXT,
+ -- 指回原文件字节的位置,形状由来源定。原文不需要(char_start / char_end 已经给出);
+ -- 扫描页记页码(引擎给了就带框);录音记起止毫秒与说话人——转写必须分得出说话人,
+ -- 分不出谁说的,承诺会记到错的人头上;图片描述记它在哪一页、哪一张图
+ ADD COLUMN anchor JSONB;
+
+ALTER TABLE chunks ADD CONSTRAINT chunks_anchor_matches_origin CHECK (
+ (origin = 'stated' AND anchor IS NULL AND origin_model IS NULL)
+ OR (origin = 'ocr' AND anchor ? 'page')
+ OR (origin = 'transcribed' AND anchor ? 'start_ms' AND anchor ? 'end_ms' AND anchor ? 'speaker')
+ OR (origin = 'described' AND anchor IS NOT NULL)
+);
+
+-- 这份文件的字要靠哪一种模型读,而那种模型还没配(0040)。
+--
+-- 扫描件、图片、录音在没配读取模型时不再解出乱码、也不再重试三次:文件留着,文档停在
+-- failed,这一列记下缺的是哪一种。配上之后按这一列把它们重新排进处理队列;读成功了清空
+ALTER TABLE documents
+ ADD COLUMN reader_needed TEXT CHECK (reader_needed IN ('ocr', 'transcribe'));
+
+CREATE INDEX documents_reader_needed_idx ON documents (reader_needed)
+ WHERE reader_needed IS NOT NULL;
diff --git a/web/src/api.ts b/web/src/api.ts
index b2e20a84..77fc32f7 100644
--- a/web/src/api.ts
+++ b/web/src/api.ts
@@ -175,6 +175,8 @@ export interface Doc {
deleted_at: string | null;
/** 真删过:内容没了,回不来 */
purged_at: string | null;
+ /** 这份文件的字要靠哪一种模型读、而那种模型没配(0040):ocr / transcribe;文档此时是 failed */
+ reader_needed: "ocr" | "transcribe" | null;
created_at: string;
}
diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts
index cfb468ac..77affc1b 100644
--- a/web/src/i18n/en.ts
+++ b/web/src/i18n/en.ts
@@ -245,6 +245,10 @@ export const en = {
title: "The model account cannot pay for requests",
hint: "Extraction and embedding are stopped and will not resume on their own. Top up the account, or set an endpoint that can serve in Administration → Models.",
},
+ "document.needs_reader": {
+ title: "A file needs a model to be read",
+ hint: "Scans and images need a document-reading service, and recordings need a transcription model that labels speakers. Neither is configured on this deployment. The file is kept, and nothing was read from it.",
+ },
"governance.tripped": {
title: "The agent stopped deciding on its own",
hint: "Two of its merges were reverted within seven days, so the switch went off. Look at what it did under Review → Agent, then turn it back on in the base settings when you want it to resume.",
diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts
index 42e7742d..bb76c523 100644
--- a/web/src/i18n/zh.ts
+++ b/web/src/i18n/zh.ts
@@ -209,6 +209,10 @@ export const zh: Strings = {
title: "映射探索没有提出任何口径",
hint: "模型读了挂载的库表结构,但没找到可提的指标或维度。到「数据映射 > 数据源」刷新结构、给列加注释,再探索一次。",
},
+ "document.needs_reader": {
+ title: "有文件要靠模型才读得出来",
+ hint: "扫描件和图片要配文档识别服务,录音要配能分出说话人的转写模型,这个部署都还没配。文件已经留着,但里面什么都没读出来。",
+ },
"governance.tripped": {
title: "agent 停止自动裁决了",
hint: "七天内它的合并被撤回了两次,开关已自动关掉。到「审核 > Agent」看看它做了什么,想让它接着跑就在库设置里再打开。",
diff --git a/web/src/pages/Library.tsx b/web/src/pages/Library.tsx
index 7c039370..3a8a666f 100644
--- a/web/src/pages/Library.tsx
+++ b/web/src/pages/Library.tsx
@@ -650,7 +650,7 @@ export function Library() {
type="file"
multiple
hidden
- accept=".pdf,.docx,.xlsx,.xls,.ods,.pptx,.md,.txt,.html,.htm,.csv,.tsv,.json,.yaml,.yml,.xml,.log"
+ accept=".pdf,.docx,.xlsx,.xls,.ods,.pptx,.md,.txt,.html,.htm,.csv,.tsv,.json,.yaml,.yml,.xml,.log,.png,.jpg,.jpeg,.webp,.tif,.tiff,.bmp,.gif,.mp3,.wav,.m4a,.flac,.ogg,.opus,.aac,.amr"
onChange={(e) => e.target.files?.length && upload.mutate(e.target.files)}
/>