Skip to content

World's first open-source enterprise world model.

A capped spreadsheet or CSV says what was left out (#1022) (#1043)

Coderfue50b5b2f123e
diff --git a/crates/utopia-ingest/src/lib.rs b/crates/utopia-ingest/src/lib.rs
index df77ff82..3730dfee 100644
--- a/crates/utopia-ingest/src/lib.rs
+++ b/crates/utopia-ingest/src/lib.rs
@@ -179,10 +179,23 @@ pub use parsers::plain_text as decode_text;
pub use provenance::{Origin, Provenance, Segment};
pub use reading::Reading;
-/// 解析产物:纯文本 + 可选结构信息。
+/// 解析产物:纯文本 + 可选结构信息和可观测的解析警告。
#[derive(Debug)]
pub struct ParsedDoc {
pub text: String,
+ pub warnings: Vec<ParseWarning>,
+}
+
+/// 解析器到了安全上限,正文有意只保留一部分;`detail` 说明省略的范围。
+#[derive(Debug, Clone, PartialEq)]
+pub struct ParseWarning {
+ pub kind: &'static str,
+ pub detail: serde_json::Value,
+}
+
+impl ParseWarning {
+ pub const SPREADSHEET_ROWS_TRUNCATED: &'static str = "spreadsheet.rows_truncated";
+ pub const CSV_RECORDS_TRUNCATED: &'static str = "csv.records_truncated";
}
/// 读出字要靠哪一种模型(0040)
@@ -317,7 +330,7 @@ pub fn parse(filename: &str, bytes: &[u8]) -> anyhow::Result<ParsedDoc> {
// 魔数探测优先于扩展名(扩展名可能撒谎)
let kind = infer::get(bytes).map(|t| t.extension()).unwrap_or("");
- let text = match (kind, ext.as_str()) {
+ let (text, warnings) = match (kind, ext.as_str()) {
("pdf", _) | (_, "pdf") => {
let text = parsers::pdf(bytes)?;
// 文本层是空的:扫描件,字在图里
@@ -328,12 +341,12 @@ pub fn parse(filename: &str, bytes: &[u8]) -> anyhow::Result<ParsedDoc> {
}
.into());
}
- text
+ (text, Vec::new())
}
- ("docx", _) | (_, "docx") => parsers::docx(bytes)?,
+ ("docx", _) | (_, "docx") => (parsers::docx(bytes)?, Vec::new()),
("xlsx", _) | (_, "xlsx") | (_, "xls") | (_, "ods") => parsers::spreadsheet(bytes)?,
- ("pptx", _) | (_, "pptx") => parsers::pptx(bytes)?,
- (_, "html") | (_, "htm") => parsers::html(bytes)?,
+ ("pptx", _) | (_, "pptx") => (parsers::pptx(bytes)?, Vec::new()),
+ (_, "html") | (_, "htm") => (parsers::html(bytes)?, Vec::new()),
(_, "csv") | (_, "tsv") => parsers::csv_text(bytes, ext == "tsv")?,
// md/json/yaml/xml/log/txt 及一切未识别格式:按文本解码(编码探测覆盖 GBK 等)。
// 解码之前先看是不是二进制:老式 .doc、压缩包、可执行文件解出来是乱码,
@@ -343,14 +356,14 @@ pub fn parse(filename: &str, bytes: &[u8]) -> anyhow::Result<ParsedDoc> {
Unreadable("This file is not in a format that can be read as text".into()).into(),
)
}
- _ => parsers::plain_text(bytes),
+ _ => (parsers::plain_text(bytes), Vec::new()),
};
let text = normalize(&text);
if text.trim().is_empty() {
return Err(Unreadable("No text could be extracted from this file".into()).into());
}
- Ok(ParsedDoc { text })
+ Ok(ParsedDoc { text, warnings })
}
/// 压缩连续空白行,统一换行符。
diff --git a/crates/utopia-ingest/src/parsers.rs b/crates/utopia-ingest/src/parsers.rs
index 1194aac6..8070665b 100644
--- a/crates/utopia-ingest/src/parsers.rs
+++ b/crates/utopia-ingest/src/parsers.rs
@@ -6,6 +6,9 @@ use quick_xml::Reader;
use std::io::{Cursor, Read, Write};
use std::process::Command;
+const SPREADSHEET_ROW_LIMIT: usize = 2_000;
+const CSV_RECORD_LIMIT: usize = 10_000;
+
/// 文本解码:chardetng 探测编码(覆盖 GBK/GB18030/BIG5 等中文常见编码)。
pub fn plain_text(bytes: &[u8]) -> String {
use chardetng::{EncodingDetector, Iso2022JpDetection, Utf8Detection};
@@ -754,11 +757,12 @@ fn pptx_order(
}
/// xlsx / xls / ods: calamine 全格式读取,每 sheet 输出制表符表格(限前 2000 行)。
-pub fn spreadsheet(bytes: &[u8]) -> anyhow::Result<String> {
+pub fn spreadsheet(bytes: &[u8]) -> anyhow::Result<(String, Vec<crate::ParseWarning>)> {
use calamine::Reader as _;
let mut workbook = calamine::open_workbook_auto_from_rs(Cursor::new(bytes.to_vec()))
.context("Failed to open spreadsheet")?;
let mut out = String::new();
+ let mut warnings = Vec::new();
for sheet_name in workbook.sheet_names() {
let Ok(range) = workbook.worksheet_range(&sheet_name) else {
continue;
@@ -777,6 +781,19 @@ pub fn spreadsheet(bytes: &[u8]) -> anyhow::Result<String> {
_ => Vec::new(),
};
out.push_str(&format!("\n# Sheet: {sheet_name}\n\n"));
+ let rows_total = range.height();
+ if rows_total > SPREADSHEET_ROW_LIMIT {
+ let rows_read = SPREADSHEET_ROW_LIMIT;
+ warnings.push(crate::ParseWarning {
+ kind: crate::ParseWarning::SPREADSHEET_ROWS_TRUNCATED,
+ detail: serde_json::json!({
+ "sheet": sheet_name.clone(),
+ "rows_read": rows_read,
+ "rows_total": rows_total,
+ "rows_omitted": rows_total - rows_read,
+ }),
+ });
+ }
// 每张表按网格渲染成带列头的 Markdown 表;一格字都没有的表退回制表符分隔
let (rows, headers) = spreadsheet_grid(&range, &merges);
match crate::table::render_grid_with_headers(&rows, headers) {
@@ -793,7 +810,7 @@ pub fn spreadsheet(bytes: &[u8]) -> anyhow::Result<String> {
}
}
}
- Ok(out)
+ Ok((out, warnings))
}
#[derive(Clone)]
@@ -812,7 +829,7 @@ fn spreadsheet_grid(
let Some((start_row, start_col)) = range.start() else {
return (Vec::new(), None);
};
- let height = range.height().min(2000);
+ let height = range.height().min(SPREADSHEET_ROW_LIMIT);
let width = range.width();
if height == 0 || width == 0 {
return (Vec::new(), None);
@@ -971,7 +988,7 @@ pub fn html(bytes: &[u8]) -> anyhow::Result<String> {
Ok(crate::html::page_to_markdown(&plain_text(bytes), None)?)
}
-pub fn csv_text(bytes: &[u8], tsv: bool) -> anyhow::Result<String> {
+pub fn csv_text(bytes: &[u8], tsv: bool) -> anyhow::Result<(String, Vec<crate::ParseWarning>)> {
let decoded = plain_text(bytes);
let mut reader = csv::ReaderBuilder::new()
.delimiter(if tsv { b'\t' } else { b',' })
@@ -979,15 +996,28 @@ pub fn csv_text(bytes: &[u8], tsv: bool) -> anyhow::Result<String> {
.has_headers(false)
.from_reader(decoded.as_bytes());
let mut rows: Vec<GridRow> = Vec::new();
- for (i, record) in reader.records().enumerate() {
- if i >= 10_000 {
- break;
- }
+ let mut records_total = 0usize;
+ for record in reader.records() {
let record = record?;
- rows.push(record.iter().map(|s| (s.to_string(), 1, 0)).collect());
+ records_total += 1;
+ if rows.len() < CSV_RECORD_LIMIT {
+ rows.push(record.iter().map(|s| (s.to_string(), 1, 0)).collect());
+ }
+ }
+ let mut warnings = Vec::new();
+ if records_total > CSV_RECORD_LIMIT {
+ let records_read = rows.len();
+ warnings.push(crate::ParseWarning {
+ kind: crate::ParseWarning::CSV_RECORDS_TRUNCATED,
+ detail: serde_json::json!({
+ "records_read": records_read,
+ "records_total": records_total,
+ "records_omitted": records_total - records_read,
+ }),
+ });
}
// 第一条记录是列头(csv 的惯例);渲染不出表时退回竖线分隔的行
- Ok(match crate::table::render_grid(&rows, true) {
+ let text = match crate::table::render_grid(&rows, true) {
Some(md) => md + "\n",
None => {
rows.iter()
@@ -1001,7 +1031,8 @@ pub fn csv_text(bytes: &[u8], tsv: bool) -> anyhow::Result<String> {
.join("\n")
+ "\n"
}
- })
+ };
+ Ok((text, warnings))
}
// ---- 工具 ----
@@ -1251,7 +1282,9 @@ mod tests {
/// csv 的第一条记录是列头,哪怕列头是年份
#[test]
fn a_csv_is_a_table_whose_first_record_is_the_header() {
- let text = csv_text(b"Item,2025,2024\nRevenue,10,8\nCost,4,3\n", false).unwrap();
+ let text = csv_text(b"Item,2025,2024\nRevenue,10,8\nCost,4,3\n", false)
+ .unwrap()
+ .0;
assert_eq!(
text,
"| Item | 2025 | 2024 |\n| --- | --- | --- |\n| Revenue | 10 | 8 |\n| Cost | 4 | 3 |\n"
@@ -1268,7 +1301,8 @@ mod tests {
",
false,
)
- .unwrap();
+ .unwrap()
+ .0;
assert!(text.starts_with("| no | name | score |"), "{text}");
assert!(text.contains("| 1 | | |"), "{text}");
assert!(text.contains("| 2 | Ada | 9 |"), "{text}");
@@ -1277,7 +1311,9 @@ mod tests {
/// 一格数字都没有的 csv 也是表:标签列是最左那格
#[test]
fn a_csv_of_words_is_still_a_table() {
- let text = csv_text(b"name,role\nAda,engineer\nGrace,admiral\n", false).unwrap();
+ let text = csv_text(b"name,role\nAda,engineer\nGrace,admiral\n", false)
+ .unwrap()
+ .0;
assert!(
text.starts_with("| name | role |\n| --- | --- |\n| Ada | engineer |"),
"{text}"
diff --git a/crates/utopia-ingest/tests/spreadsheet_truncation.rs b/crates/utopia-ingest/tests/spreadsheet_truncation.rs
new file mode 100644
index 00000000..e4f8b7cd
--- /dev/null
+++ b/crates/utopia-ingest/tests/spreadsheet_truncation.rs
@@ -0,0 +1,115 @@
+//! A spreadsheet or CSV that hits a parser cap must leave a warning behind;
+//! otherwise the document reads as if the file were complete.
+
+use std::fmt::Write as _;
+use std::io::{Cursor, Write};
+
+fn workbook(data_rows: usize) -> Vec<u8> {
+ let mut sheet_data = String::from(
+ r#"<row r="1"><c r="A1" t="inlineStr"><is><t>id</t></is></c><c r="B1" t="inlineStr"><is><t>name</t></is></c></row>"#,
+ );
+ for id in 1..=data_rows {
+ let row = id + 1;
+ sheet_data.push_str(&format!(
+ r#"<row r="{row}"><c r="A{row}"><v>{id}</v></c><c r="B{row}" t="inlineStr"><is><t>customer-{id}</t></is></c></row>"#
+ ));
+ }
+ let worksheet = format!(
+ r#"<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData>{sheet_data}</sheetData></worksheet>"#
+ );
+ let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new()));
+ for (part, xml) in [
+ ("[Content_Types].xml", r#"<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/><Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/><Override PartName="/xl/worksheets/sheet1.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/></Types>"#.to_string()),
+ ("_rels/.rels", r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/></Relationships>"#.to_string()),
+ ("xl/workbook.xml", r#"<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><sheets><sheet name="Ledger" sheetId="1" r:id="rId1"/></sheets></workbook>"#.to_string()),
+ ("xl/_rels/workbook.xml.rels", r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet1.xml"/></Relationships>"#.to_string()),
+ ("xl/worksheets/sheet1.xml", worksheet),
+ ] {
+ zip.start_file(part, zip::write::SimpleFileOptions::default())
+ .unwrap();
+ zip.write_all(xml.as_bytes()).unwrap();
+ }
+ zip.finish().unwrap().into_inner()
+}
+
+fn csv(records: usize) -> String {
+ let mut csv = String::from("id,name\n");
+ for id in 1..=records {
+ writeln!(csv, "{id},customer-{id}").unwrap();
+ }
+ csv
+}
+
+#[test]
+fn a_spreadsheet_past_the_row_cap_reports_the_rows_it_left_out() {
+ let parsed = utopia_ingest::parse("ledger.xlsx", &workbook(2_000)).unwrap();
+
+ assert!(
+ parsed.text.contains("| 1999 | customer-1999 |"),
+ "the last row within the cap is still read:\n{}",
+ parsed.text
+ );
+ assert!(
+ !parsed.text.contains("customer-2000"),
+ "the first row past the cap is not presented as read"
+ );
+ assert_eq!(parsed.warnings.len(), 1, "{:?}", parsed.warnings);
+ let warning = &parsed.warnings[0];
+ assert_eq!(warning.kind, "spreadsheet.rows_truncated");
+ assert_eq!(
+ warning.detail,
+ serde_json::json!({
+ "sheet": "Ledger",
+ "rows_read": 2_000,
+ "rows_total": 2_001,
+ "rows_omitted": 1,
+ })
+ );
+}
+
+#[test]
+fn a_spreadsheet_at_the_row_cap_has_no_warning() {
+ let parsed = utopia_ingest::parse("ledger.xlsx", &workbook(1_999)).unwrap();
+
+ assert!(
+ parsed.text.contains("| 1999 | customer-1999 |"),
+ "the last data row is inside the cap"
+ );
+ assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings);
+}
+
+#[test]
+fn a_csv_past_the_record_cap_reports_the_records_it_left_out() {
+ let parsed = utopia_ingest::parse("customers.csv", csv(10_000).as_bytes()).unwrap();
+
+ assert!(
+ parsed.text.contains("| 9999 | customer-9999 |"),
+ "the last record within the cap is still read"
+ );
+ assert!(
+ !parsed.text.contains("customer-10000"),
+ "the first record past the cap is not presented as read"
+ );
+ assert_eq!(parsed.warnings.len(), 1, "{:?}", parsed.warnings);
+ let warning = &parsed.warnings[0];
+ assert_eq!(warning.kind, "csv.records_truncated");
+ assert_eq!(
+ warning.detail,
+ serde_json::json!({
+ "records_read": 10_000,
+ "records_total": 10_001,
+ "records_omitted": 1,
+ })
+ );
+}
+
+#[test]
+fn a_csv_at_the_record_cap_has_no_warning() {
+ let parsed = utopia_ingest::parse("customers.csv", csv(9_999).as_bytes()).unwrap();
+
+ assert!(
+ parsed.text.contains("| 9999 | customer-9999 |"),
+ "the last record is inside the cap"
+ );
+ assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings);
+}
diff --git a/crates/utopia-server/src/alerting.rs b/crates/utopia-server/src/alerting.rs
index 29dc1da1..40f2d722 100644
--- a/crates/utopia-server/src/alerting.rs
+++ b/crates/utopia-server/src/alerting.rs
@@ -56,6 +56,51 @@ pub async fn observe_job_failure(
state.emit_alert();
}
+/// 解析器只读了一部分正文:保留可用的前缀,但把截断范围写进告警详情。
+/// 名字存一份——文档删掉之后这条告警还读得懂。
+pub async fn observe_document_truncated(
+ state: &AppState,
+ kb_id: uuid::Uuid,
+ document_id: uuid::Uuid,
+ filename: &str,
+ warnings: &[utopia_ingest::ParseWarning],
+) {
+ if warnings.is_empty() {
+ return;
+ }
+ if let Err(e) = alerts::raise(
+ &state.pool,
+ alerts::NewAlert {
+ kb_id: Some(kb_id),
+ severity: "warning",
+ kind: alerts::kind::DOCUMENT_CONTENTS_TRUNCATED,
+ min_role: Role::Editor,
+ subject_type: Some("document"),
+ subject_id: Some(document_id),
+ detail: document_truncation_detail(filename, warnings),
+ },
+ )
+ .await
+ {
+ tracing::warn!(error = %e, "上报告警失败");
+ return;
+ }
+ state.emit_alert();
+}
+
+fn document_truncation_detail(
+ filename: &str,
+ warnings: &[utopia_ingest::ParseWarning],
+) -> serde_json::Value {
+ serde_json::json!({
+ "name": filename,
+ "warnings": warnings
+ .iter()
+ .map(|warning| serde_json::json!({ "kind": warning.kind, "detail": warning.detail }))
+ .collect::<Vec<_>>(),
+ })
+}
+
/// 一份文件的字要靠没配的那种模型读(0040):报一条库级告警,名字存进告警里——
/// 文件删了告警还读得懂
pub async fn observe_document_needs_reader(
@@ -190,6 +235,7 @@ pub async fn observe_schema_sync_failure(
#[cfg(test)]
mod tests {
use super::alert_for;
+ use utopia_ingest::ParseWarning;
use utopia_llm::{OutOfCredit, RateLimited};
use utopia_store::alerts::kind;
@@ -289,4 +335,31 @@ mod tests {
fn an_ordinary_failure_still_gets_its_retries() {
assert!(!super::hopeless(&anyhow::anyhow!("结果解析失败")));
}
+
+ #[test]
+ fn a_truncation_alert_says_what_was_left_out() {
+ let warning = ParseWarning {
+ kind: ParseWarning::CSV_RECORDS_TRUNCATED,
+ detail: serde_json::json!({
+ "records_read": 10_000,
+ "records_total": 10_001,
+ "records_omitted": 1,
+ }),
+ };
+
+ assert_eq!(
+ super::document_truncation_detail("customers.csv", &[warning]),
+ serde_json::json!({
+ "name": "customers.csv",
+ "warnings": [{
+ "kind": "csv.records_truncated",
+ "detail": {
+ "records_read": 10_000,
+ "records_total": 10_001,
+ "records_omitted": 1,
+ },
+ }],
+ })
+ );
+ }
}
diff --git a/crates/utopia-server/src/pipeline.rs b/crates/utopia-server/src/pipeline.rs
index fe9e6200..dfd1a1bb 100644
--- a/crates/utopia-server/src/pipeline.rs
+++ b/crates/utopia-server/src/pipeline.rs
@@ -124,8 +124,9 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
source_kind(state, doc.source_id).await?.as_deref() == Some("statements");
// 2. 分块 + 入库
- let (text, pieces) = match parsed {
+ let (text, pieces, warnings) = match parsed {
Ok(parsed) => {
+ let warnings = parsed.warnings;
// 解析出来的正文可能夹着 NUL(PDF 文本层常见),入库之前剥掉(#611)——与记忆
// 那条路共用 `utopia_core::without_nul`(#665)。剥必须在算长度、分块之前:之后的
// text_len、分块偏移、全文索引、嵌入读的都是这一份,彼此才对得上
@@ -144,7 +145,7 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
} else {
utopia_ingest::chunk_with_budget(&text, state.chunk_tokens)
};
- (text, pieces)
+ (text, pieces, warnings)
}
// 没有文本层的扫描件、图片:工作区配了版面识别服务就交给它读(0040 第二刀),
// 按页分段切块,每块记着页码和框。录音交给会标说话人的转写模型(第三刀),
@@ -175,7 +176,7 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
.into());
}
let pieces = reading.chunk(state.chunk_tokens);
- (reading.text, pieces)
+ (reading.text, pieces, Vec::new())
}
};
let text_len = text.chars().count() as i32;
@@ -194,6 +195,17 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
return Ok(());
};
let chunk_count = chunk_pairs.len() as i32;
+ // Only the current version gets an alert: if this read was superseded, the warning is stale.
+ if !warnings.is_empty() {
+ crate::alerting::observe_document_truncated(
+ state,
+ doc.kb_id,
+ document_id,
+ &doc.filename,
+ &warnings,
+ )
+ .await;
+ }
// 3. 全文索引(Tantivy)
utopia_store::documents::set_status(&state.pool, document_id, "indexing").await?;
diff --git a/crates/utopia-server/src/pipeline_tests.rs b/crates/utopia-server/src/pipeline_tests.rs
index 499957bf..2f4a31de 100644
--- a/crates/utopia-server/src/pipeline_tests.rs
+++ b/crates/utopia-server/src/pipeline_tests.rs
@@ -21,6 +21,7 @@
//!
//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆,绝不碰已有的库。
+use std::fmt::Write as _;
use std::sync::{Arc, Mutex};
use std::time::{Duration, Instant};
use utopia_core::models::Proposer;
@@ -425,6 +426,43 @@ async fn a_file_that_needs_a_reader_waits_and_says_so() -> anyhow::Result<()> {
f.cleanup().await
}
+/// 解析器到达安全上限时,文档照旧可用,但消息中心必须明说只读了一部分。
+#[tokio::test]
+async fn a_truncated_csv_keeps_the_document_ready_and_reports_the_omitted_records(
+) -> anyhow::Result<()> {
+ let Some(mut f) = fixture(FakeEmbed::new(Duration::ZERO)).await? else {
+ return Ok(());
+ };
+ // The assertion is about parser observability, not chunking. One large chunk keeps the
+ // database-backed test focused and cheap.
+ f.state.chunk_tokens = 1_000_000;
+ let mut csv = String::from("id,name\n");
+ for id in 1..=10_000 {
+ writeln!(csv, "{id},customer-{id}").unwrap();
+ }
+ let doc = f
+ .document_with_bytes("customers.csv", csv.as_bytes())
+ .await?;
+
+ super::process_document(&f.state, doc).await?;
+
+ let row = utopia_store::documents::get(&f.pool, doc).await?;
+ assert_eq!(row.status, "ready", "the readable prefix remains available");
+ let alerts = f.alerts("document.contents_truncated").await?;
+ assert_eq!(alerts.len(), 1, "{alerts:?}");
+ assert_eq!(alerts[0]["name"], "customers.csv");
+ assert_eq!(alerts[0]["warnings"][0]["kind"], "csv.records_truncated");
+ assert_eq!(
+ alerts[0]["warnings"][0]["detail"],
+ serde_json::json!({
+ "records_read": 10_000,
+ "records_total": 10_001,
+ "records_omitted": 1,
+ })
+ );
+ f.cleanup().await
+}
+
/// 假的 `mineru-api`:交一次拿到 `t-n`;问状态前 `processing_polls` 次说还在读,之后说读完;
/// `fail` 时说读失败。记下交了几次、每次带了什么
#[derive(Clone, Default)]
diff --git a/crates/utopia-store/src/alerts.rs b/crates/utopia-store/src/alerts.rs
index 034e7c32..8f2e7cef 100644
--- a/crates/utopia-store/src/alerts.rs
+++ b/crates/utopia-store/src/alerts.rs
@@ -63,6 +63,9 @@ pub mod kind {
/// 在管理页「模型」里配上之后会自己重新处理。没有这一条,传上来的扫描件只是悄悄地
/// 什么都没有——0005 里一直空着的 `document.no_text_layer` 就是它
pub const DOCUMENT_NEEDS_READER: &str = "document.needs_reader";
+ /// 库级:解析器的安全上限到了,正文只是文件的一部分。`severity = warning`,
+ /// `min_role = editor`:文件已经可搜,但搜索和回答不包含被省略的那一段。
+ pub const DOCUMENT_CONTENTS_TRUNCATED: &str = "document.contents_truncated";
}
/// 一次故障。打包成结构体不只是为了参数个数——调用点写 `severity: "error"`
diff --git a/web/src/api.ts b/web/src/api.ts
index 8dc3212b..4f4a9f49 100644
--- a/web/src/api.ts
+++ b/web/src/api.ts
@@ -1580,7 +1580,15 @@ export type AlertGroup = {
/** 跟 latest_at 一起圈出这一组,标已读时原样发回去 */
earliest_at: string;
/** 明细,最多几条,新的在前 */
- lines: { name?: string; error?: string; job?: string }[];
+ lines: {
+ name?: string;
+ error?: string;
+ job?: string;
+ warnings?: {
+ kind: string;
+ detail: Record<string, unknown>;
+ }[];
+ }[];
};
export const api = {
diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts
index 1d09c166..674d4963 100644
--- a/web/src/i18n/en.ts
+++ b/web/src/i18n/en.ts
@@ -281,6 +281,10 @@ export const en = {
noMatch: "Nothing matches",
andMore: (n: number) => `and ${n} more`,
system: "System",
+ partialRows: (sheet: string, read: number, total: number) =>
+ `${sheet}: read ${read} of ${total} rows`,
+ partialRecords: (read: number, total: number) =>
+ `read ${read} of ${total} records`,
// kind → 一句说清出了什么事。第二句说该做什么——这才是告警比日志多出来的东西。
// **一条告警就是一次故障**,所以标题里没有数量
kinds: {
@@ -312,6 +316,10 @@ export const en = {
title: "A file needs a model to be read",
hint: "Scans and images need a document-reading service, and recordings need a transcription model that labels speakers. Each line says what was missing. The file is kept and nothing was read from it yet; it is read as soon as the reader is saved under Administration → Models.",
},
+ "document.contents_truncated": {
+ title: "A file was only partly read",
+ hint: "A parser safety cap was reached. The line says which sheet or record range was left out; search and answers do not contain the omitted part.",
+ },
"governance.tripped": {
title: "The agent stopped deciding on its own",
hint: "Two of its merges were reverted within seven days, so the switch went off. Look at what it did under Review → Agent, then turn it back on in the base settings when you want it to resume.",
diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts
index 166a0efb..6de81b81 100644
--- a/web/src/i18n/zh.ts
+++ b/web/src/i18n/zh.ts
@@ -251,6 +251,10 @@ export const zh: Strings = {
noMatch: "没有匹配的",
andMore: (n: number) => `还有 ${n} 条`,
system: "系统",
+ partialRows: (sheet: string, read: number, total: number) =>
+ `${sheet}:只读了 ${read}/${total} 行`,
+ partialRecords: (read: number, total: number) =>
+ `只读了 ${read}/${total} 条记录`,
kinds: {
"source.sync_failed": {
title: "来源同步失败",
@@ -268,6 +272,10 @@ export const zh: Strings = {
title: "有文件要靠模型才读得出来",
hint: "扫描件和图片要配文档识别服务,录音要配能分出说话人的转写模型;每一行写着缺的是什么。文件已经留着,还没读出任何内容;到「管理 > 模型」存好读取模型,就会自动读。",
},
+ "document.contents_truncated": {
+ title: "文件只读了一部分",
+ hint: "解析器到了安全上限。提示行会写明漏掉的是哪个工作表或记录范围;检索和回答不包含被省略的部分。",
+ },
"governance.tripped": {
title: "agent 停止自动裁决了",
hint: "七天内它的合并被撤回了两次,开关已自动关掉。到「审核 > Agent」看看它做了什么,想让它接着跑就在库设置里再打开。",
diff --git a/web/src/pages/AlertBell.tsx b/web/src/pages/AlertBell.tsx
index 4f7de17d..0992ae2b 100644
--- a/web/src/pages/AlertBell.tsx
+++ b/web/src/pages/AlertBell.tsx
@@ -30,9 +30,28 @@ import {
const PAGE = 8;
-/** 明细里给人看的那一行:对象名 — 报错原文 */
+/** 明细里给人看的那一行:对象名 — 报错原文或截断范围 */
function line(d: AlertGroup["lines"][number]): string | null {
- const parts = [d.name ?? d.job, d.error].filter(Boolean);
+ const partial = d.warnings?.flatMap((warning) => {
+ const anchor = warning.detail;
+ if (
+ warning.kind === "spreadsheet.rows_truncated" &&
+ typeof anchor?.sheet === "string" &&
+ typeof anchor.rows_read === "number" &&
+ typeof anchor.rows_total === "number"
+ ) {
+ return [S.alerts.partialRows(anchor.sheet, anchor.rows_read, anchor.rows_total)];
+ }
+ if (
+ warning.kind === "csv.records_truncated" &&
+ typeof anchor?.records_read === "number" &&
+ typeof anchor.records_total === "number"
+ ) {
+ return [S.alerts.partialRecords(anchor.records_read, anchor.records_total)];
+ }
+ return [];
+ });
+ const parts = [d.name ?? d.job, d.error, ...(partial ?? [])].filter(Boolean);
return parts.length ? parts.join(" — ") : null;
}