Skip to content

World's first open-source enterprise world model.

Extraction stops discarding in silence (#41)

WaylandYang3c98dd795d93
diff --git a/crates/utopia-core/src/models.rs b/crates/utopia-core/src/models.rs
index 2dd47a86..df499fef 100644
--- a/crates/utopia-core/src/models.rs
+++ b/crates/utopia-core/src/models.rs
@@ -329,6 +329,18 @@ pub struct OntologyMiss {
pub count: i32,
}
+/// 抽取丢弃信号:事实抽出来了却没能落地,以及为什么。
+/// 与 `OntologyMiss` 分开——那个说"你的本体缺这些"(读者是本体维护者),
+/// 这个说"这些事实没落地"(读者是上传文档的人)。
+#[derive(Debug, Clone, Serialize, sqlx::FromRow)]
+pub struct ExtractionDrop {
+ pub document_id: Uuid,
+ pub reason: String,
+ pub detail: String,
+ pub count: i32,
+ pub example: Option<String>,
+}
+
#[derive(Debug, Clone, Serialize, sqlx::FromRow)]
pub struct RelationType {
pub id: Uuid,
@@ -503,6 +515,9 @@ pub struct FactReviewItem {
/// 事实的证据(引句 + 原文定位)。
#[derive(Debug, Clone, Serialize, sqlx::FromRow)]
pub struct EvidenceView {
+ /// 模型在这一块里实际用的谓词说法。词表外谓词被降级成 related_to 后,
+ /// 事实行上只剩"有关联"——原意只在这里
+ pub surface_predicate: Option<String>,
pub quote: Option<String>,
pub chunk_id: Uuid,
pub document_id: Uuid,
diff --git a/crates/utopia-extract/src/lib.rs b/crates/utopia-extract/src/lib.rs
index 82aeff79..d48623f5 100644
--- a/crates/utopia-extract/src/lib.rs
+++ b/crates/utopia-extract/src/lib.rs
@@ -93,7 +93,7 @@ pub fn build_messages(
let attr_rules = if attributes.is_empty() {
String::new()
} else {
- "\n8. Attribute facts carry \"value\" (no \"object\"): number = plain number without \
+ "\n9. Attribute facts carry \"value\" (no \"object\"): number = plain number without \
thousands separators or unit symbols; date = \"YYYY[-MM[-DD]]\"; bool = true/false; \
text = a short string. Only attach an attribute to a subject of its listed class. \
valid_from = when this value took effect, if the text says so."
@@ -105,7 +105,7 @@ pub fn build_messages(
\n\
Entity types (use only these keys):\n{type_list}\n\
\n\
- Relation types (use only these keys):\n{rel_list}\n\
+ Relation types (prefer these keys):\n{rel_list}\n\
{attr_section}\
\n\
Output format:\n\
@@ -122,7 +122,11 @@ pub fn build_messages(
4. {time_ctx}\n\
5. quote must be a contiguous excerpt from the source text; every fact needs one.\n\
6. confidence in 0~1: 0.9 explicitly stated, 0.7 inferred, 0.5 uncertain.\n\
- 7. If nothing can be extracted, output {{\"entities\":[],\"facts\":[]}}.{attr_rules}"
+ 7. If nothing can be extracted, output {{\"entities\":[],\"facts\":[]}}.\n\
+ 8. If no listed relation fits, do not force the nearest one — write the predicate the \
+ text itself uses, in snake_case (e.g. \"available_on\", \"runs_on\"). A relation \
+ named after the text is worth more than a listed one that says something false.\
+ {attr_rules}"
);
let user = format!("Source file: \"{filename}\"\n\nText:\n{chunk_text}");
diff --git a/crates/utopia-server/src/api/documents_routes.rs b/crates/utopia-server/src/api/documents_routes.rs
index 6255100b..6c04ef23 100644
--- a/crates/utopia-server/src/api/documents_routes.rs
+++ b/crates/utopia-server/src/api/documents_routes.rs
@@ -190,3 +190,16 @@ pub async fn reprocess(
fn hex(bytes: &[u8]) -> String {
bytes.iter().map(|b| format!("{b:02x}")).collect()
}
+
+/// 抽取丢弃信号:哪些事实抽出来了却没能落地。整库一次取回——按
+/// (文档 × 原因 × 具体对象) 聚合后行数很小,Library 既算总数又展开详情,
+/// 不必逐行发请求。
+pub async fn extraction_drops(
+ State(state): State<AppState>,
+ AuthUser(user): AuthUser,
+ Path(kb_id): Path<Uuid>,
+) -> ApiResult<Json<serde_json::Value>> {
+ utopia_store::access::require_kb(&state.pool, &user, kb_id, Role::Viewer).await?;
+ let drops = utopia_store::extraction_drops::for_kb(&state.pool, kb_id).await?;
+ Ok(Json(json!({ "drops": drops })))
+}
diff --git a/crates/utopia-server/src/api/mod.rs b/crates/utopia-server/src/api/mod.rs
index 989744db..ff47b723 100644
--- a/crates/utopia-server/src/api/mod.rs
+++ b/crates/utopia-server/src/api/mod.rs
@@ -115,6 +115,10 @@ pub fn router(state: AppState, cfg: &AppConfig) -> Router {
.post(documents_routes::upload)
.layer(DefaultBodyLimit::max(MAX_UPLOAD_BYTES)),
)
+ .route(
+ "/kbs/{id}/extraction-drops",
+ get(documents_routes::extraction_drops),
+ )
.route("/kbs/{id}/ontology", get(ontology_routes::get))
.route(
"/kbs/{id}/ontology/entity-types",
diff --git a/crates/utopia-server/src/extraction.rs b/crates/utopia-server/src/extraction.rs
index bb2827b2..47eb3086 100644
--- a/crates/utopia-server/src/extraction.rs
+++ b/crates/utopia-server/src/extraction.rs
@@ -8,6 +8,29 @@ use std::collections::HashMap;
use uuid::Uuid;
const MIN_CONFIDENCE: f32 = 0.6;
+/// 词表外谓词的代码层兜底。刻意不进提示词——理由见 rel_pairs 的注释。
+const FALLBACK_RELATION_KEY: &str = "related_to";
+
+/// 记一条丢弃信号。抽取器有七处 `continue`,每一处都是"事实抽出来了、被挡掉、
+/// 什么都不说"。信号写失败绝不能带垮整篇文档的抽取,所以这里吞掉错误。
+async fn drop_signal(
+ state: &AppState,
+ kb_id: Uuid,
+ document_id: Uuid,
+ reason: &str,
+ detail: &str,
+ example: Option<&str>,
+) {
+ let _ = utopia_store::extraction_drops::record(
+ &state.pool,
+ kb_id,
+ document_id,
+ reason,
+ detail,
+ example,
+ )
+ .await;
+}
pub async fn extract_document(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
match run(state, document_id).await {
@@ -86,10 +109,16 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
.iter()
.map(|t| (t.key.clone(), t.label.clone(), t.description.clone()))
.collect();
- // 关系与属性分道:属性走字面值通道,不进关系清单
+ // 关系与属性分道:属性走字面值通道,不进关系清单。
+ //
+ // related_to 刻意不列给模型:它在本体里是代码层的兜底,但摆进提示词就成了
+ // 逃生舱——模型读到说不清的关系时不会去写原文说法,直接挑这个万能选项,
+ // 而它什么都没说。实测 359 条 related_to 里只有 38 条是词表外降级,
+ // 其余 321 条是模型自己挑的。撤掉之后模型要么用真关系、要么写出原文说法,
+ // 兜底改由下面的代码执行,原词落进 fact_evidence.surface_predicate 留待消解。
let rel_pairs: Vec<(String, String, String)> = rtypes
.iter()
- .filter(|r| r.kind != "attribute")
+ .filter(|r| r.kind != "attribute" && r.key != FALLBACK_RELATION_KEY)
.map(|r| (r.key.clone(), r.label.clone(), r.description.clone()))
.collect();
let type_ids: HashMap<&str, Uuid> = etypes.iter().map(|t| (t.key.as_str(), t.id)).collect();
@@ -149,7 +178,9 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
let concept_type = *type_ids
.get("concept")
.ok_or_else(|| anyhow::anyhow!("Ontology missing the 'concept' type"))?;
- let related_rel = rel_ids.get("related_to").copied();
+ let related_rel = rel_ids.get(FALLBACK_RELATION_KEY).copied();
+ // 本轮要从头讲一遍这篇文档的故事,旧信号先清掉(重抽自动作数)
+ let _ = utopia_store::extraction_drops::clear_for_document(&state.pool, document_id).await;
let doc_time = doc.doc_time.map(|t| t.format("%Y-%m-%d").to_string());
let chunks = utopia_store::documents::chunks_for_extraction(&state.pool, document_id).await?;
@@ -232,6 +263,16 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
for f in &extraction.facts {
let confidence = f.confidence.unwrap_or(0.7).clamp(0.0, 1.0);
if confidence < MIN_CONFIDENCE {
+ // 设计上的阈值,但用户同样无从知道"抽到了,只是不够自信"
+ drop_signal(
+ state,
+ doc.kb_id,
+ document_id,
+ utopia_store::extraction_drops::reason::LOW_CONFIDENCE,
+ &f.predicate,
+ Some(&format!("{} ({:.0}%)", f.subject, confidence * 100.0)),
+ )
+ .await;
continue;
}
let from = f.valid_from.as_deref().and_then(utopia_extract::parse_time);
@@ -248,16 +289,41 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
});
if let Some(attr) = attr_hit {
let subject_name = f.subject.trim();
+ // 主语没在 entities 里声明:类型不明,domain 无从校验,属性不落。
+ // 关系路径遇到同样的缺失会兜底按 concept 消解——这里学不来,
+ // 按 concept 解出来 domain 照样不匹配,只是从这里掉进下面那一档
let Some((&subject_id, &subject_type)) = entity_ids
.get(subject_name)
.zip(entity_type_of.get(subject_name))
else {
- continue; // 主语没在 entities 里声明:类型不明,属性不落
+ drop_signal(
+ state,
+ doc.kb_id,
+ document_id,
+ utopia_store::extraction_drops::reason::SUBJECT_NOT_DECLARED,
+ &attr.key,
+ Some(subject_name),
+ )
+ .await;
+ continue;
};
+ // 不可达:store 层强制 attribute 必有 domain,且 kind/domain 不可改,
+ // 无 domain 的属性连提示词都进不去。留着是防御,不需要信号
let Some(domain) = attr.domain_type_id else {
continue;
};
if !type_matches_domain(subject_type, domain) {
+ let subj_key = type_key_by_id.get(&subject_type).copied().unwrap_or("?");
+ let dom_key = type_key_by_id.get(&domain).copied().unwrap_or("?");
+ drop_signal(
+ state,
+ doc.kb_id,
+ document_id,
+ utopia_store::extraction_drops::reason::ATTR_DOMAIN_MISMATCH,
+ &format!("{}@{subj_key} (wants {dom_key})", attr.key),
+ Some(subject_name),
+ )
+ .await;
continue;
}
let raw = match (&f.value, &f.object) {
@@ -266,11 +332,31 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
(None, Some(o)) if !o.trim().is_empty() => {
serde_json::Value::String(o.trim().to_string())
}
- _ => continue,
+ _ => {
+ drop_signal(
+ state,
+ doc.kb_id,
+ document_id,
+ utopia_store::extraction_drops::reason::ATTR_NO_VALUE,
+ &attr.key,
+ Some(subject_name),
+ )
+ .await;
+ continue;
+ }
};
let datatype = attr.datatype.as_deref().unwrap_or("text");
let Some(normalized) = utopia_extract::normalize_attr_value(datatype, &raw) else {
tracing::debug!(%document_id, attr = attr.key, ?raw, "属性值不合 datatype,跳过");
+ drop_signal(
+ state,
+ doc.kb_id,
+ document_id,
+ utopia_store::extraction_drops::reason::ATTR_DATATYPE,
+ &format!("{} ({datatype})", attr.key),
+ Some(&format!("{subject_name} → {raw}")),
+ )
+ .await;
continue;
};
let mut object_value = serde_json::json!({ "value": normalized });
@@ -290,11 +376,14 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
confidence,
)
.await?;
+ // 属性谓词也留原词:模型偶尔照抄 "person.salary" 全限定名,
+ // 命中的是剥掉前缀后的 key,原样是什么值得留着
utopia_store::graph::add_evidence(
&state.pool,
fact_id,
chunk.id,
f.quote.as_deref(),
+ Some(f.predicate.as_str()),
)
.await?;
if !created {
@@ -327,6 +416,15 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
// 关系事实:宾语必填
let Some(object_name) = f.object.as_deref().map(str::trim).filter(|s| !s.is_empty())
else {
+ drop_signal(
+ state,
+ doc.kb_id,
+ document_id,
+ utopia_store::extraction_drops::reason::OBJECT_MISSING,
+ &f.predicate,
+ Some(f.subject.trim()),
+ )
+ .await;
continue;
};
// 主宾未在 entities 中声明时,按 concept 兜底消解(模型偶尔漏报)
@@ -363,7 +461,9 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
if subject_id == object_id {
continue;
}
- // 未知关系降级为 related_to,并记入未匹配统计
+ // 未知关系降级为 related_to,并记入未匹配统计。
+ // 降级会把原意抹平成"有关联"——原词写进证据行的 surface_predicate,
+ // 是这条事实身上唯一还留着原意的地方(谓词消解据此把它映射回本体)
let predicate_id = match rel_ids.get(f.predicate.as_str()) {
Some(id) => *id,
None => {
@@ -377,7 +477,19 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
.await;
match related_rel {
Some(id) => id,
- None => continue,
+ // 本体里连兜底关系都被删了 → 整条事实消失,这个必须说出来
+ None => {
+ drop_signal(
+ state,
+ doc.kb_id,
+ document_id,
+ utopia_store::extraction_drops::reason::FALLBACK_RELATION_MISSING,
+ &f.predicate,
+ Some(&format!("{} → {object_name}", f.subject)),
+ )
+ .await;
+ continue;
+ }
}
}
};
@@ -395,12 +507,15 @@ async fn run(state: &AppState, document_id: Uuid) -> anyhow::Result<()> {
confidence,
)
.await?;
- // 重复观察也要挂证据:多来源相互印证,任一来源删除后事实不孤儿化
+ // 重复观察也要挂证据:多来源相互印证,任一来源删除后事实不孤儿化。
+ // 表层谓词随每次观察落笔——甲块说 "runs on"、乙块说 "optimized for"
+ // 会并进同一条事实,放事实上就是先写者胜,放证据上两个都留着
utopia_store::graph::add_evidence(
&state.pool,
fact_id,
chunk.id,
f.quote.as_deref(),
+ Some(f.predicate.as_str()),
)
.await?;
if !created {
diff --git a/crates/utopia-server/src/mappings.rs b/crates/utopia-server/src/mappings.rs
index 4bb0184c..49c7f839 100644
--- a/crates/utopia-server/src/mappings.rs
+++ b/crates/utopia-server/src/mappings.rs
@@ -172,11 +172,13 @@ pub async fn explore_mappings(state: &AppState, kb_id: Uuid) -> anyhow::Result<(
.await?
{
let rationale = p["rationale"].as_str().unwrap_or("");
+ // 谓词是代码写死的 mapped_to,不是模型给的说法,没有表层形式可留
utopia_store::graph::add_evidence(
&state.pool,
fact_id,
chunk_id,
(!rationale.is_empty()).then_some(rationale),
+ None,
)
.await?;
}
diff --git a/crates/utopia-store/src/extraction_drops.rs b/crates/utopia-store/src/extraction_drops.rs
new file mode 100644
index 00000000..610fcaef
--- /dev/null
+++ b/crates/utopia-store/src/extraction_drops.rs
@@ -0,0 +1,78 @@
+//! 抽取丢弃信号:哪些事实抽出来了却没能落地,以及为什么。
+//!
+//! 与 `ontology_misses` 分开是刻意的——那张表说的是"你的本体缺这些",读者是
+//! 本体维护者,动作是加类型;这张说的是"这些事实没落地",读者是上传文档的人,
+//! 动作是改文档或改本体。混在一个面板里两边都讲不清。
+//!
+//! 记录失败不影响抽取(调用方一律 `let _ =`)——信号缺一条,远好过因为记信号
+//! 失败而中断整篇文档的抽取。
+
+use sqlx::PgPool;
+use utopia_core::models::ExtractionDrop;
+use utopia_core::AppResult;
+use uuid::Uuid;
+
+/// 原因码。前端按这个查文案,所以是稳定契约,不要改字面量。
+pub mod reason {
+ /// 主语没在 entities 里声明 → 类型不明,属性无法校验 domain
+ pub const SUBJECT_NOT_DECLARED: &str = "subject_not_declared";
+ /// 属性挂在了不该挂的类上(salary 挂到 Organization)
+ pub const ATTR_DOMAIN_MISMATCH: &str = "attr_domain_mismatch";
+ /// 属性事实既没给 value 也没给 object
+ pub const ATTR_NO_VALUE: &str = "attr_no_value";
+ /// 值不合 datatype,归一化失败
+ pub const ATTR_DATATYPE: &str = "attr_datatype";
+ /// 模型自报置信度低于阈值
+ pub const LOW_CONFIDENCE: &str = "low_confidence";
+ /// 关系事实缺宾语
+ pub const OBJECT_MISSING: &str = "object_missing";
+ /// 词表外谓词本该降级,但本体里连兜底关系都没有 → 整条消失
+ pub const FALLBACK_RELATION_MISSING: &str = "fallback_relation_missing";
+}
+
+pub async fn record(
+ pool: &PgPool,
+ kb_id: Uuid,
+ document_id: Uuid,
+ reason: &str,
+ detail: &str,
+ example: Option<&str>,
+) -> AppResult<()> {
+ sqlx::query(
+ "INSERT INTO extraction_drops (kb_id, document_id, reason, detail, example)
+ VALUES ($1, $2, $3, left($4, 120), left($5, 200))
+ ON CONFLICT (kb_id, document_id, reason, detail)
+ DO UPDATE SET count = extraction_drops.count + 1,
+ example = COALESCE(EXCLUDED.example, extraction_drops.example),
+ updated_at = now()",
+ )
+ .bind(kb_id)
+ .bind(document_id)
+ .bind(reason)
+ .bind(detail)
+ .bind(example)
+ .execute(pool)
+ .await?;
+ Ok(())
+}
+
+/// 重抽开始时清掉这篇文档的旧信号——本轮要从头讲一遍这篇文档的故事。
+pub async fn clear_for_document(pool: &PgPool, document_id: Uuid) -> AppResult<()> {
+ sqlx::query("DELETE FROM extraction_drops WHERE document_id = $1")
+ .bind(document_id)
+ .execute(pool)
+ .await?;
+ Ok(())
+}
+
+/// 一个 KB 的全部丢弃信号。行数按 (文档 × 原因 × 具体对象) 聚合后很小,
+/// 一次取回让 Library 既能算每篇的总数、又能直接展开详情,不必逐行发请求。
+pub async fn for_kb(pool: &PgPool, kb_id: Uuid) -> AppResult<Vec<ExtractionDrop>> {
+ Ok(sqlx::query_as(
+ "SELECT document_id, reason, detail, count, example FROM extraction_drops
+ WHERE kb_id = $1 ORDER BY count DESC, reason LIMIT 2000",
+ )
+ .bind(kb_id)
+ .fetch_all(pool)
+ .await?)
+}
diff --git a/crates/utopia-store/src/graph.rs b/crates/utopia-store/src/graph.rs
index cafb8ddf..90f536c2 100644
--- a/crates/utopia-store/src/graph.rs
+++ b/crates/utopia-store/src/graph.rs
@@ -312,21 +312,30 @@ pub async fn confirmed_mappings(
Ok(rows)
}
+/// `surface`:模型在这一块里实际用的谓词说法。谓词命中本体时它等于 key,
+/// 词表外被降级成 related_to 时它是唯一还留着原意的东西——事实行上只剩
+/// "有关联",原文说的"runs on"就靠这里活下来。
pub async fn add_evidence(
pool: &PgPool,
fact_id: Uuid,
chunk_id: Uuid,
quote: Option<&str>,
+ surface: Option<&str>,
) -> AppResult<()> {
// 证据落笔即记版本:出自哪份文档的第几版(S3 版本对账与"证据过期"判定的依据)
+ // 冲突时补写表层谓词而非整行跳过:重抽命中的多是已有的 (事实, 分块) 对,
+ // DO NOTHING 会让存量证据永远填不上这一列。只在原值为空时补,不覆盖——
+ // 同一分块的同一条事实,第一次记下的说法就是它的说法
sqlx::query(
- "INSERT INTO fact_evidence (fact_id, chunk_id, quote, document_id, doc_version)
- SELECT $1, $2, $3, c.document_id, c.doc_version FROM chunks c WHERE c.id = $2
- ON CONFLICT DO NOTHING",
+ "INSERT INTO fact_evidence (fact_id, chunk_id, quote, surface_predicate, document_id, doc_version)
+ SELECT $1, $2, $3, left($4, 120), c.document_id, c.doc_version FROM chunks c WHERE c.id = $2
+ ON CONFLICT (fact_id, chunk_id) DO UPDATE
+ SET surface_predicate = COALESCE(fact_evidence.surface_predicate, EXCLUDED.surface_predicate)",
)
.bind(fact_id)
.bind(chunk_id)
.bind(quote)
+ .bind(surface)
.execute(pool)
.await?;
Ok(())
@@ -742,7 +751,7 @@ pub async fn document_extractions(
/// stale = 证据版本落后于文档当前版本(UI 标 "from v{n}")。
pub async fn fact_evidence(pool: &PgPool, fact_id: Uuid) -> AppResult<Vec<EvidenceView>> {
let rows: Vec<EvidenceView> = sqlx::query_as(
- "SELECT fe.quote, fe.chunk_id, c.document_id, d.filename, c.seq,
+ "SELECT fe.quote, fe.surface_predicate, fe.chunk_id, c.document_id, d.filename, c.seq,
c.doc_version,
c.doc_version < COALESCE(
(SELECT MAX(version) FROM document_versions dv
diff --git a/crates/utopia-store/src/lib.rs b/crates/utopia-store/src/lib.rs
index a30a2034..33b128b7 100644
--- a/crates/utopia-store/src/lib.rs
+++ b/crates/utopia-store/src/lib.rs
@@ -8,6 +8,7 @@ pub mod conversations;
pub mod datasources;
pub mod db;
pub mod documents;
+pub mod extraction_drops;
pub mod graph;
pub mod jobs;
pub mod kbs;
diff --git a/migrations/0022_extraction_transparency.sql b/migrations/0022_extraction_transparency.sql
new file mode 100644
index 00000000..d2869429
--- /dev/null
+++ b/migrations/0022_extraction_transparency.sql
@@ -0,0 +1,35 @@
+-- 抽取的两处静默丢失。
+--
+-- 一、被挡掉的事实不留痕。抽取器有七处 `continue`:主语类型不明、属性 domain
+-- 不匹配、值不合 datatype、置信度不够……事实抽出来了、被挡掉、什么都不说,
+-- 用户只看到图里少了东西。与"账本 append-only、每条事实都有证据、不确定性
+-- 浮到人面前"三条原则直接冲突。
+--
+-- 二、降级成 related_to 时原词丢了。`ontology_misses` 只记了"available_from
+-- 出现过 9 次"这样的总数,事实行上一字未留——今天无法回答"哪条 related_to
+-- 原本说的是什么"。
+
+-- 按 document 归集:既能算单篇的"多少条没落地",也能在 KB 层聚合。
+-- 同时天然修好生命周期——ontology_misses 只在整库重建时清(graph.rs),
+-- 来源级重抽不清,会攒陈旧计数;按 document 清则每次重抽自动作数。
+CREATE TABLE extraction_drops (
+ kb_id UUID NOT NULL REFERENCES knowledge_bases(id) ON DELETE CASCADE,
+ document_id UUID NOT NULL REFERENCES documents(id) ON DELETE CASCADE,
+ -- 机器可聚合的原因码(attr_domain_mismatch / low_confidence / ...)
+ reason TEXT NOT NULL,
+ -- 该原因下的具体对象(属性 key、谓词名、"salary@organization")
+ detail TEXT NOT NULL,
+ count INT NOT NULL DEFAULT 1,
+ -- 一个样例,让人一眼看出丢的是什么("Acme Corp → salary")
+ example TEXT,
+ updated_at TIMESTAMPTZ NOT NULL DEFAULT now(),
+ PRIMARY KEY (kb_id, document_id, reason, detail)
+);
+
+CREATE INDEX extraction_drops_doc_idx ON extraction_drops (document_id);
+
+-- 表层谓词落在证据上而不是事实上:事实按 (kb, 主, 谓, 宾) 去重,甲块说
+-- "runs on"、乙块说 "optimized for" 会并成同一行,放事实上就是先写者胜、
+-- 其余静默丢弃——正是本次要修的毛病。证据是每分块一行,粒度对,而且它
+-- 已经带着 quote(该块的原文佐证),表层谓词是同一种东西:每次观察的原始形态。
+ALTER TABLE fact_evidence ADD COLUMN surface_predicate TEXT;
diff --git a/web/src/api.ts b/web/src/api.ts
index 8dafc0fb..b7f17011 100644
--- a/web/src/api.ts
+++ b/web/src/api.ts
@@ -101,6 +101,17 @@ export interface Doc {
created_at: string;
}
+/** 一类抽取丢弃在一篇文档里的聚合:事实抽出来了,却没能落地。 */
+export interface ExtractionDrop {
+ document_id: string;
+ /** 稳定的原因码,前端按它查文案(attr_domain_mismatch / low_confidence / ...) */
+ reason: string;
+ /** 该原因下的具体对象(属性 key、谓词名、"salary@organization") */
+ detail: string;
+ count: number;
+ example: string | null;
+}
+
export interface SourceView {
id: string;
kind: "folder" | "url" | "rss" | "api" | "custom" | "memory" | "upload";
@@ -324,6 +335,8 @@ export interface EntityHistoryEvent {
}
export interface Evidence {
+ /** 模型在这一块里实际用的谓词说法。词表外谓词降级成 related_to 后,原意只剩这里 */
+ surface_predicate: string | null;
quote: string | null;
chunk_id: string;
document_id: string;
@@ -529,6 +542,9 @@ export const api = {
),
documents: (kbId: string) => request<Doc[]>(`/api/v1/kbs/${kbId}/documents`),
+ /** 整库一次取回:行数按 (文档 × 原因 × 对象) 聚合后很小,避免逐行发请求 */
+ extractionDrops: (kbId: string) =>
+ request<{ drops: ExtractionDrop[] }>(`/api/v1/kbs/${kbId}/extraction-drops`),
upload: (kbId: string, files: File[], sourceId?: string) => {
const form = new FormData();
for (const f of files) form.append("files", f, f.name);
diff --git a/web/src/i18n.ts b/web/src/i18n.ts
index 54498f45..9acfb90f 100644
--- a/web/src/i18n.ts
+++ b/web/src/i18n.ts
@@ -212,6 +212,22 @@ export const S = {
errorGraph: "Graph extraction",
copyError: "Copy",
errorCopied: "Error copied",
+ /* 抽取丢弃:事实抽出来了却没能落地。此前完全无声——图里少了东西,没人说得出少了什么 */
+ dropsChip: (n: number) => `${n} dropped`,
+ dropsTitle: "Facts that did not land",
+ dropsNote:
+ "These were extracted from the document but blocked on the way in. " +
+ "Each line says why, and how many.",
+ dropsExample: "e.g.",
+ dropReason: {
+ attr_domain_mismatch: "Attribute on the wrong class",
+ subject_not_declared: "Subject type unknown",
+ attr_no_value: "Attribute had no value",
+ attr_datatype: "Value did not match the datatype",
+ low_confidence: "Below the confidence threshold",
+ object_missing: "Relation had no object",
+ fallback_relation_missing: "No fallback relation in the ontology",
+ } as Record<string, string>,
// 来源级重抽:不危险,只是费时费钱——轻确认,文案直说成本与保留项
reExtractSource: "Re-extract",
reExtractTitle: "Re-extract this source?",
@@ -404,6 +420,8 @@ export const S = {
evidence: "evidence",
noEvidence: "No evidence recorded",
noQuote: "(no quote)",
+ /* 原文说的谓词。词表外的词会被降级成 related to,原意只在这里活着 */
+ surfacePredicate: (p: string) => `the text said “${p}”`,
sectionRef: (filename: string, seq: number) => `${filename} · section ${seq} →`,
fromVersion: (v: number) => `v${v}`,
staleEvidenceHint:
diff --git a/web/src/pages/Graph.tsx b/web/src/pages/Graph.tsx
index dbf8a949..a3a66a6f 100644
--- a/web/src/pages/Graph.tsx
+++ b/web/src/pages/Graph.tsx
@@ -1701,6 +1701,13 @@ function EvidenceList({ kbId, fact }: { kbId: string; fact: EntityFact }) {
search={{ chunk: ev.chunk_id }}
className="block text-xs text-neutral-500 hover:text-neutral-300"
>
+ {/* 原文说的谓词,只在它与本体谓词不同时显示——词表外的词被降级成
+ "related to" 后,事实行上只剩"有关联",原意仅存于此。相同则是噪声 */}
+ {ev.surface_predicate && ev.surface_predicate !== fact.predicate_key && (
+ <div className="mb-0.5 text-[11px] text-neutral-400">
+ {S.graph.surfacePredicate(ev.surface_predicate)}
+ </div>
+ )}
<div className="line-clamp-2 italic">
{ev.quote ? `“${ev.quote}”` : S.graph.noQuote}
</div>
diff --git a/web/src/pages/Library.tsx b/web/src/pages/Library.tsx
index 564dae3e..54079efd 100644
--- a/web/src/pages/Library.tsx
+++ b/web/src/pages/Library.tsx
@@ -1,4 +1,4 @@
-import { useCallback, useEffect, useRef, useState } from "react";
+import { useCallback, useEffect, useMemo, useRef, useState } from "react";
import { useMutation, useQuery, useQueryClient } from "@tanstack/react-query";
import { Link, useSearch } from "@tanstack/react-router";
import {
@@ -12,7 +12,7 @@ import {
Waypoints,
X,
} from "lucide-react";
-import { api, type Doc, type SourceView } from "../api";
+import { api, type Doc, type ExtractionDrop, type SourceView } from "../api";
import { S } from "../i18n";
import { useKb } from "../kb";
import { toast } from "../toast";
@@ -319,6 +319,10 @@ export function Library() {
kind: string;
text: string;
} | null>(null);
+ // 丢弃详情弹窗:这篇文档抽出来却没落地的事实
+ const [dropsView, setDropsView] = useState<{ file: string; rows: ExtractionDrop[] } | null>(
+ null,
+ );
const [showHistory, setShowHistory] = useState(false);
const [page, setPage] = useState(0);
const [filter, setFilter] = useState("");
@@ -343,10 +347,27 @@ export function Library() {
queryFn: () => api.sources(kb!.id),
enabled: !!kb,
});
+ // 整库一次取回后按文档分组:抽取丢弃的行数很小,好过每行发一个请求
+ const drops = useQuery({
+ queryKey: ["extraction-drops", kb?.id],
+ queryFn: () => api.extractionDrops(kb!.id),
+ enabled: !!kb,
+ });
+ const dropsByDoc = useMemo(() => {
+ const m = new Map<string, ExtractionDrop[]>();
+ for (const d of drops.data?.drops ?? []) {
+ const list = m.get(d.document_id);
+ if (list) list.push(d);
+ else m.set(d.document_id, [d]);
+ }
+ return m;
+ }, [drops.data]);
const invalidate = () => {
queryClient.invalidateQueries({ queryKey: ["documents", kb?.id] });
queryClient.invalidateQueries({ queryKey: ["sources", kb?.id] });
+ // 重抽会重写这篇文档的丢弃信号,跟着文档一起失效
+ queryClient.invalidateQueries({ queryKey: ["extraction-drops", kb?.id] });
};
const upload = useMutation({
@@ -613,6 +634,8 @@ export function Library() {
onShowError={(kind, text) =>
setErrorView({ file: d.filename, kind, text })
}
+ drops={dropsByDoc.get(d.id)}
+ onShowDrops={(rows) => setDropsView({ file: d.filename, rows })}
/>
))}
</tbody>
@@ -713,6 +736,13 @@ export function Library() {
onClose={() => setErrorView(null)}
/>
)}
+ {dropsView && (
+ <DropsModal
+ file={dropsView.file}
+ rows={dropsView.rows}
+ onClose={() => setDropsView(null)}
+ />
+ )}
{/* 全库重建:打字级确认(清空图层不可逆) */}
{rebuilding && (
<DangerConfirm
@@ -886,6 +916,70 @@ function SourceBar({
}
/** 失败详情:完整原文,可复制。tooltip 装不下也拷不走,所以要弹窗。 */
+/** 抽取丢弃详情:抽出来了、被挡掉了,这里说清楚挡在哪、丢了多少。 */
+function DropsModal({
+ file,
+ rows,
+ onClose,
+}: {
+ file: string;
+ rows: ExtractionDrop[];
+ onClose: () => void;
+}) {
+ useEffect(() => {
+ const onKey = (e: KeyboardEvent) => e.key === "Escape" && onClose();
+ document.addEventListener("keydown", onKey);
+ return () => document.removeEventListener("keydown", onKey);
+ }, [onClose]);
+
+ return (
+ <div
+ className="fixed inset-0 z-50 grid place-items-center bg-black/60 backdrop-blur-sm"
+ onMouseDown={(e) => {
+ if (e.target === e.currentTarget) onClose();
+ }}
+ >
+ <div className="glass-strong w-[36rem] max-w-[calc(100vw-2rem)] rounded-2xl shadow-2xl">
+ <div className="px-5 pt-4 pb-3 border-b border-white/10">
+ <div className="flex items-center justify-between">
+ <h2 className="u-title text-[15px]">{S.library.dropsTitle}</h2>
+ <button onClick={onClose} className="text-neutral-500 hover:text-neutral-200">
+ <X size={15} />
+ </button>
+ </div>
+ <p className="mt-1 text-xs text-neutral-500 truncate" title={file}>
+ {file}
+ </p>
+ <p className="mt-1.5 text-xs text-neutral-500">{S.library.dropsNote}</p>
+ </div>
+ <div className="u-scroll max-h-80 overflow-y-auto px-5 py-3">
+ {rows.map((r) => (
+ <div
+ key={`${r.reason}:${r.detail}`}
+ className="border-b border-white/5 py-2.5 last:border-0"
+ >
+ <div className="flex items-baseline gap-2">
+ <span className="text-sm text-neutral-200">
+ {S.library.dropReason[r.reason] ?? r.reason}
+ </span>
+ <span className="u-num ml-auto text-xs text-neutral-500">×{r.count}</span>
+ </div>
+ <div className="mt-0.5 font-mono text-[11px] text-neutral-400 break-all">
+ {r.detail}
+ </div>
+ {r.example && (
+ <div className="mt-0.5 text-[11px] text-neutral-600 break-all">
+ {S.library.dropsExample} {r.example}
+ </div>
+ )}
+ </div>
+ ))}
+ </div>
+ </div>
+ </div>
+ );
+}
+
function ErrorModal({
file,
kind,
@@ -1560,6 +1654,8 @@ function DocRow({
onExtract,
onReprocess,
onShowError,
+ drops,
+ onShowDrops,
}: {
doc: Doc;
/** undefined = 不渲染来源列;null = Uploads(source_id 为空) */
@@ -1568,7 +1664,11 @@ function DocRow({
onExtract: () => void;
onReprocess: () => void;
onShowError: (kind: string, text: string) => void;
+ /** 这篇文档抽出来却没落地的事实;undefined = 一条都没有 */
+ drops?: ExtractionDrop[];
+ onShowDrops: (rows: ExtractionDrop[]) => void;
}) {
+ const dropTotal = drops?.reduce((n, d) => n + d.count, 0) ?? 0;
const statusText =
S.library.status[doc.status as keyof typeof S.library.status] ?? doc.status;
const graphText =
@@ -1632,6 +1732,13 @@ function DocRow({
) : (
<Chip tone={GRAPH_TONE[doc.graph_status] ?? "neutral"}>{graphText}</Chip>
)}
+ {/* 抽出来却没落地的事实。抽取成功不代表全须全尾,所以这个 chip 与
+ graph_status 并列而不是替代它——"done" 和 "3 dropped" 同时为真 */}
+ {dropTotal > 0 && drops && (
+ <button onClick={() => onShowDrops(drops)} className="ml-1.5 align-middle">
+ <Chip tone="warn">{S.library.dropsChip(dropTotal)}</Chip>
+ </button>
+ )}
{/* done 也可重抽:本体(描述/新类)调整后强制全量重抽正是常规操作 */}
{doc.status === "ready" && ["none", "failed", "done"].includes(doc.graph_status) && (
<button onClick={onExtract} className="u-link ml-1.5 text-xs">