Skip to content

World's first open-source enterprise world model.

refactor(extraction): split run_open loop into call/parse + apply helpers (#588 cut 1) (#980)

Royce79822c992356
diff --git a/crates/utopia-core/src/config.rs b/crates/utopia-core/src/config.rs
index e1af1683..c00a2cf4 100644
--- a/crates/utopia-core/src/config.rs
+++ b/crates/utopia-core/src/config.rs
@@ -29,6 +29,11 @@ pub struct AppConfig {
/// 一个分块的预算(cl100k token)。是个旋钮而不是常量,因为它要能被量:
/// 结构(表头跟着走)与大小是两件事,召回台子得能只动一个。`UTOPIA_CHUNK_TOKENS`
pub chunk_tokens: usize,
+ /// 抽取的提示词里带不带「这篇文档前面的分块已经认下的实体」(`known`)。缺省带。
+ /// 也是个为了量而设的旋钮(#588):分块并发调模型之后,后面的块看不到前面的块认下了
+ /// 什么;关掉它跑一轮召回台子,就是那种情形下的输出。只动提示词——落库时按名字认回
+ /// 前面分块的实体照旧。`UTOPIA_EXTRACT_KNOWN_IN_PROMPT`
+ pub extract_known_in_prompt: bool,
/// 数据库连接池上限。缺省 32,与 worker 并发的缺省对齐——池子小于并发时
/// 症状是请求变慢而不是任何一处说"池子不够",所以它必须可调。
pub db_max_connections: Option<u32>,
@@ -50,6 +55,7 @@ impl Default for AppConfig {
web_dist: "web/dist".into(),
data_dir: "data".into(),
chunk_tokens: 300,
+ extract_known_in_prompt: true,
db_max_connections: None,
cookie_secure: false,
open_registration: true,
diff --git a/crates/utopia-server/src/extraction_open.rs b/crates/utopia-server/src/extraction_open.rs
index c70157d9..430f1ade 100644
--- a/crates/utopia-server/src/extraction_open.rs
+++ b/crates/utopia-server/src/extraction_open.rs
@@ -27,12 +27,662 @@ use crate::state::AppState;
use sqlx::PgPool;
use std::collections::{HashMap, HashSet};
use utopia_core::models::{Document, KnowledgeBase, LlmSettings, Proposer};
+use utopia_extract::KnownEntity;
+use utopia_store::documents::ChunkForExtract;
use utopia_store::extraction_drops::reason;
use utopia_store::graph::FactObject;
use utopia_store::graph::Validity;
use utopia_store::pending::{Outcome, Proposal};
use uuid::Uuid;
+/// 一块调 + 解析完后的中间产物,apply 半步读它落库(#588 cut 1)。
+/// `name_vecs`(0041 决定 3 通道 2)紧跟解析算出来——apply 里的 `resolve_handle`
+/// 立刻就要它,分两处算只会让 helper 之间传更多参数。
+struct ChunkParsed {
+ extraction: utopia_extract::open::OpenExtraction,
+ name_vecs: HashMap<String, Vec<f32>>,
+}
+
+/// 一个分块的前半步:调模型、解析回复、算名字向量(#588 cut 1)。这半步不写图,
+/// 所以分块之间可以不按顺序做——并发(cut 2)动的就是它;后半步 `apply_open_extraction`
+/// 要按分块顺序。`pushed`(0054)的块本身就是契约,不调模型,直接解析。
+///
+/// `Ok(None)` 是这一块跳过了:调用失败或回复解析不出,原因已经记进 `drop_signal` 和
+/// `unextracted`,主循环接着走下一块。`Err` 只有一种:对话模型没配。
+///
+/// 被接管的检查(epoch)留在主循环顶上,不在这里:它要在调模型之前
+#[allow(clippy::too_many_arguments)]
+async fn call_and_parse_one_chunk(
+ state: &AppState,
+ settings: Option<&LlmSettings>,
+ client: Option<&utopia_llm::LlmClient>,
+ embed_client: Option<&utopia_llm::LlmClient>,
+ kb_id: Uuid,
+ document_id: Uuid,
+ filename: &str,
+ chunk: &ChunkForExtract,
+ opening: Option<&str>,
+ known: &[KnownEntity],
+ pushed: bool,
+ unextracted: &mut Vec<(i32, String)>,
+) -> anyhow::Result<Option<ChunkParsed>> {
+ // 推送来的陈述:块就是契约,解析它而不是问模型(0054)。下面从解析起一步不变
+ let (reply_text, cut_by_ceiling) = if pushed {
+ (chunk.text.clone(), false)
+ } else {
+ let (settings, client) = match (settings, client) {
+ (Some(s), Some(c)) => (s, c),
+ _ => anyhow::bail!("Chat model not configured; cannot extract"),
+ };
+ let messages =
+ utopia_extract::open::build_open_messages(filename, known, opening, &chunk.text);
+ // 温度 0:照抄原文的活不该靠采样。端点缺省 1.0 时同一块两次回复密度差三倍
+ let reply = match chat_retrying_rate_limits_at(
+ state,
+ settings,
+ client,
+ &messages,
+ Some(0.0),
+ )
+ .await
+ {
+ Ok(r) => r,
+ Err(e) => {
+ tracing::warn!(%document_id, seq = chunk.seq, error = %e, "开放抽取调用失败,跳过该分块");
+ drop_signal(
+ state,
+ kb_id,
+ document_id,
+ reason::CHUNK_UNEXTRACTED,
+ "调用失败,这一块没有进图",
+ Some(&format!("#{}:{e}", chunk.seq)),
+ )
+ .await;
+ unextracted.push((chunk.seq, format!("调用失败:{e}")));
+ return Ok(None);
+ }
+ };
+ tracing::debug!(%document_id, seq = chunk.seq, reply = %reply.text, "开放抽取的原始回复");
+ // 端点说它是撞上 token 上限停的。解析器只看得见 JSON 少了尾巴,看不见
+ // 少的原因,所以这句话得从回复里带过来(#760)
+ let cut_by_ceiling = reply.hit_token_ceiling();
+ (reply.text, cut_by_ceiling)
+ };
+ let extraction = match utopia_extract::open::parse_open_response(&reply_text) {
+ Ok(x) => x,
+ Err(e) => {
+ tracing::warn!(%document_id, seq = chunk.seq, error = %e, hit_token_ceiling = cut_by_ceiling, "开放抽取回复解析失败,跳过该分块");
+ // 一整块没进图,而原因不同:撞上上限是「答案太长」,改得动;
+ // 别的解析失败是「回复不合结构」。两种都写成同一句话就分不开了
+ let (why, detail) = if cut_by_ceiling {
+ (
+ "回复撞上 token 上限被截断,剩下的解析不了,这一块没有进图",
+ format!("#{}:hit the token ceiling;{e}", chunk.seq),
+ )
+ } else {
+ (
+ "回复解析不了,这一块没有进图",
+ format!("#{}:{e}", chunk.seq),
+ )
+ };
+ drop_signal(
+ state,
+ kb_id,
+ document_id,
+ reason::CHUNK_UNEXTRACTED,
+ why,
+ Some(&detail),
+ )
+ .await;
+ unextracted.push((chunk.seq, format!("回复解析失败:{e}")));
+ return Ok(None);
+ }
+ };
+ if extraction.truncated {
+ drop_signal(
+ state,
+ kb_id,
+ document_id,
+ reason::TRUNCATED_REPLY,
+ if cut_by_ceiling {
+ "the open reply hit the token ceiling; kept up to the last complete item"
+ } else {
+ "the open reply was cut off; kept up to the last complete item"
+ },
+ None,
+ )
+ .await;
+ }
+ if extraction.skipped > 0 {
+ drop_signal(
+ state,
+ kb_id,
+ document_id,
+ reason::MALFORMED_ITEM,
+ &format!(
+ "{} items in the open reply were malformed",
+ extraction.skipped
+ ),
+ None,
+ )
+ .await;
+ }
+ // 名字向量(0041 决定 3 通道 2):这一块里有名字的东西,名字字符串各算一条,
+ // 消解时拿它在同库的名字向量里找近邻。一块一批;算不出来(端点抖了)不拦抽取,
+ // 只是这一块少一条召回通道
+ let name_vecs: HashMap<String, Vec<f32>> = match (settings, embed_client) {
+ (Some(settings), Some(client)) => {
+ let mut wanted: Vec<(String, String)> = Vec::new();
+ let mut seen: HashSet<String> = HashSet::new();
+ for e in &extraction.entities {
+ let n = e.name.trim();
+ if e.named && !n.is_empty() && seen.insert(name_key(n)) {
+ wanted.push((name_key(n), n.to_string()));
+ }
+ }
+ embed_names(state, settings, client, &wanted).await
+ }
+ _ => HashMap::new(),
+ };
+ Ok(Some(ChunkParsed {
+ extraction,
+ name_vecs,
+ }))
+}
+
+/// 一个分块的后半步:把解析出来的回复写成陈述、名字和证据(#588 cut 1)。
+///
+/// **要按分块顺序跑**:前面的块认下的实体(`doc_entities`、`known_by_name`)、描述出来的
+/// 东西(`described`)、待裁决的名字,后面的块都要读。调模型那半步并发之后这里仍然串行。
+///
+/// `pushed`(0054):块本身就是契约,时间词在整块里定位,不在引文里找(0054 决定 4)
+#[allow(clippy::too_many_arguments)]
+async fn apply_open_extraction(
+ state: &AppState,
+ pool: &PgPool,
+ kb_id: Uuid,
+ document_id: Uuid,
+ chunk: &ChunkForExtract,
+ ctx: Option<&[f32]>,
+ extraction: &utopia_extract::open::OpenExtraction,
+ name_vecs: HashMap<String, Vec<f32>>,
+ attested_at: Option<chrono::DateTime<chrono::Utc>>,
+ bound_types: &HashMap<String, Uuid>,
+ pushed: bool,
+ await_nod: bool,
+ proposer: Proposer,
+ pending_count: &mut usize,
+ statement_count: &mut usize,
+ needs_adjudication: &mut bool,
+ human_reviews_found: &mut bool,
+ doc_entities: &mut Vec<(Uuid, String, String)>,
+ known_by_name: &mut HashMap<String, Uuid>,
+ described: &mut HashMap<String, Uuid>,
+ handled_by_name: &mut HashMap<String, Vec<Uuid>>,
+ ambiguous_bare_cache: &mut HashMap<String, Uuid>,
+ touched_names: &mut HashSet<String>,
+) -> anyhow::Result<()> {
+ // ---- 东西:有名字的走身份消解,被描述的建成没有名字事实的实体 ----
+ let mut response_claims: HashMap<String, Vec<Uuid>> = HashMap::new();
+ let mut local: HashMap<String, Uuid> = HashMap::new();
+ // 被描述的东西:先记下名字和类别词,等陈述指到它再建(见 `place`)
+ let mut deferred: HashMap<String, (String, String)> = HashMap::new();
+ for e in &extraction.entities {
+ let name = e.name.trim();
+ if name.is_empty() {
+ continue;
+ }
+ let kind = e.kind.trim();
+ let key = name_key(name);
+ if local.contains_key(&key) {
+ continue;
+ }
+ let id = if e.named {
+ let bound = bound_types
+ .get(&utopia_store::type_bindings::normalize(kind))
+ .copied();
+ let id = resolve_handle(
+ pool,
+ kb_id,
+ bound,
+ name,
+ ctx,
+ name_vecs.get(&key).map(Vec::as_slice),
+ Some(&chunk.text),
+ &mut response_claims,
+ handled_by_name,
+ ambiguous_bare_cache,
+ needs_adjudication,
+ human_reviews_found,
+ )
+ .await?;
+ // 模型说它是什么(「a British film」里的 film):存成它自己的说法,
+ // 类的绑定等对齐来做
+ if !kind.is_empty() {
+ let _ = utopia_store::resolution::set_specific_type(pool, id, kind).await;
+ }
+ // 名字就在这一块原文里时,给名字事实补出处(0041)。记忆日志里的不补:
+ // 那一句算不算出处,要等人点头(0018)
+ if !await_nod && span_in_quote(name, &chunk.text) {
+ let _ = utopia_store::names::record(
+ pool,
+ kb_id,
+ id,
+ name,
+ Some(utopia_store::names::NameSource {
+ chunk_id: chunk.id,
+ quote: name,
+ }),
+ attested_at,
+ )
+ .await;
+ }
+ touched_names.insert(utopia_store::resolution::normalize_name(name).to_lowercase());
+ if !doc_entities.iter().any(|(x, _, _)| *x == id) {
+ doc_entities.push((id, kind.to_string(), name.to_string()));
+ known_by_name.entry(key.clone()).or_insert(id);
+ }
+ id
+ } else {
+ deferred.insert(key, (name.to_string(), kind.to_string()));
+ continue;
+ };
+ local.insert(key, id);
+ }
+
+ // ---- 陈述 ----
+ for s in &extraction.statements {
+ let phrase = s.phrase.trim();
+ if phrase.is_empty() {
+ continue;
+ }
+ let Some(subject) = place(
+ pool,
+ kb_id,
+ &mut local,
+ known_by_name,
+ &deferred,
+ described,
+ &s.subject,
+ )
+ .await?
+ else {
+ drop_signal(
+ state,
+ kb_id,
+ document_id,
+ reason::UNKNOWN_REF,
+ "a statement's subject is not a listed thing",
+ Some(&s.subject),
+ )
+ .await;
+ continue;
+ };
+ // 宾语写了名字但没在清单上:模型漏列了它。陈述照落,宾语落成字面值——
+ // 不凭空建实体,也不丢这条话(#559 的那一档)
+ // 一个字母数字都没有的值(表格里的「—」、空格)什么也没说:按没有值处理。
+ // 这是结构判断,不是词表——看的是有没有内容,不是内容是什么
+ let mut value = s
+ .value
+ .as_deref()
+ .map(str::trim)
+ .filter(|v| v.chars().any(char::is_alphanumeric));
+ // 模型自己写的那个值,在下面被「宾语没声明」顶替之前记下来:引文要核的是它,
+ // 不是一个实体的名字(#729)
+ let stated_value = value;
+ let object = match s.object.as_deref().map(str::trim).filter(|o| !o.is_empty()) {
+ Some(name) => match place(
+ pool,
+ kb_id,
+ &mut local,
+ known_by_name,
+ &deferred,
+ described,
+ name,
+ )
+ .await?
+ {
+ Some(id) => Some(id),
+ None => {
+ drop_signal(
+ state,
+ kb_id,
+ document_id,
+ reason::OBJECT_UNDECLARED,
+ "a statement's object is not a listed thing; written as a value",
+ Some(name),
+ )
+ .await;
+ value = value.or(Some(name));
+ None
+ }
+ },
+ None => None,
+ };
+ // 短语就是值、短语就是主语:表格丢了列头(或没有说明句)时模型的两种写法
+ // (#743 的财报长文里各占一类)。陈述照落——值和主语都在,信息没丢——只记一笔
+ // 让它可量;只比字面,不认词
+ let same = |a: &str, b: &str| a.trim().eq_ignore_ascii_case(b.trim());
+ if value.is_some_and(|v| same(v, phrase)) {
+ drop_signal(
+ state,
+ kb_id,
+ document_id,
+ reason::PHRASE_IS_VALUE,
+ "kept, but the phrase is the value itself",
+ Some(phrase),
+ )
+ .await;
+ } else if same(&s.subject, phrase) {
+ drop_signal(
+ state,
+ kb_id,
+ document_id,
+ reason::PHRASE_IS_SUBJECT,
+ "kept, but the phrase is the subject's own name",
+ Some(phrase),
+ )
+ .await;
+ }
+ let value_json;
+ let fact_object = match (object, value) {
+ (Some(o), _) if o == subject => {
+ drop_signal(
+ state,
+ kb_id,
+ document_id,
+ reason::OBJECT_MISSING,
+ "a statement points at its own subject",
+ Some(phrase),
+ )
+ .await;
+ continue;
+ }
+ (Some(o), _) => FactObject::Entity(o),
+ (None, Some(v)) => {
+ value_json = serde_json::json!({ "value": v });
+ FactObject::Value(&value_json)
+ }
+ (None, None) => {
+ drop_signal(
+ state,
+ kb_id,
+ document_id,
+ reason::OBJECT_MISSING,
+ "a statement has neither an object nor a value",
+ Some(phrase),
+ )
+ .await;
+ continue;
+ }
+ };
+ // 引文定位。定位不到的那句照样当证据文字记,只是没有偏移,并记一笔——
+ // 模型没照抄的句子是「不是原文说的」那一类的苗头,量它
+ let quote_text = s.quote.as_deref().map(str::trim).filter(|q| !q.is_empty());
+ let quote: Option<(&str, Option<(i32, i32)>)> =
+ quote_text.map(|q| (q, locate(&chunk.text, q)));
+ if let Some((q, None)) = quote {
+ drop_signal(
+ state,
+ kb_id,
+ document_id,
+ reason::QUOTE_NOT_IN_CHUNK,
+ "a quoted sentence is not in the chunk verbatim",
+ Some(q),
+ )
+ .await;
+ }
+ // **值必须出现在它自己的引文里**(#729)。一行五列被压成五条只有值不同的
+ // 陈述、引文却是表上面那句导语时,五条里至多一条对,而谁也分不出是哪一条。
+ // 只查带数字的值:实体名当值写的那一类(object_undeclared)不在此列
+ if let (Some(v), Some((q, _))) = (stated_value, quote) {
+ if v.chars().any(|c| c.is_ascii_digit()) && !shows_value(q, v) {
+ drop_signal(
+ state,
+ kb_id,
+ document_id,
+ reason::VALUE_NOT_IN_QUOTE,
+ "a statement's value is not in its own quoted sentence",
+ Some(v),
+ )
+ .await;
+ continue;
+ }
+ }
+ // 开放陈述没有模型自报的置信度:它说的是「文档这么说了」。看图描述出来的块
+ // 照旧压上限(0040 决定 4)
+ let confidence = origin_ceiling(&chunk.origin, 1.0);
+ // 限定和时间词先算好:直接落库和等人点头两条路用同一份。
+ // 限定:值是清单上某个东西的名字就挂实体,否则挂文字
+ let mut qualifiers: Vec<(&str, Option<serde_json::Value>, Option<Uuid>)> = Vec::new();
+ for (role, text) in &s.qualifiers {
+ let (role, text) = (role.trim(), text.trim());
+ if role.is_empty() || text.is_empty() {
+ continue;
+ }
+ match place(
+ pool,
+ kb_id,
+ &mut local,
+ known_by_name,
+ &deferred,
+ described,
+ text,
+ )
+ .await?
+ {
+ Some(id) => qualifiers.push((role, None, Some(id))),
+ None => qualifiers.push((role, Some(serde_json::json!(text)), None)),
+ }
+ }
+ // 时间词:起与止各是一条提及,必须原样在这条陈述的引文里
+ let mut time_words: Vec<(&str, i32, &str)> = Vec::new();
+ for (role, words) in [("when", s.when.as_deref()), ("ended", s.ended.as_deref())]
+ .into_iter()
+ .filter_map(|(role, words)| Some((role, words?.trim())))
+ .filter(|(_, w)| !w.is_empty())
+ {
+ // 推送来的陈述没有引文:条目自己就是证据(0054 决定 4),这一块就是这一份
+ // 载荷,时间词在块里找。走 `locate_time` 会在 `quote?` 上退出,起止就都丢了
+ let located = if pushed {
+ locate(&chunk.text, words).map(|(start, _)| start)
+ } else {
+ locate_time(&chunk.text, quote, words)
+ };
+ match located {
+ Some(start) => time_words.push((words, start, role)),
+ None => {
+ drop_signal(
+ state,
+ kb_id,
+ document_id,
+ reason::TIME_NOT_IN_QUOTE,
+ "a time mention's words are not in the statement's own sentence",
+ Some(words),
+ )
+ .await;
+ }
+ }
+ }
+ if await_nod {
+ // 记忆日志:一切原样进待确认表(0015),人点头时 `pending::confirm` 才把它
+ // 落成开放陈述——同样的短语、限定、时间词、引文偏移
+ let (object_id, object_value) = match fact_object {
+ FactObject::Entity(id) => (Some(id), None),
+ FactObject::Value(v) => (None, Some(v)),
+ };
+ let qualifiers_json = serde_json::Value::Array(
+ qualifiers
+ .iter()
+ .map(|(role, value, entity)| match entity {
+ Some(id) => serde_json::json!({ "role": role, "entity_id": id }),
+ None => serde_json::json!({ "role": role, "value": value }),
+ })
+ .collect(),
+ );
+ let time_json = serde_json::Value::Array(
+ time_words
+ .iter()
+ .map(|(text, start, role)| {
+ serde_json::json!({ "text": text, "char_start": start, "role": role })
+ })
+ .collect(),
+ );
+ let outcome = utopia_store::pending::propose(
+ pool,
+ Proposal {
+ kb_id,
+ subject_id: subject,
+ predicate_id: None,
+ object_id,
+ object_value,
+ proposed_predicate: Some(phrase),
+ validity: Validity {
+ attested_at,
+ ..Default::default()
+ },
+ confidence,
+ chunk_id: chunk.id,
+ proposed_by: proposer.user_id,
+ proposed_token: proposer.token_id,
+ phrase: Some(phrase),
+ qualifiers: Some(&qualifiers_json),
+ time_words: Some(&time_json),
+ quote_span: quote.and_then(|(_, span)| span),
+ },
+ )
+ .await?;
+ if let Outcome::Proposed(_) = outcome {
+ *pending_count += 1;
+ }
+ *statement_count += 1;
+ continue;
+ }
+ let (fact_id, _created) = utopia_store::graph::insert_open_statement(
+ pool,
+ kb_id,
+ subject,
+ phrase,
+ fact_object,
+ attested_at,
+ confidence,
+ )
+ .await?;
+ // 表层谓词也写短语:今天的读路径都从 `fact_surface_predicate` 取名字
+ utopia_store::graph::add_evidence_located(
+ pool,
+ fact_id,
+ chunk.id,
+ quote.map(|(q, _)| q),
+ Some(phrase),
+ quote.and_then(|(_, span)| span),
+ )
+ .await?;
+ for (role, value, entity) in &qualifiers {
+ utopia_store::graph::add_statement_qualifier(
+ pool,
+ fact_id,
+ role,
+ value.as_ref(),
+ *entity,
+ )
+ .await?;
+ }
+ for (words, start, role) in &time_words {
+ utopia_store::time_mentions::record(
+ pool, kb_id, fact_id, chunk.id, words, *start, role,
+ )
+ .await?;
+ }
+ *statement_count += 1;
+ }
+
+ // ---- 别名(0041 决定 2):服务端只核对名字确实在这一块原文里 ----
+ // 记忆日志里的别名不记:那一句算不算出处要等人点头(0018)
+ for n in extraction.names.iter().filter(|_| !await_nod) {
+ let name = n.name.trim();
+ if name.is_empty() {
+ continue;
+ }
+ let Some(id) = place(
+ pool,
+ kb_id,
+ &mut local,
+ known_by_name,
+ &deferred,
+ described,
+ &n.entity,
+ )
+ .await?
+ else {
+ drop_signal(
+ state,
+ kb_id,
+ document_id,
+ reason::UNKNOWN_REF,
+ "a name's thing is not a listed thing",
+ Some(&n.entity),
+ )
+ .await;
+ continue;
+ };
+ if !span_in_quote(name, &chunk.text) {
+ drop_signal(
+ state,
+ kb_id,
+ document_id,
+ reason::NAME_NOT_IN_TEXT,
+ "an extra name is not in the chunk",
+ Some(name),
+ )
+ .await;
+ continue;
+ }
+ // 一个名字不会同时是两样东西的名字:这一块或本文档前面已经把它认成
+ // 另一个实体的,不记
+ let key = utopia_store::resolution::normalize_name(name).to_lowercase();
+ let claimed = handled_by_name
+ .get(&key)
+ .is_some_and(|ids| ids.iter().any(|x| *x != id));
+ if claimed {
+ drop_signal(
+ state,
+ kb_id,
+ document_id,
+ reason::NAME_CLAIMED_BY_ANOTHER,
+ "an extra name is already another thing's name",
+ Some(name),
+ )
+ .await;
+ continue;
+ }
+ let quote = n
+ .quote
+ .as_deref()
+ .map(str::trim)
+ .filter(|q| !q.is_empty())
+ .unwrap_or(&chunk.text);
+ let _ = utopia_store::names::record(
+ pool,
+ kb_id,
+ id,
+ name,
+ Some(utopia_store::names::NameSource {
+ chunk_id: chunk.id,
+ quote,
+ }),
+ attested_at,
+ )
+ .await;
+ touched_names.insert(key);
+ }
+
+ // 本块抽完即打标:更新时被认领的块携带标记跳过;中断的抽取可续跑
+ utopia_store::documents::mark_chunk_extracted(pool, chunk.id).await?;
+
+ Ok(())
+}
+
/// 文档日期只在它来自内容或来源系统时才算证据日期(与 `temporal::DATED_AT` 同一口径)。
pub(crate) fn dated_at(doc: &Document) -> Option<chrono::DateTime<chrono::Utc>> {
if matches!(doc.doc_time_source.as_str(), "content" | "source") {
@@ -291,587 +941,61 @@ pub(crate) async fn run_open(
.as_ref()
.filter(|(id, _)| *id != chunk.id)
.map(|(_, text)| text.as_str());
- // 推送来的陈述:块就是契约,解析它而不是问模型(0054)。下面从解析起一步不变
- let (reply_text, cut_by_ceiling) = if pushed {
- (chunk.text.clone(), false)
+ // 提示词里带不带前面分块认下的实体,是个为了量而设的旋钮(#588);落库那半步不受它管
+ let shown: &[utopia_extract::KnownEntity] = if state.extract_known_in_prompt {
+ &known
} else {
- let (settings, client) = match (settings, client) {
- (Some(s), Some(c)) => (s, c),
- _ => anyhow::bail!("Chat model not configured; cannot extract"),
- };
- let messages = utopia_extract::open::build_open_messages(
- &doc.filename,
- &known,
- opening,
- &chunk.text,
- );
- // 温度 0:照抄原文的活不该靠采样。端点缺省 1.0 时同一块两次回复密度差三倍
- let reply = match chat_retrying_rate_limits_at(
- state,
- settings,
- client,
- &messages,
- Some(0.0),
- )
- .await
- {
- Ok(r) => r,
- Err(e) => {
- tracing::warn!(%document_id, seq = chunk.seq, error = %e, "开放抽取调用失败,跳过该分块");
- drop_signal(
- state,
- kb_id,
- document_id,
- reason::CHUNK_UNEXTRACTED,
- "调用失败,这一块没有进图",
- Some(&format!("#{}:{e}", chunk.seq)),
- )
- .await;
- unextracted.push((chunk.seq, format!("调用失败:{e}")));
- continue;
- }
- };
- tracing::debug!(%document_id, seq = chunk.seq, reply = %reply.text, "开放抽取的原始回复");
- // 端点说它是撞上 token 上限停的。解析器只看得见 JSON 少了尾巴,看不见
- // 少的原因,所以这句话得从回复里带过来(#760)
- let cut_by_ceiling = reply.hit_token_ceiling();
- (reply.text, cut_by_ceiling)
+ &[]
};
- let extraction = match utopia_extract::open::parse_open_response(&reply_text) {
- Ok(x) => x,
- Err(e) => {
- tracing::warn!(%document_id, seq = chunk.seq, error = %e, hit_token_ceiling = cut_by_ceiling, "开放抽取回复解析失败,跳过该分块");
- // 一整块没进图,而原因不同:撞上上限是「答案太长」,改得动;
- // 别的解析失败是「回复不合结构」。两种都写成同一句话就分不开了
- let (why, detail) = if cut_by_ceiling {
- (
- "回复撞上 token 上限被截断,剩下的解析不了,这一块没有进图",
- format!("#{}:hit the token ceiling;{e}", chunk.seq),
- )
- } else {
- (
- "回复解析不了,这一块没有进图",
- format!("#{}:{e}", chunk.seq),
- )
- };
- drop_signal(
- state,
- kb_id,
- document_id,
- reason::CHUNK_UNEXTRACTED,
- why,
- Some(&detail),
- )
- .await;
- unextracted.push((chunk.seq, format!("回复解析失败:{e}")));
- continue;
- }
- };
- if extraction.truncated {
- drop_signal(
- state,
- kb_id,
- document_id,
- reason::TRUNCATED_REPLY,
- if cut_by_ceiling {
- "the open reply hit the token ceiling; kept up to the last complete item"
- } else {
- "the open reply was cut off; kept up to the last complete item"
- },
- None,
- )
- .await;
- }
- if extraction.skipped > 0 {
- drop_signal(
- state,
- kb_id,
- document_id,
- reason::MALFORMED_ITEM,
- &format!(
- "{} items in the open reply were malformed",
- extraction.skipped
- ),
- None,
- )
- .await;
- }
-
- // 名字向量(0041 决定 3 通道 2):这一块里有名字的东西,名字字符串各算一条,
- // 消解时拿它在同库的名字向量里找近邻。一块一批;算不出来(端点抖了)不拦抽取,
- // 只是这一块少一条召回通道
- let name_vecs: HashMap<String, Vec<f32>> = match (settings, &embed) {
- (Some(settings), Some(client)) => {
- let mut wanted: Vec<(String, String)> = Vec::new();
- let mut seen: HashSet<String> = HashSet::new();
- for e in &extraction.entities {
- let n = e.name.trim();
- if e.named && !n.is_empty() && seen.insert(name_key(n)) {
- wanted.push((name_key(n), n.to_string()));
- }
- }
- embed_names(state, settings, client, &wanted).await
- }
- _ => HashMap::new(),
+ let parsed = match call_and_parse_one_chunk(
+ state,
+ settings,
+ client,
+ embed.as_ref(),
+ kb_id,
+ document_id,
+ &doc.filename,
+ chunk,
+ opening,
+ shown,
+ pushed,
+ &mut unextracted,
+ )
+ .await?
+ {
+ Some(p) => p,
+ None => continue,
};
-
- // ---- 东西:有名字的走身份消解,被描述的建成没有名字事实的实体 ----
- let mut response_claims: HashMap<String, Vec<Uuid>> = HashMap::new();
- let mut local: HashMap<String, Uuid> = HashMap::new();
- // 被描述的东西:先记下名字和类别词,等陈述指到它再建(见 `place`)
- let mut deferred: HashMap<String, (String, String)> = HashMap::new();
- for e in &extraction.entities {
- let name = e.name.trim();
- if name.is_empty() {
- continue;
- }
- let kind = e.kind.trim();
- let key = name_key(name);
- if local.contains_key(&key) {
- continue;
- }
- let id = if e.named {
- let bound = bound_types
- .get(&utopia_store::type_bindings::normalize(kind))
- .copied();
- let id = resolve_handle(
- pool,
- kb_id,
- bound,
- name,
- ctx,
- name_vecs.get(&key).map(Vec::as_slice),
- Some(&chunk.text),
- &mut response_claims,
- &mut handled_by_name,
- &mut ambiguous_bare_cache,
- &mut needs_adjudication,
- &mut human_reviews_found,
- )
- .await?;
- // 模型说它是什么(「a British film」里的 film):存成它自己的说法,
- // 类的绑定等对齐来做
- if !kind.is_empty() {
- let _ = utopia_store::resolution::set_specific_type(pool, id, kind).await;
- }
- // 名字就在这一块原文里时,给名字事实补出处(0041)。记忆日志里的不补:
- // 那一句算不算出处,要等人点头(0018)
- if !await_nod && span_in_quote(name, &chunk.text) {
- let _ = utopia_store::names::record(
- pool,
- kb_id,
- id,
- name,
- Some(utopia_store::names::NameSource {
- chunk_id: chunk.id,
- quote: name,
- }),
- attested_at,
- )
- .await;
- }
- touched_names.insert(utopia_store::resolution::normalize_name(name).to_lowercase());
- if !doc_entities.iter().any(|(x, _, _)| *x == id) {
- doc_entities.push((id, kind.to_string(), name.to_string()));
- known_by_name.entry(key.clone()).or_insert(id);
- }
- id
- } else {
- deferred.insert(key, (name.to_string(), kind.to_string()));
- continue;
- };
- local.insert(key, id);
- }
-
- // ---- 陈述 ----
- for s in &extraction.statements {
- let phrase = s.phrase.trim();
- if phrase.is_empty() {
- continue;
- }
- let Some(subject) = place(
- pool,
- kb_id,
- &mut local,
- &known_by_name,
- &deferred,
- &mut described,
- &s.subject,
- )
- .await?
- else {
- drop_signal(
- state,
- kb_id,
- document_id,
- reason::UNKNOWN_REF,
- "a statement's subject is not a listed thing",
- Some(&s.subject),
- )
- .await;
- continue;
- };
- // 宾语写了名字但没在清单上:模型漏列了它。陈述照落,宾语落成字面值——
- // 不凭空建实体,也不丢这条话(#559 的那一档)
- // 一个字母数字都没有的值(表格里的「—」、空格)什么也没说:按没有值处理。
- // 这是结构判断,不是词表——看的是有没有内容,不是内容是什么
- let mut value = s
- .value
- .as_deref()
- .map(str::trim)
- .filter(|v| v.chars().any(char::is_alphanumeric));
- // 模型自己写的那个值,在下面被「宾语没声明」顶替之前记下来:引文要核的是它,
- // 不是一个实体的名字(#729)
- let stated_value = value;
- let object = match s.object.as_deref().map(str::trim).filter(|o| !o.is_empty()) {
- Some(name) => match place(
- pool,
- kb_id,
- &mut local,
- &known_by_name,
- &deferred,
- &mut described,
- name,
- )
- .await?
- {
- Some(id) => Some(id),
- None => {
- drop_signal(
- state,
- kb_id,
- document_id,
- reason::OBJECT_UNDECLARED,
- "a statement's object is not a listed thing; written as a value",
- Some(name),
- )
- .await;
- value = value.or(Some(name));
- None
- }
- },
- None => None,
- };
- // 短语就是值、短语就是主语:表格丢了列头(或没有说明句)时模型的两种写法
- // (#743 的财报长文里各占一类)。陈述照落——值和主语都在,信息没丢——只记一笔
- // 让它可量;只比字面,不认词
- let same = |a: &str, b: &str| a.trim().eq_ignore_ascii_case(b.trim());
- if value.is_some_and(|v| same(v, phrase)) {
- drop_signal(
- state,
- kb_id,
- document_id,
- reason::PHRASE_IS_VALUE,
- "kept, but the phrase is the value itself",
- Some(phrase),
- )
- .await;
- } else if same(&s.subject, phrase) {
- drop_signal(
- state,
- kb_id,
- document_id,
- reason::PHRASE_IS_SUBJECT,
- "kept, but the phrase is the subject's own name",
- Some(phrase),
- )
- .await;
- }
- let value_json;
- let fact_object = match (object, value) {
- (Some(o), _) if o == subject => {
- drop_signal(
- state,
- kb_id,
- document_id,
- reason::OBJECT_MISSING,
- "a statement points at its own subject",
- Some(phrase),
- )
- .await;
- continue;
- }
- (Some(o), _) => FactObject::Entity(o),
- (None, Some(v)) => {
- value_json = serde_json::json!({ "value": v });
- FactObject::Value(&value_json)
- }
- (None, None) => {
- drop_signal(
- state,
- kb_id,
- document_id,
- reason::OBJECT_MISSING,
- "a statement has neither an object nor a value",
- Some(phrase),
- )
- .await;
- continue;
- }
- };
- // 引文定位。定位不到的那句照样当证据文字记,只是没有偏移,并记一笔——
- // 模型没照抄的句子是「不是原文说的」那一类的苗头,量它
- let quote_text = s.quote.as_deref().map(str::trim).filter(|q| !q.is_empty());
- let quote: Option<(&str, Option<(i32, i32)>)> =
- quote_text.map(|q| (q, locate(&chunk.text, q)));
- if let Some((q, None)) = quote {
- drop_signal(
- state,
- kb_id,
- document_id,
- reason::QUOTE_NOT_IN_CHUNK,
- "a quoted sentence is not in the chunk verbatim",
- Some(q),
- )
- .await;
- }
- // **值必须出现在它自己的引文里**(#729)。一行五列被压成五条只有值不同的
- // 陈述、引文却是表上面那句导语时,五条里至多一条对,而谁也分不出是哪一条。
- // 只查带数字的值:实体名当值写的那一类(object_undeclared)不在此列
- if let (Some(v), Some((q, _))) = (stated_value, quote) {
- if v.chars().any(|c| c.is_ascii_digit()) && !shows_value(q, v) {
- drop_signal(
- state,
- kb_id,
- document_id,
- reason::VALUE_NOT_IN_QUOTE,
- "a statement's value is not in its own quoted sentence",
- Some(v),
- )
- .await;
- continue;
- }
- }
- // 开放陈述没有模型自报的置信度:它说的是「文档这么说了」。看图描述出来的块
- // 照旧压上限(0040 决定 4)
- let confidence = origin_ceiling(&chunk.origin, 1.0);
- // 限定和时间词先算好:直接落库和等人点头两条路用同一份。
- // 限定:值是清单上某个东西的名字就挂实体,否则挂文字
- let mut qualifiers: Vec<(&str, Option<serde_json::Value>, Option<Uuid>)> = Vec::new();
- for (role, text) in &s.qualifiers {
- let (role, text) = (role.trim(), text.trim());
- if role.is_empty() || text.is_empty() {
- continue;
- }
- match place(
- pool,
- kb_id,
- &mut local,
- &known_by_name,
- &deferred,
- &mut described,
- text,
- )
- .await?
- {
- Some(id) => qualifiers.push((role, None, Some(id))),
- None => qualifiers.push((role, Some(serde_json::json!(text)), None)),
- }
- }
- // 时间词:起与止各是一条提及,必须原样在这条陈述的引文里
- let mut time_words: Vec<(&str, i32, &str)> = Vec::new();
- for (role, words) in [("when", s.when.as_deref()), ("ended", s.ended.as_deref())]
- .into_iter()
- .filter_map(|(role, words)| Some((role, words?.trim())))
- .filter(|(_, w)| !w.is_empty())
- {
- // 推送来的陈述没有引文:条目自己就是证据(0054 决定 4),这一块就是这一份
- // 载荷,时间词在块里找。走 `locate_time` 会在 `quote?` 上退出,起止就都丢了
- let located = if pushed {
- locate(&chunk.text, words).map(|(start, _)| start)
- } else {
- locate_time(&chunk.text, quote, words)
- };
- match located {
- Some(start) => time_words.push((words, start, role)),
- None => {
- drop_signal(
- state,
- kb_id,
- document_id,
- reason::TIME_NOT_IN_QUOTE,
- "a time mention's words are not in the statement's own sentence",
- Some(words),
- )
- .await;
- }
- }
- }
- if await_nod {
- // 记忆日志:一切原样进待确认表(0015),人点头时 `pending::confirm` 才把它
- // 落成开放陈述——同样的短语、限定、时间词、引文偏移
- let (object_id, object_value) = match fact_object {
- FactObject::Entity(id) => (Some(id), None),
- FactObject::Value(v) => (None, Some(v)),
- };
- let qualifiers_json = serde_json::Value::Array(
- qualifiers
- .iter()
- .map(|(role, value, entity)| match entity {
- Some(id) => serde_json::json!({ "role": role, "entity_id": id }),
- None => serde_json::json!({ "role": role, "value": value }),
- })
- .collect(),
- );
- let time_json = serde_json::Value::Array(
- time_words
- .iter()
- .map(|(text, start, role)| {
- serde_json::json!({ "text": text, "char_start": start, "role": role })
- })
- .collect(),
- );
- let outcome = utopia_store::pending::propose(
- pool,
- Proposal {
- kb_id,
- subject_id: subject,
- predicate_id: None,
- object_id,
- object_value,
- proposed_predicate: Some(phrase),
- validity: Validity {
- attested_at,
- ..Default::default()
- },
- confidence,
- chunk_id: chunk.id,
- proposed_by: proposer.user_id,
- proposed_token: proposer.token_id,
- phrase: Some(phrase),
- qualifiers: Some(&qualifiers_json),
- time_words: Some(&time_json),
- quote_span: quote.and_then(|(_, span)| span),
- },
- )
- .await?;
- if let Outcome::Proposed(_) = outcome {
- pending_count += 1;
- }
- statement_count += 1;
- continue;
- }
- let (fact_id, _created) = utopia_store::graph::insert_open_statement(
- pool,
- kb_id,
- subject,
- phrase,
- fact_object,
- attested_at,
- confidence,
- )
- .await?;
- // 表层谓词也写短语:今天的读路径都从 `fact_surface_predicate` 取名字
- utopia_store::graph::add_evidence_located(
- pool,
- fact_id,
- chunk.id,
- quote.map(|(q, _)| q),
- Some(phrase),
- quote.and_then(|(_, span)| span),
- )
- .await?;
- for (role, value, entity) in &qualifiers {
- utopia_store::graph::add_statement_qualifier(
- pool,
- fact_id,
- role,
- value.as_ref(),
- *entity,
- )
- .await?;
- }
- for (words, start, role) in &time_words {
- utopia_store::time_mentions::record(
- pool, kb_id, fact_id, chunk.id, words, *start, role,
- )
- .await?;
- }
- statement_count += 1;
- }
-
- // ---- 别名(0041 决定 2):服务端只核对名字确实在这一块原文里 ----
- // 记忆日志里的别名不记:那一句算不算出处要等人点头(0018)
- for n in extraction.names.iter().filter(|_| !await_nod) {
- let name = n.name.trim();
- if name.is_empty() {
- continue;
- }
- let Some(id) = place(
- pool,
- kb_id,
- &mut local,
- &known_by_name,
- &deferred,
- &mut described,
- &n.entity,
- )
- .await?
- else {
- drop_signal(
- state,
- kb_id,
- document_id,
- reason::UNKNOWN_REF,
- "a name's thing is not a listed thing",
- Some(&n.entity),
- )
- .await;
- continue;
- };
- if !span_in_quote(name, &chunk.text) {
- drop_signal(
- state,
- kb_id,
- document_id,
- reason::NAME_NOT_IN_TEXT,
- "an extra name is not in the chunk",
- Some(name),
- )
- .await;
- continue;
- }
- // 一个名字不会同时是两样东西的名字:这一块或本文档前面已经把它认成
- // 另一个实体的,不记
- let key = utopia_store::resolution::normalize_name(name).to_lowercase();
- let claimed = handled_by_name
- .get(&key)
- .is_some_and(|ids| ids.iter().any(|x| *x != id));
- if claimed {
- drop_signal(
- state,
- kb_id,
- document_id,
- reason::NAME_CLAIMED_BY_ANOTHER,
- "an extra name is already another thing's name",
- Some(name),
- )
- .await;
- continue;
- }
- let quote = n
- .quote
- .as_deref()
- .map(str::trim)
- .filter(|q| !q.is_empty())
- .unwrap_or(&chunk.text);
- let _ = utopia_store::names::record(
- pool,
- kb_id,
- id,
- name,
- Some(utopia_store::names::NameSource {
- chunk_id: chunk.id,
- quote,
- }),
- attested_at,
- )
- .await;
- touched_names.insert(key);
- }
-
- // 本块抽完即打标:更新时被认领的块携带标记跳过;中断的抽取可续跑
- utopia_store::documents::mark_chunk_extracted(pool, chunk.id).await?;
+ let ChunkParsed {
+ extraction,
+ name_vecs,
+ } = parsed;
+ apply_open_extraction(
+ state,
+ pool,
+ kb_id,
+ document_id,
+ chunk,
+ ctx,
+ &extraction,
+ name_vecs,
+ attested_at,
+ &bound_types,
+ pushed,
+ await_nod,
+ proposer,
+ &mut pending_count,
+ &mut statement_count,
+ &mut needs_adjudication,
+ &mut human_reviews_found,
+ &mut doc_entities,
+ &mut known_by_name,
+ &mut described,
+ &mut handled_by_name,
+ &mut ambiguous_bare_cache,
+ &mut touched_names,
+ )
+ .await?;
}
// 消歧后缀在实体创建时算会早于其事实写入——收尾时对本文档涉及的名字统一刷新
@@ -946,6 +1070,10 @@ pub(crate) async fn run_open(
Ok(())
}
+#[cfg(test)]
+#[path = "extraction_open_known_flow_tests.rs"]
+mod known_flow_tests;
+
#[cfg(test)]
mod tests {
use super::*;
diff --git a/crates/utopia-server/src/extraction_open_known_flow_tests.rs b/crates/utopia-server/src/extraction_open_known_flow_tests.rs
new file mode 100644
index 00000000..73719408
--- /dev/null
+++ b/crates/utopia-server/src/extraction_open_known_flow_tests.rs
@@ -0,0 +1,216 @@
+//! 前面的分块认下的实体,后面的分块认得(#588 cut 1)。
+//!
+//! 抽取按分块走:每一块的提示词里列着这篇文档前面的块已经认下的实体(`known`),模型
+//! 在后面的块里提到它,不必再列一遍,落库时认回同一行。分块并发调模型(cut 2)要保住的
+//! 就是这一条,所以先把它钉住:第一块列出 `Acme`,第二块的回复**不列**它、只在陈述里
+//! 提到——两条陈述挂在同一个实体上,而且第二块的提示词里有 `- Acme (organization)`。
+//! 第二块要是自己再列一遍 `Acme`,按名字去库里也认得回来,那测的就不是 `known` 了。
+//!
+//! `extract_known_in_prompt` 关掉之后(召回台子量并发的影响用它)提示词里没有那一行,
+//! 落库照样认回去。
+//!
+//! 脚本化的模型端点、真库,与 `phrase_alignment_tests.rs` 同一个做法。
+//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆,绝不碰已有的库。
+use super::*;
+use axum::{extract::State, response::IntoResponse, routing::post, Json, Router};
+use serde_json::{json, Value};
+use std::sync::{Arc, Mutex};
+use uuid::Uuid;
+
+#[derive(Clone)]
+struct Model {
+ replies: Arc<Mutex<Vec<String>>>,
+ requests: Arc<Mutex<Vec<Value>>>,
+}
+
+/// 开放抽取的回复:脚本里的字符串就是 chat 流里的 `delta.content`。
+/// 与 `errata_tests.rs::reply` 同一档(text/event-stream + `usage` 收尾),
+/// 但 `chat_retrying_rate_limits_at` 解析流时只读 `content` 这一段。
+async fn reply(State(m): State<Model>, Json(body): Json<Value>) -> impl IntoResponse {
+ m.requests.lock().unwrap().push(body);
+ let text = {
+ let mut replies = m.replies.lock().unwrap();
+ if replies.is_empty() {
+ panic!("unexpected model request");
+ }
+ replies.remove(0)
+ };
+ let frame = json!({"choices":[{"delta":{"content":text}}]});
+ let done = json!({"choices":[{"delta":{},"finish_reason":"stop"}]});
+ let usage = json!({"choices":[],"usage":{"prompt_tokens":12,"completion_tokens":34}});
+ (
+ [("content-type", "text/event-stream")],
+ format!("data: {frame}\n\ndata: {done}\n\ndata: {usage}\n\ndata: [DONE]\n\n"),
+ )
+}
+
+struct Fx {
+ pool: sqlx::PgPool,
+ state: AppState,
+ org: Uuid,
+ kb: Uuid,
+ doc: Uuid,
+ model: Model,
+ _server: tokio::task::JoinHandle<()>,
+ _dir: tempfile::TempDir,
+}
+
+impl Fx {
+ async fn new(replies: Vec<String>, known_in_prompt: bool) -> anyhow::Result<Option<Self>> {
+ let Some(url) = utopia_store::test_db::url() else {
+ return Ok(None);
+ };
+ let pool = sqlx::PgPool::connect(&url).await?;
+ utopia_store::db::migrate(&pool).await?;
+ let ids: Vec<Uuid> = (0..6).map(|_| Uuid::now_v7()).collect();
+ let (org, ws, kb, doc, chunk0, chunk1) = (ids[0], ids[1], ids[2], ids[3], ids[4], ids[5]);
+ // 插进去的只有本地生成的 UUID
+ sqlx::raw_sql(&format!(
+ "INSERT INTO organizations(id,name) VALUES ('{org}','open-known-flow');
+ INSERT INTO workspaces(id,org_id,name) VALUES ('{ws}','{org}','open-known-flow');
+ INSERT INTO knowledge_bases(id,workspace_id,name) VALUES ('{kb}','{ws}','open-known-flow');
+ INSERT INTO documents(id,kb_id,filename,sha256) VALUES ('{doc}','{kb}','acme.txt','x');
+ INSERT INTO chunks(id,kb_id,document_id,seq,text) VALUES
+ ('{chunk0}','{kb}','{doc}',0,'Acme is based in London.'),
+ ('{chunk1}','{kb}','{doc}',1,'Acme runs the harbour facility.');"
+ ))
+ .execute(&pool)
+ .await?;
+ let model = Model {
+ replies: Arc::new(Mutex::new(replies)),
+ requests: Arc::new(Mutex::new(Vec::new())),
+ };
+ let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await?;
+ let endpoint = format!("http://{}", listener.local_addr()?);
+ let router = Router::new()
+ .route("/chat/completions", post(reply))
+ .with_state(model.clone());
+ let server = tokio::spawn(async move { axum::serve(listener, router).await.unwrap() });
+ utopia_store::settings::upsert(
+ &pool,
+ ws,
+ Some(&endpoint),
+ None,
+ Some("scripted"),
+ None,
+ None,
+ None,
+ None,
+ )
+ .await?;
+ let dir = tempfile::tempdir()?;
+ let cfg = utopia_core::config::AppConfig {
+ data_dir: dir.path().to_string_lossy().into_owned(),
+ extract_known_in_prompt: known_in_prompt,
+ ..Default::default()
+ };
+ let search = Arc::new(utopia_search::SearchIndex::open(
+ &dir.path().join("search"),
+ )?);
+ let state = AppState::new(pool.clone(), &cfg, search, "test-only".into());
+ Ok(Some(Self {
+ pool,
+ state,
+ org,
+ kb,
+ doc,
+ model,
+ _server: server,
+ _dir: dir,
+ }))
+ }
+
+ async fn cleanup(&self) -> anyhow::Result<()> {
+ sqlx::query("DELETE FROM organizations WHERE id=$1")
+ .bind(self.org)
+ .execute(&self.pool)
+ .await?;
+ Ok(())
+ }
+}
+
+/// 两块的回复:第一块列出 `Acme`,第二块不列、只在陈述里提到它。
+/// 紧凑数组的 8 格是 `[quote, subject, phrase, object, value, qualifiers, when, ended]`
+fn replies() -> Vec<String> {
+ vec![
+ json!({
+ "e": [["Acme", "organization", 1]],
+ "s": [["Acme is based in London.", "Acme", "based in", null, "London", null, null, null]],
+ "n": []
+ })
+ .to_string(),
+ json!({
+ "e": [],
+ "s": [["Acme runs the harbour facility.", "Acme", "runs", null, "harbour", null, null, null]],
+ "n": []
+ })
+ .to_string(),
+ ]
+}
+
+/// 跑完抽取:只有一个 `Acme`,两条陈述都挂在它上面。交回第二块的提示词
+async fn extract_and_read_the_second_prompt(f: &Fx) -> anyhow::Result<String> {
+ crate::extraction::extract_document(&f.state, f.doc, utopia_core::models::Proposer::default())
+ .await?;
+ let acme: Vec<Uuid> =
+ sqlx::query_scalar("SELECT id FROM entities WHERE kb_id=$1 AND canonical_name='Acme'")
+ .bind(f.kb)
+ .fetch_all(&f.pool)
+ .await?;
+ anyhow::ensure!(
+ acme.len() == 1,
+ "one Acme across both chunks, got {}",
+ acme.len()
+ );
+ let subjects: Vec<Uuid> =
+ sqlx::query_scalar("SELECT subject_id FROM facts WHERE kb_id=$1 AND layer='open'")
+ .bind(f.kb)
+ .fetch_all(&f.pool)
+ .await?;
+ anyhow::ensure!(
+ subjects == vec![acme[0], acme[0]],
+ "both statements hang on the one Acme: {subjects:?}"
+ );
+ let requests = f.model.requests.lock().unwrap();
+ anyhow::ensure!(
+ requests.len() == 2,
+ "one request a chunk, got {}",
+ requests.len()
+ );
+ Ok(requests[1]["messages"][1]["content"]
+ .as_str()
+ .unwrap_or("")
+ .to_string())
+}
+
+#[tokio::test]
+async fn a_thing_listed_by_an_earlier_chunk_reaches_a_later_one_via_known() -> anyhow::Result<()> {
+ let Some(f) = Fx::new(replies(), true).await? else {
+ return Ok(());
+ };
+ let run = extract_and_read_the_second_prompt(&f).await;
+ f.cleanup().await?;
+ let prompt = run?;
+ assert!(
+ prompt.contains("- Acme (organization)"),
+ "the second chunk is told what the first one listed: {prompt}"
+ );
+ Ok(())
+}
+
+/// 旋钮关掉:第二块的提示词里不再列前面认下的实体,落库仍然认回同一个 `Acme`
+#[tokio::test]
+async fn with_known_left_out_of_the_prompt_a_later_chunk_still_lands_on_the_same_entity(
+) -> anyhow::Result<()> {
+ let Some(f) = Fx::new(replies(), false).await? else {
+ return Ok(());
+ };
+ let run = extract_and_read_the_second_prompt(&f).await;
+ f.cleanup().await?;
+ let prompt = run?;
+ assert!(
+ !prompt.contains("- Acme (organization)"),
+ "nothing listed as known: {prompt}"
+ );
+ Ok(())
+}
diff --git a/crates/utopia-server/src/state.rs b/crates/utopia-server/src/state.rs
index b2efbbfc..e2fb48ef 100644
--- a/crates/utopia-server/src/state.rs
+++ b/crates/utopia-server/src/state.rs
@@ -31,6 +31,8 @@ pub struct AppState {
pub cookie_secure: bool,
/// 分块预算,见 `AppConfig::chunk_tokens`
pub chunk_tokens: usize,
+ /// 见 `AppConfig::extract_known_in_prompt`
+ pub extract_known_in_prompt: bool,
/// worker 并发数:调度循环每轮热读——系统设置改动即时生效
pub worker_concurrency: Arc<std::sync::atomic::AtomicUsize>,
/// 各工作区的对话客户端。端点报过的上下文窗口记在客户端上,留着它下一轮才记得(#964)
@@ -63,6 +65,7 @@ impl AppState {
open_registration: cfg.open_registration,
cookie_secure: cfg.cookie_secure,
chunk_tokens: cfg.chunk_tokens,
+ extract_known_in_prompt: cfg.extract_known_in_prompt,
// 与 `deployment_settings.worker_concurrency` 的列缺省保持一致(迁移 0011):
// access::worker_concurrency 的「行不存在」兜底是 64,main.rs 的「函数本身
// 报错」兜底也是 64——这是**服务起来后第一次读这个值之前**的值,写错了
diff --git a/scripts/bench/README.md b/scripts/bench/README.md
index 13347be8..42b58e9f 100644
--- a/scripts/bench/README.md
+++ b/scripts/bench/README.md
@@ -68,6 +68,10 @@ node scripts/bench/recall.mjs --kb <id> --reprocess # 改了解析器
node scripts/bench/judge_open.mjs --kb <id> --sample 200 # 开放陈述有多少不是原文说的(门槛 2%)
```
+量「后面的分块看不到前面的分块认下了什么」对输出的影响(#588,分块并发调模型之前要知道的数):
+服务端带 `UTOPIA_EXTRACT_KNOWN_IN_PROMPT=false` 起,照常跑 `recall.mjs`,与不带时的几轮比。
+这个开关只动提示词,落库时按名字认回前面分块的实体照旧。
+
抽取写的是开放图谱(0044 决定 2,没有带本体的第二条路):陈述按原文短语落成 `layer='open'` 的
事实行,限定挂在 `statement_qualifiers`,时间词进 `time_mentions`;打分口径照旧(谓词取
`proposed_predicate`,限定也拼进那一行),所以下表里带本体那些轮次的分数仍然可比。`judge_open.mjs` 是