Skip to content

World's first open-source enterprise world model.

55.8 KB
Raw Blame
//! 开放图谱的写入路径(0044 第 1 刀,#729):文档说了什么,就按它自己的话记下来。
//!
//! 和 `extraction::run` 的分别只有一处:**提示词里没有本体**,模型不选关系、不选类、
//! 不算日期。回复里是它提到的东西(`e`,有名字的和只被描述的)、它做的陈述(`s`,关系
//! 短语照抄,主宾按名字写,时间词照抄,引文整句照抄)和别名(`n`)。落库时陈述成
//! `layer = 'open'` 的事实行,短语留在行上;限定按文档自己的角色词挂在
//! `statement_qualifiers`;时间词原样进 `time_mentions`,谁也不把它算成日期——那是
//! 0045 的事。类型化的事实由对齐(第 2 刀)从这些行算出来,不在这里写。
//!
//! **陈述按名字指东西,不按编号。** 第一版让 `e` 带编号、陈述写编号、引文按句号索引,
//! 省的是输出 token;实测 deepseek-v4-flash 在密的段落里会把编号对错——同一块两次回复
//! 一次「NVIDIA has reached AI」一次「tokens are tokens」,原文说的是「AI has reached its
//! inflection point」。原型按名字写主宾、每条陈述抄引文,333 条里编造 0 条。忠实先于省钱
//!
//! 复用的是身份那一段:名字照旧走 `resolve_handle`(同一回复里两个同名的东西不会
//! 塌成一个,0041 的名字事实照记),被描述的东西建成没有名字事实的实体
//! (`create_described`),免得「一家医院」成了召回的桥。
//!
//! 两根时间轴(0022 / #714 / 0064):陈述入库时的见证只认来源系统给文档的日期,没有就
//! 留空——处理文档的时刻不是文档说的日期。文档自己说的日期(各块报上来的 `t`)在抽完
//! 之后由时间解析那一步按节写到陈述上(`time_resolution::attest_statements`)。

use crate::extraction::{
    chat_retrying_rate_limits_at, drop_signal, incomplete_reason, origin_ceiling, resolve_handle,
    span_in_quote,
};
use crate::state::AppState;
use sqlx::PgPool;
use std::collections::{HashMap, HashSet};
use utopia_core::models::{Document, KnowledgeBase, LlmSettings, Proposer};
use utopia_extract::KnownEntity;
use utopia_store::documents::ChunkForExtract;
use utopia_store::extraction_drops::reason;
use utopia_store::graph::FactObject;
use utopia_store::graph::Validity;
use utopia_store::pending::{Outcome, Proposal};
pub(crate) use utopia_store::resolution::name_key;
use uuid::Uuid;

/// 一块调 + 解析完后的中间产物,apply 半步读它落库(#588 cut 1)。
/// `name_vecs`(0041 决定 3 通道 2)紧跟解析算出来——apply 里的 `resolve_handle`
/// 立刻就要它,分两处算只会让 helper 之间传更多参数。
struct ChunkParsed {
    extraction: utopia_extract::open::OpenExtraction,
    name_vecs: HashMap<String, Vec<f32>>,
}

/// 回复为什么是半截的:端点撞上了 token 上限(#760),或护栏看见它在复读、停了读(#1301)。
/// 两种都当半截处理,丢弃行要说清是哪一种——复读的去查上限,是找错了地方
#[derive(Debug, Clone, Copy)]
enum Cut {
    Ceiling,
    Repeating,
}

/// 一个分块的前半步:调模型、解析回复、算名字向量(#588 cut 1)。这半步不写图,
/// 所以分块之间可以不按顺序做——并发(cut 2)动的就是它;后半步 `apply_open_extraction`
/// 要按分块顺序。`pushed`(0054)的块本身就是契约,不调模型,直接解析。
///
/// `Ok(None)` 是这一块跳过了:调用失败或回复解析不出,原因已经记进 `drop_signal` 和
/// `unextracted`,主循环接着走下一块。
///
/// `Err` has two causes: no chat model, or an empty balance (#1095). An empty balance
/// stops the document, and the error keeps its chain for `alerting::hopeless`.
///
/// 被接管的检查(epoch)留在主循环顶上,不在这里:它要在调模型之前
#[allow(clippy::too_many_arguments)]
async fn call_and_parse_one_chunk(
    state: &AppState,
    settings: Option<&LlmSettings>,
    client: Option<&utopia_llm::LlmClient>,
    embed_client: Option<&utopia_llm::LlmClient>,
    kb_id: Uuid,
    document_id: Uuid,
    filename: &str,
    chunk: &ChunkForExtract,
    opening: Option<&str>,
    known: &[KnownEntity],
    pushed: bool,
    unextracted: &mut Vec<(i32, String)>,
) -> anyhow::Result<Option<ChunkParsed>> {
    // 推送来的陈述:块就是契约,解析它而不是问模型(0054)。下面从解析起一步不变
    let (reply_text, cut) = if pushed {
        (chunk.text.clone(), None)
    } else {
        let (settings, client) = match (settings, client) {
            (Some(s), Some(c)) => (s, c),
            _ => anyhow::bail!("Chat model not configured; cannot extract"),
        };
        let messages =
            utopia_extract::open::build_open_messages(filename, known, opening, &chunk.text);
        // 温度 0:照抄原文的活不该靠采样。端点缺省 1.0 时同一块两次回复密度差三倍
        let reply = match chat_retrying_rate_limits_at(
            state,
            settings,
            client,
            &messages,
            Some(0.0),
        )
        .await
        {
            Ok(r) => r,
            Err(e) if utopia_llm::out_of_credit(&e).is_some() => return Err(e),
            Err(e) => {
                tracing::warn!(%document_id, seq = chunk.seq, error = %e, "开放抽取调用失败,跳过该分块");
                drop_signal(
                    state,
                    kb_id,
                    document_id,
                    reason::CHUNK_UNEXTRACTED,
                    "request failed; this chunk did not enter the graph",
                    Some(&format!("#{}: {e}", chunk.seq)),
                )
                .await;
                unextracted.push((chunk.seq, format!("request failed: {e}")));
                return Ok(None);
            }
        };
        tracing::debug!(%document_id, seq = chunk.seq, reply = %reply.text, "开放抽取的原始回复");
        // 解析器只看得见 JSON 少了尾巴,看不见少的原因,所以原因得从回复里带过来(#760)
        let cut = if reply.repeating {
            Some(Cut::Repeating)
        } else if reply.hit_token_ceiling() {
            Some(Cut::Ceiling)
        } else {
            None
        };
        (reply.text, cut)
    };
    let mut extraction = match utopia_extract::open::parse_open_response(&reply_text) {
        Ok(x) => x,
        Err(e) => {
            tracing::warn!(%document_id, seq = chunk.seq, error = %e, hit_token_ceiling = matches!(cut, Some(Cut::Ceiling)), repeating = matches!(cut, Some(Cut::Repeating)), "开放抽取回复解析失败,跳过该分块");
            // 一整块没进图,而原因不同:撞上上限是「答案太长」,改得动;复读是模型卡在一行上;
            // 别的解析失败是「回复不合结构」。写成同一句话就分不开了
            let (why, detail) = match cut {
                Some(Cut::Ceiling) => (
                    "response hit the token ceiling and could not be parsed; this chunk did not enter the graph",
                    format!("#{}: hit the token ceiling; {e}", chunk.seq),
                ),
                Some(Cut::Repeating) => (
                    "response started repeating itself and could not be parsed; this chunk did not enter the graph",
                    format!("#{}: started repeating itself; {e}", chunk.seq),
                ),
                None => (
                    "response could not be parsed; this chunk did not enter the graph",
                    format!("#{}: {e}", chunk.seq),
                ),
            };
            drop_signal(
                state,
                kb_id,
                document_id,
                reason::CHUNK_UNEXTRACTED,
                why,
                Some(&detail),
            )
            .await;
            unextracted.push((chunk.seq, format!("response parsing failed: {e}")));
            return Ok(None);
        }
    };
    // 模型把值写成了宾语的,挪回值并各记一笔;推送来的是契约,不动(0054)
    if !pushed {
        for name in utopia_extract::open::values_are_not_things(&mut extraction) {
            drop_signal(
                state,
                kb_id,
                document_id,
                reason::OBJECT_IS_VALUE,
                "kept, with the object read as a value",
                Some(&name),
            )
            .await;
        }
    }
    if extraction.truncated {
        drop_signal(
            state,
            kb_id,
            document_id,
            reason::TRUNCATED_REPLY,
            match cut {
                Some(Cut::Ceiling) => {
                    "the open reply hit the token ceiling; kept up to the last complete item"
                }
                Some(Cut::Repeating) => {
                    "the reply started repeating itself; kept up to the last complete item"
                }
                // 回复正常结束而整体不合法:模型写坏了括号。逐条读出来的都收了,写坏的
                // 条数在下面那条信号里
                None => "the open reply was not valid JSON as a whole; read item by item",
            },
            None,
        )
        .await;
    }
    if extraction.skipped > 0 {
        drop_signal(
            state,
            kb_id,
            document_id,
            reason::MALFORMED_ITEM,
            &format!(
                "{} items in the open reply were malformed",
                extraction.skipped
            ),
            None,
        )
        .await;
    }
    // 名字向量(0041 决定 3 通道 2):这一块里有名字的东西,名字字符串各算一条,
    // 消解时拿它在同库的名字向量里找近邻。一块一批;算不出来(端点抖了)不拦抽取,
    // 只是这一块少一条召回通道
    let name_vecs: HashMap<String, Vec<f32>> = match (settings, embed_client) {
        (Some(settings), Some(client)) => {
            let mut wanted: Vec<(String, String)> = Vec::new();
            let mut seen: HashSet<String> = HashSet::new();
            for e in &extraction.entities {
                let n = e.name.trim();
                if e.named && !n.is_empty() && seen.insert(name_key(n)) {
                    wanted.push((name_key(n), n.to_string()));
                }
            }
            embed_names(state, settings, client, &wanted).await
        }
        _ => HashMap::new(),
    };
    Ok(Some(ChunkParsed {
        extraction,
        name_vecs,
    }))
}

/// 一个分块的后半步:把解析出来的回复写成陈述、名字和证据(#588 cut 1)。
///
/// **要按分块顺序跑**:前面的块认下的实体(`doc_entities`、`known_by_name`)、描述出来的
/// 东西(`described`)、待裁决的名字,后面的块都要读。调模型那半步并发之后这里仍然串行。
///
/// `pushed`(0054):块本身就是契约,时间词在整块里定位,不在引文里找(0054 决定 4)
#[allow(clippy::too_many_arguments)]
async fn apply_open_extraction(
    state: &AppState,
    pool: &PgPool,
    kb_id: Uuid,
    document_id: Uuid,
    chunk: &ChunkForExtract,
    ctx: Option<&[f32]>,
    extraction: &utopia_extract::open::OpenExtraction,
    name_vecs: HashMap<String, Vec<f32>>,
    attested_at: Option<chrono::DateTime<chrono::Utc>>,
    bound_types: &HashMap<String, Uuid>,
    pushed: bool,
    await_nod: bool,
    proposer: Proposer,
    pending_count: &mut usize,
    statement_count: &mut usize,
    needs_adjudication: &mut bool,
    human_reviews_found: &mut bool,
    doc_entities: &mut Vec<(Uuid, String, String)>,
    known_by_name: &mut HashMap<String, Uuid>,
    described: &mut HashMap<String, Uuid>,
    handled_by_name: &mut HashMap<String, Vec<Uuid>>,
    ambiguous_bare_cache: &mut HashMap<String, Uuid>,
    touched_names: &mut HashSet<String>,
) -> anyhow::Result<()> {
    // ---- 东西:有名字的走身份消解,被描述的建成没有名字事实的实体 ----
    let mut response_claims: HashMap<String, Vec<Uuid>> = HashMap::new();
    let mut local: HashMap<String, Uuid> = HashMap::new();
    // 被描述的东西:先记下名字和类别词,等陈述指到它再建(见 `place`)
    let mut deferred: HashMap<String, (String, String)> = HashMap::new();
    for e in &extraction.entities {
        let name = e.name.trim();
        if name.is_empty() {
            continue;
        }
        let kind = e.kind.trim();
        let key = name_key(name);
        if local.contains_key(&key) {
            continue;
        }
        let id = if e.named {
            let bound = bound_types
                .get(&utopia_store::type_bindings::normalize(kind))
                .copied();
            let id = resolve_handle(
                pool,
                kb_id,
                bound,
                name,
                ctx,
                name_vecs.get(&key).map(Vec::as_slice),
                Some(&chunk.text),
                &mut response_claims,
                handled_by_name,
                ambiguous_bare_cache,
                needs_adjudication,
                human_reviews_found,
            )
            .await?;
            // 模型说它是什么(「a British film」里的 film):存成它自己的说法,
            // 类的绑定等对齐来做
            if !kind.is_empty() {
                let _ = utopia_store::resolution::set_specific_type(pool, id, kind).await;
            }
            // 名字就在这一块原文里时,给名字事实补出处(0041)。记忆日志里的不补:
            // 那一句算不算出处,要等人点头(0018)
            if !await_nod && span_in_quote(name, &chunk.text) {
                let _ = utopia_store::names::record(
                    pool,
                    kb_id,
                    id,
                    name,
                    Some(utopia_store::names::NameSource {
                        chunk_id: chunk.id,
                        quote: name,
                    }),
                    attested_at,
                )
                .await;
            }
            touched_names.insert(name_key(name));
            if !doc_entities.iter().any(|(x, _, _)| *x == id) {
                doc_entities.push((id, kind.to_string(), name.to_string()));
                known_by_name.entry(key.clone()).or_insert(id);
            }
            id
        } else {
            deferred.insert(key, (name.to_string(), kind.to_string()));
            continue;
        };
        local.insert(key, id);
    }

    // ---- 陈述 ----
    for s in &extraction.statements {
        let phrase = s.phrase.trim();
        if phrase.is_empty() {
            continue;
        }
        let Some(subject) = place(
            pool,
            kb_id,
            &mut local,
            known_by_name,
            &deferred,
            described,
            &s.subject,
        )
        .await?
        else {
            drop_signal(
                state,
                kb_id,
                document_id,
                reason::UNKNOWN_REF,
                "a statement's subject is not a listed thing",
                Some(&s.subject),
            )
            .await;
            continue;
        };
        // 宾语写了名字但没在清单上:模型漏列了它。陈述照落,宾语落成字面值——
        // 不凭空建实体,也不丢这条话(#559 的那一档)
        // 一个字母数字都没有的值(表格里的「—」、空格)什么也没说:按没有值处理。
        // 这是结构判断,不是词表——看的是有没有内容,不是内容是什么
        let mut value = s
            .value
            .as_deref()
            .map(str::trim)
            .filter(|v| v.chars().any(char::is_alphanumeric));
        // 模型自己写的那个值,在下面被「宾语没声明」顶替之前记下来:引文要核的是它,
        // 不是一个实体的名字(#729)
        let stated_value = value;
        let object = match s.object.as_deref().map(str::trim).filter(|o| !o.is_empty()) {
            Some(name) => match place(
                pool,
                kb_id,
                &mut local,
                known_by_name,
                &deferred,
                described,
                name,
            )
            .await?
            {
                Some(id) => Some(id),
                None => {
                    drop_signal(
                        state,
                        kb_id,
                        document_id,
                        reason::OBJECT_UNDECLARED,
                        "a statement's object is not a listed thing; written as a value",
                        Some(name),
                    )
                    .await;
                    value = value.or(Some(name));
                    None
                }
            },
            None => None,
        };
        // 短语就是值、短语就是主语:表格丢了列头(或没有说明句)时模型的两种写法
        // (#743 的财报长文里各占一类)。陈述照落——值和主语都在,信息没丢——只记一笔
        // 让它可量;只比字面,不认词
        let same = |a: &str, b: &str| a.trim().eq_ignore_ascii_case(b.trim());
        if value.is_some_and(|v| same(v, phrase)) {
            drop_signal(
                state,
                kb_id,
                document_id,
                reason::PHRASE_IS_VALUE,
                "kept, but the phrase is the value itself",
                Some(phrase),
            )
            .await;
        } else if same(&s.subject, phrase) {
            drop_signal(
                state,
                kb_id,
                document_id,
                reason::PHRASE_IS_SUBJECT,
                "kept, but the phrase is the subject's own name",
                Some(phrase),
            )
            .await;
        }
        let value_json;
        let fact_object = match (object, value) {
            (Some(o), _) if o == subject => {
                drop_signal(
                    state,
                    kb_id,
                    document_id,
                    reason::OBJECT_MISSING,
                    "a statement points at its own subject",
                    Some(phrase),
                )
                .await;
                continue;
            }
            (Some(o), _) => FactObject::Entity(o),
            (None, Some(v)) => {
                value_json = serde_json::json!({ "value": v });
                FactObject::Value(&value_json)
            }
            (None, None) => {
                drop_signal(
                    state,
                    kb_id,
                    document_id,
                    reason::OBJECT_MISSING,
                    "a statement has neither an object nor a value",
                    Some(phrase),
                )
                .await;
                continue;
            }
        };
        // 引文定位。定位不到的那句照样当证据文字记,只是没有偏移,并记一笔——
        // 模型没照抄的句子是「不是原文说的」那一类的苗头,量它
        let quote_text = s.quote.as_deref().map(str::trim).filter(|q| !q.is_empty());
        let quote: Option<(&str, Option<(i32, i32)>)> =
            quote_text.map(|q| (q, locate(&chunk.text, q)));
        if let Some((q, None)) = quote {
            drop_signal(
                state,
                kb_id,
                document_id,
                reason::QUOTE_NOT_IN_CHUNK,
                "a quoted sentence is not in the chunk verbatim",
                Some(q),
            )
            .await;
        }
        // **值必须出现在它自己的引文里**(#729)。一行五列被压成五条只有值不同的
        // 陈述、引文却是表上面那句导语时,五条里至多一条对,而谁也分不出是哪一条。
        // 只查带数字的值:实体名当值写的那一类(object_undeclared)不在此列
        if let (Some(v), Some((q, _))) = (stated_value, quote) {
            if v.chars().any(|c| c.is_ascii_digit()) && !shows_value(q, v) {
                drop_signal(
                    state,
                    kb_id,
                    document_id,
                    reason::VALUE_NOT_IN_QUOTE,
                    "a statement's value is not in its own quoted sentence",
                    Some(v),
                )
                .await;
                continue;
            }
        }
        // 开放陈述没有模型自报的置信度:它说的是「文档这么说了」。看图描述出来的块
        // 照旧压上限(0040 决定 4)
        let confidence = origin_ceiling(&chunk.origin, 1.0);
        // 限定和时间词先算好:直接落库和等人点头两条路用同一份。
        // 限定:值是清单上某个东西的名字就挂实体,否则挂文字
        let mut qualifiers: Vec<(&str, Option<serde_json::Value>, Option<Uuid>)> = Vec::new();
        for (role, text) in &s.qualifiers {
            let (role, text) = (role.trim(), text.trim());
            if role.is_empty() || text.is_empty() {
                continue;
            }
            match place(
                pool,
                kb_id,
                &mut local,
                known_by_name,
                &deferred,
                described,
                text,
            )
            .await?
            {
                Some(id) => qualifiers.push((role, None, Some(id))),
                None => qualifiers.push((role, Some(serde_json::json!(text)), None)),
            }
        }
        // 时间词:起与止各是一条提及,必须原样在这条陈述的引文里
        let mut time_words: Vec<(&str, i32, &str)> = Vec::new();
        for (role, words) in [("when", s.when.as_deref()), ("ended", s.ended.as_deref())]
            .into_iter()
            .filter_map(|(role, words)| Some((role, words?.trim())))
            .filter(|(_, w)| !w.is_empty())
        {
            // 推送来的陈述没有引文:条目自己就是证据(0054 决定 4),这一块就是这一份
            // 载荷,时间词在块里找。走 `locate_time` 会在 `quote?` 上退出,起止就都丢了
            let located = if pushed {
                locate(&chunk.text, words).map(|(start, _)| start)
            } else {
                locate_time(&chunk.text, quote, words)
            };
            match located {
                Some(start) => time_words.push((words, start, role)),
                None => {
                    drop_signal(
                        state,
                        kb_id,
                        document_id,
                        reason::TIME_NOT_IN_QUOTE,
                        "a time mention's words are not in the statement's own sentence",
                        Some(words),
                    )
                    .await;
                }
            }
        }
        if await_nod {
            // 记忆日志:一切原样进待确认表(0015),人点头时 `pending::confirm` 才把它
            // 落成开放陈述——同样的短语、限定、时间词、引文偏移
            let (object_id, object_value) = match fact_object {
                FactObject::Entity(id) => (Some(id), None),
                FactObject::Value(v) => (None, Some(v)),
            };
            let qualifiers_json = serde_json::Value::Array(
                qualifiers
                    .iter()
                    .map(|(role, value, entity)| match entity {
                        Some(id) => serde_json::json!({ "role": role, "entity_id": id }),
                        None => serde_json::json!({ "role": role, "value": value }),
                    })
                    .collect(),
            );
            let time_json = serde_json::Value::Array(
                time_words
                    .iter()
                    .map(|(text, start, role)| {
                        serde_json::json!({ "text": text, "char_start": start, "role": role })
                    })
                    .collect(),
            );
            let outcome = utopia_store::pending::propose(
                pool,
                Proposal {
                    kb_id,
                    subject_id: subject,
                    predicate_id: None,
                    object_id,
                    object_value,
                    proposed_predicate: Some(phrase),
                    validity: Validity {
                        attested_at,
                        ..Default::default()
                    },
                    confidence,
                    chunk_id: chunk.id,
                    proposed_by: proposer.user_id,
                    proposed_token: proposer.token_id,
                    phrase: Some(phrase),
                    qualifiers: Some(&qualifiers_json),
                    time_words: Some(&time_json),
                    quote_span: quote.and_then(|(_, span)| span),
                },
            )
            .await?;
            if let Outcome::Proposed(_) = outcome {
                *pending_count += 1;
            }
            *statement_count += 1;
            continue;
        }
        let (fact_id, _created) = utopia_store::graph::insert_open_statement(
            pool,
            kb_id,
            subject,
            phrase,
            fact_object,
            attested_at,
            confidence,
        )
        .await?;
        // 表层谓词也写短语:今天的读路径都从 `fact_surface_predicate` 取名字
        utopia_store::graph::add_evidence_located(
            pool,
            fact_id,
            chunk.id,
            quote.map(|(q, _)| q),
            Some(phrase),
            quote.and_then(|(_, span)| span),
        )
        .await?;
        for (role, value, entity) in &qualifiers {
            utopia_store::graph::add_statement_qualifier(
                pool,
                fact_id,
                role,
                value.as_ref(),
                *entity,
            )
            .await?;
        }
        for (words, start, role) in &time_words {
            utopia_store::time_mentions::record(
                pool, kb_id, fact_id, chunk.id, words, *start, role,
            )
            .await?;
        }
        *statement_count += 1;
    }

    // ---- 别名(0041 决定 2):服务端只核对名字确实在这一块原文里 ----
    // 记忆日志里的别名不记:那一句算不算出处要等人点头(0018)
    for n in extraction.names.iter().filter(|_| !await_nod) {
        let name = n.name.trim();
        if name.is_empty() {
            continue;
        }
        let Some(id) = place(
            pool,
            kb_id,
            &mut local,
            known_by_name,
            &deferred,
            described,
            &n.entity,
        )
        .await?
        else {
            drop_signal(
                state,
                kb_id,
                document_id,
                reason::UNKNOWN_REF,
                "a name's thing is not a listed thing",
                Some(&n.entity),
            )
            .await;
            continue;
        };
        if !span_in_quote(name, &chunk.text) {
            drop_signal(
                state,
                kb_id,
                document_id,
                reason::NAME_NOT_IN_TEXT,
                "an extra name is not in the chunk",
                Some(name),
            )
            .await;
            continue;
        }
        // 一个名字不会同时是两样东西的名字:这一块或本文档前面已经把它认成
        // 另一个实体的,不记
        let key = name_key(name);
        let claimed = handled_by_name
            .get(&key)
            .is_some_and(|ids| ids.iter().any(|x| *x != id));
        if claimed {
            drop_signal(
                state,
                kb_id,
                document_id,
                reason::NAME_CLAIMED_BY_ANOTHER,
                "an extra name is already another thing's name",
                Some(name),
            )
            .await;
            continue;
        }
        let quote = n
            .quote
            .as_deref()
            .map(str::trim)
            .filter(|q| !q.is_empty())
            .unwrap_or(&chunk.text);
        let _ = utopia_store::names::record(
            pool,
            kb_id,
            id,
            name,
            Some(utopia_store::names::NameSource {
                chunk_id: chunk.id,
                quote,
            }),
            attested_at,
        )
        .await;
        touched_names.insert(key);
    }

    // 本块抽完即打标:更新时被认领的块携带标记跳过;中断的抽取可续跑
    utopia_store::documents::mark_chunk_extracted(pool, chunk.id).await?;

    Ok(())
}

/// 文档日期只在它来自内容或来源系统时才算证据日期(与 `temporal::DATED_AT` 同一口径)。
pub(crate) fn dated_at(doc: &Document) -> Option<chrono::DateTime<chrono::Utc>> {
    if matches!(doc.doc_time_source.as_str(), "content" | "source") {
        doc.doc_time
    } else {
        None
    }
}

/// 一段字在块里的**字符**偏移(起、止)。字符不是字节:界面和 SQL 的 `substr` 都按字符数,
/// 中文一个字三个字节,按字节存的偏移到界面上就错位。找不到的就 `None`——
/// 偏移只能由服务端从原文算出来,模型报的数字不算数。
///
/// **一段空白对一段空白**,别的逐字比。PDF 的文字层在行尾换行,「December⏎31, 2024」
/// 模型抄回来是「December 31, 2024」;逐字比的话这句引文和它的日期都对不上,陈述落了、
/// 时间却丢了(一份英文董事会纪要四个日期全没了)。换行还是空格不是原文说了什么,
/// 偏移照原文里的位置算
fn locate(hay: &str, needle: &str) -> Option<(i32, i32)> {
    let (needle, _) = squashed(needle.trim());
    if needle.is_empty() {
        return None;
    }
    let (hay, at) = squashed(hay);
    let found = hay
        .windows(needle.len())
        .position(|w| w == needle.as_slice())?;
    let start = at[found];
    let end = at[found + needle.len() - 1] + 1;
    Some((start as i32, end as i32))
}

/// 连着的空白压成一个空格。第二项是压过之后每个字符在原文里的字符位置
fn squashed(text: &str) -> (Vec<char>, Vec<usize>) {
    let (mut out, mut at) = (Vec::new(), Vec::new());
    let mut in_space = false;
    for (i, c) in text.chars().enumerate() {
        if c.is_whitespace() {
            if !in_space {
                out.push(' ');
                at.push(i);
            }
            in_space = true;
        } else {
            out.push(c);
            at.push(i);
            in_space = false;
        }
    }
    (out, at)
}

/// 时间词在块里的字符起点。**必须在这条陈述自己的那句引文里**:模型会把一个时间词挂到
/// 好几条陈述上(FDA 语料实测「week 4」挂到了「不应由过敏患者服用」上,六条带时间的陈述错了三条),
/// 整块里搜得到不等于这句说了它。引文里有、但引文本身没在块里定位到的,起点退回整块里的第一处
/// 这条陈述的值有没有真的写在它的引文里(#729)。
///
/// 千分位逗号、空白和货币符号两边都不算数;别的按原样比。**故意放得松**:这是一条丢弃
/// 规则,放过一条可疑的,好过丢掉一条对的
fn shows_value(quote: &str, value: &str) -> bool {
    let strip = |s: &str| {
        s.chars()
            .filter(|c| !c.is_whitespace() && *c != ',' && !matches!(c, '$' | '¥' | '€' | '£'))
            .collect::<String>()
    };
    let v = strip(value);
    v.is_empty() || strip(quote).contains(&v)
}

fn locate_time(chunk: &str, quote: Option<(&str, Option<(i32, i32)>)>, words: &str) -> Option<i32> {
    let (q, span) = quote?;
    if let Some((inner, _)) = locate(q, words) {
        return match span {
            // 在原文的这一句里找:引文和原文的空白未必一样长(换行加缩进对一个空格),
            // 按引文里的位置加上去会偏
            Some((start, end)) => {
                let sentence: String = chunk
                    .chars()
                    .skip(start.max(0) as usize)
                    .take((end - start).max(0) as usize)
                    .collect();
                Some(start + locate(&sentence, words).map_or(inner, |(i, _)| i))
            }
            None => locate(chunk, words).map(|(s, _)| s),
        };
    }
    // 表格的一行:期数写在表头行里,不在这一行里;这一块就是这张表(分块器让表头
    // 跟着每一块走),所以在整块里找。判据是结构的:引文落在块里一行 `|` 开头的
    // 表格行上(模型常把行首的 `| ` 抄掉,所以看块里那一行,不只看引文自己)
    let on_table_row = match span {
        Some((start, _)) => {
            let start = chunk
                .char_indices()
                .nth(start.max(0) as usize)
                .map_or(chunk.len(), |(b, _)| b);
            let line_start = chunk[..start].rfind('\n').map_or(0, |i| i + 1);
            chunk[line_start..].trim_start().starts_with('|')
        }
        None => q.trim_start().starts_with('|') || q.contains(" | "),
    };
    if on_table_row {
        return locate(chunk, words).map(|(s, _)| s);
    }
    // 标题上的时间(0064):一节的标题只说了一个时间(「## 2025年第三季度」),它下面没写
    // 时间的陈述拿它当 when。判据也是结构的:这几个字在块里一行 `#` 开头的标题行上
    let (start, _) = locate(chunk, words)?;
    let byte = chunk
        .char_indices()
        .nth(start.max(0) as usize)
        .map_or(chunk.len(), |(b, _)| b);
    let line_start = chunk[..byte].rfind('\n').map_or(0, |i| i + 1);
    let line = chunk[line_start..].lines().next().unwrap_or("");
    let level = line.trim_start().chars().take_while(|c| *c == '#').count();
    if (1..=6).contains(&level) {
        return Some(start);
    }
    None
}

/// 名字的查找键:空白折叠、小写。陈述里写的名字和 `e` 里列的名字要一字不差,
/// 差的只许是空白和大小写
/// 一次送去嵌入的名字数。嵌入端点按请求限批,与 `pipeline` 的 chunk 批同一档
const NAME_EMBED_BATCH: usize = 16;
/// 抽完一篇文档补多少条还没有向量的名字。一次一批,剩下的下一篇再补
const NAME_VECTOR_PENDING: i64 = 256;

/// 一批名字各算一条向量,键是 `name_key`。数量对不上整批放弃(配对按位置,错一条全体
/// 错位,与 `pipeline::embed_pending` 同一条规矩);任何失败只记日志、返回空——名字向量
/// 是召回的辅助,抽取不因它失败
async fn embed_names(
    state: &AppState,
    settings: &LlmSettings,
    client: &utopia_llm::LlmClient,
    names: &[(String, String)],
) -> HashMap<String, Vec<f32>> {
    let mut out: HashMap<String, Vec<f32>> = HashMap::new();
    for batch in names.chunks(NAME_EMBED_BATCH) {
        let texts: Vec<String> = batch.iter().map(|(_, t)| t.clone()).collect();
        let _permit = crate::llm_util::acquire_embed(state, settings).await;
        match client.embed(&texts).await {
            Ok(vectors) if vectors.len() == batch.len() => {
                out.extend(batch.iter().map(|(k, _)| k.clone()).zip(vectors));
            }
            Ok(vectors) => {
                tracing::warn!(
                    sent = batch.len(),
                    got = vectors.len(),
                    "名字向量数量对不上,这一批放弃"
                );
            }
            Err(e) => {
                tracing::warn!(error = %e, "名字向量没算出来,这一批退回字面召回");
            }
        }
    }
    out
}

/// 抽完一篇文档,把这个库里还没有向量的名字事实补上一批(这篇新写的名字都在里面)。
/// 消解时算过的那些这里会再算一次——消解拿不到名字事实的 id(本名在 `create_entity`
/// 的一条语句里落下);省的只是一次嵌入调用,不值得为它改消解的返回值
async fn embed_pending_names(
    state: &AppState,
    settings: &LlmSettings,
    client: &utopia_llm::LlmClient,
    kb_id: Uuid,
) -> anyhow::Result<usize> {
    let pending =
        utopia_store::name_vectors::pending(&state.pool, kb_id, NAME_VECTOR_PENDING).await?;
    if pending.is_empty() {
        return Ok(0);
    }
    let mut items: Vec<(Uuid, Uuid, Vec<f32>)> = Vec::with_capacity(pending.len());
    for batch in pending.chunks(NAME_EMBED_BATCH) {
        let texts: Vec<String> = batch.iter().map(|(_, _, n)| n.clone()).collect();
        let _permit = crate::llm_util::acquire_embed(state, settings).await;
        let vectors = client.embed(&texts).await?;
        if vectors.len() != batch.len() {
            anyhow::bail!("嵌入返回 {} 条,送去的是 {} 条", vectors.len(), batch.len());
        }
        items.extend(
            batch
                .iter()
                .map(|(f, e, _)| (*f, *e))
                .zip(vectors)
                .map(|((f, e), v)| (f, e, v)),
        );
    }
    utopia_store::name_vectors::set(&state.pool, kb_id, &items).await?;
    Ok(items.len())
}

/// 陈述里写的名字 → 实体。先看这一块列出的,再看本文档前面认下的(提示词里的清单)。
///
/// **被描述的东西用到才建。** 模型会把段落里每个名词短语都列进 `e`:实测一个 25 篇的库里
/// 848 个实体有 616 个是被描述的,其中 181 个没挂任何陈述,还有 155 字的从句。没人指着的
/// 描述不是实体,只是一句话的一部分——所以描述先记在 `deferred` 里,第一条指到它的陈述
/// 才把它建出来;同一篇里同一段描述只建一次(`described`)
async fn place(
    pool: &PgPool,
    kb_id: Uuid,
    local: &mut HashMap<String, Uuid>,
    known: &HashMap<String, Uuid>,
    deferred: &HashMap<String, (String, String)>,
    described: &mut HashMap<String, Uuid>,
    name: &str,
) -> anyhow::Result<Option<Uuid>> {
    let key = name_key(name);
    if let Some(id) = local.get(&key).or_else(|| known.get(&key)) {
        return Ok(Some(*id));
    }
    let Some((text, kind)) = deferred.get(&key) else {
        return Ok(None);
    };
    let id = match described.get(&key) {
        Some(id) => *id,
        None => {
            let id = utopia_store::resolution::create_described(
                pool,
                kb_id,
                text,
                (!kind.is_empty()).then_some(kind.as_str()),
            )
            .await?;
            described.insert(key.clone(), id);
            id
        }
    };
    local.insert(key, id);
    Ok(Some(id))
}

/// `await_nod`:这是记忆日志(0015)——陈述不直接落库,原样进待确认表,人点头时才成为开放陈述。
/// `proposer`:那句话是谁、经哪枚令牌说的(0026),随待确认项一起记。
/// `pushed`:块本身就是契约(0054 的 `statements` 来源)——不建提示词、不问模型,直接解析;
/// 这时 `client`(对话模型)为 None;`settings` 有就照传,名字向量的嵌入模型从它来。其余一步不变
#[allow(clippy::too_many_arguments)]
pub(crate) async fn run_open(
    state: &AppState,
    doc: &Document,
    kb: &KnowledgeBase,
    settings: Option<&LlmSettings>,
    client: Option<&utopia_llm::LlmClient>,
    my_epoch: i32,
    proposer: Proposer,
    await_nod: bool,
    pushed: bool,
) -> anyhow::Result<()> {
    let pool = &state.pool;
    let document_id = doc.id;
    let kb_id = kb.id;
    // 本轮从头讲一遍这篇文档的故事,旧信号先清掉(重抽自动作数)
    let _ = utopia_store::extraction_drops::clear_for_document(pool, document_id).await;
    let attested_at = dated_at(doc);
    let chunks = utopia_store::documents::chunks_for_extraction(pool, document_id).await?;
    // 记忆日志一句一块,前一句不是后一句的开头,不附(与老路同一条规矩)
    let opening_chunk = if await_nod {
        None
    } else {
        utopia_store::documents::opening_chunk(pool, document_id).await?
    };
    let mut pending_count = 0usize;

    // 本文档已认下的**有名字的**实体,按首次出现排序,送进后续分块的提示词;
    // 陈述按名字指它们。被描述的东西不进清单:「一家医院」在下一块里指的未必是同一家
    let mut doc_entities: Vec<(Uuid, String, String)> = Vec::new();
    let mut known_by_name: HashMap<String, Uuid> = HashMap::new();
    // 类别词已经绑到类的(0044 对齐第一片):提及带着类去消解,同名不同类才分得开
    let bound_types = utopia_store::type_bindings::bound_map(pool, kb_id).await?;
    // 被描述的东西按描述文字在本文档内复用:同一块里「the northern wing」说了三次是一个东西
    let mut described: HashMap<String, Uuid> = HashMap::new();
    let mut handled_by_name: HashMap<String, Vec<Uuid>> = HashMap::new();
    let mut ambiguous_bare_cache: HashMap<String, Uuid> = HashMap::new();
    let mut touched_names: HashSet<String> = HashSet::new();
    let mut unextracted: Vec<(i32, String)> = Vec::new();
    let mut needs_adjudication = false;
    let mut human_reviews_found = false;
    let mut statement_count = 0usize;

    // 名字向量的嵌入客户端(0041 决定 3 通道 2)。没配嵌入模型就是 None:召回退回
    // 字面相等,抽取照常
    let embed = settings.and_then(crate::llm_util::embed_client);
    // 文档自己说到的日期(0064 决定 1):每一块报上来的,连它所在的标题路径。块与块之间
    // 不传——一条日期管到哪,看的是它自己那一块正文里的标题,块并行抽的时候照样成立(#588)
    let mut time_entries: Vec<utopia_extract::time::TimeEntry> = Vec::new();
    let mut extracted_chunks: Vec<Uuid> = Vec::new();
    for chunk in chunks.iter() {
        // 被接管则安静退场(重抽自增 epoch):检查放在调用模型之前
        if utopia_store::documents::extract_epoch(pool, document_id).await? != my_epoch {
            tracing::info!(%document_id, "抽取任务已被新一轮接管,退出");
            return Ok(());
        }
        let ctx: Option<&[f32]> = chunk.embedding.as_ref().map(|v| v.as_slice());
        let known: Vec<utopia_extract::KnownEntity> = doc_entities
            .iter()
            .enumerate()
            .map(|(index, (_, kind, name))| utopia_extract::KnownEntity {
                handle: format!("k{}", index + 1),
                type_key: kind.clone(),
                name: name.clone(),
            })
            .collect();
        let opening = opening_chunk
            .as_ref()
            .filter(|(id, _)| *id != chunk.id)
            .map(|(_, text)| text.as_str());
        // 提示词里带不带前面分块认下的实体,是个为了量而设的旋钮(#588);落库那半步不受它管
        let shown: &[utopia_extract::KnownEntity] = if state.extract_known_in_prompt {
            &known
        } else {
            &[]
        };
        let parsed = match call_and_parse_one_chunk(
            state,
            settings,
            client,
            embed.as_ref(),
            kb_id,
            document_id,
            &doc.filename,
            chunk,
            opening,
            shown,
            pushed,
            &mut unextracted,
        )
        .await?
        {
            Some(p) => p,
            None => continue,
        };
        let ChunkParsed {
            extraction,
            name_vecs,
        } = parsed;
        extracted_chunks.push(chunk.id);
        for entry in &extraction.time {
            // 字要原样在这一块里(与名字、引文同一条规矩);核不到的不算
            let Some((start, _)) = locate(&chunk.text, &entry.words) else {
                drop_signal(
                    state,
                    kb_id,
                    document_id,
                    reason::TIME_NOT_IN_QUOTE,
                    "a date the passage was said to state is not in the passage",
                    Some(&entry.words),
                )
                .await;
                continue;
            };
            let scope =
                utopia_extract::time::headings_at(&chunk.text, usize::try_from(start).unwrap_or(0));
            if time_entries
                .iter()
                .any(|e| e.kind == entry.kind && e.words == entry.words && e.scope == scope)
            {
                continue;
            }
            time_entries.push(utopia_extract::time::TimeEntry {
                scope,
                chunk: Some(chunk.id.to_string()),
                char_start: Some(start),
                ..entry.clone()
            });
        }
        apply_open_extraction(
            state,
            pool,
            kb_id,
            document_id,
            chunk,
            ctx,
            &extraction,
            name_vecs,
            attested_at,
            &bound_types,
            pushed,
            await_nod,
            proposer,
            &mut pending_count,
            &mut statement_count,
            &mut needs_adjudication,
            &mut human_reviews_found,
            &mut doc_entities,
            &mut known_by_name,
            &mut described,
            &mut handled_by_name,
            &mut ambiguous_bare_cache,
            &mut touched_names,
        )
        .await?;
    }

    // 消歧后缀在实体创建时算会早于其事实写入——收尾时对本文档涉及的名字统一刷新
    for name in &touched_names {
        utopia_store::resolution::refresh_disambiguators(pool, kb_id, name).await?;
    }
    if utopia_store::documents::extract_epoch(pool, document_id).await? != my_epoch {
        tracing::info!(%document_id, "抽取任务已被新一轮接管,收尾时退出");
        return Ok(());
    }
    // 有分块没抽成就不许标 done(与类型化那条路同一条规矩)。**但抽成了的那些照常收尾**:
    // 从前在这里直接返回,后面的时间解析、待确认的那一声通知都跟着没了——记忆日志里一句
    // 抽不成的话,让它后面每一句的确认卡都不出现(#1187)。错留到最后再报
    let incomplete = incomplete_reason(&unextracted, chunks.len());
    if incomplete.is_none() {
        utopia_store::documents::set_graph_status(pool, document_id, "done").await?;
        state.emit_document(kb_id, document_id);
    }
    state.emit_graph(kb_id);
    // 文档的时间语境(0064):这一轮各块报上来的日期并进文档上存着的那份。没抽到的块
    // (增量抽取时认领的未变段落)原来报的留着;这一轮抽过的块以这一轮的为准。
    // 一条都没有就不写:解析那一步照旧读一次开头(老路是它的兜底)
    let mut context: utopia_extract::time::DocumentDating = doc
        .time_context
        .clone()
        .and_then(|json| serde_json::from_value(json).ok())
        .unwrap_or_default();
    let redone: HashSet<String> = extracted_chunks.iter().map(Uuid::to_string).collect();
    context
        .entries
        .retain(|e| e.chunk.as_ref().is_some_and(|c| !redone.contains(c)));
    context.entries.extend(time_entries);
    if !context.entries.is_empty() {
        // 整篇的起算点:范围最浅的那个 `now`,同一层取先说的。它只是相对时间词在没有更近的
        // 起算点时的基准,不盖在任何一条事实上(0064 决定 2)
        let whole = context
            .entries
            .iter()
            .filter(|e| e.kind == "now")
            .min_by_key(|e| e.scope.len());
        context.date = whole.map(|e| e.from.clone());
        context.date_words = whole.map(|e| e.words.clone());
        utopia_store::documents::set_time_context(
            pool,
            document_id,
            &serde_json::to_value(&context)?,
        )
        .await?;
    }
    // 时间词的解析是它后面的任务(0045):文档时间上下文 + 每个提及的解释 + 算区间
    utopia_store::jobs::enqueue_unless_queued(
        pool,
        "resolve_time",
        serde_json::json!({ "document_id": document_id }),
    )
    .await?;
    // 新出现的类别词绑到类(0044 对齐第一片):库级,同库排着就不重复
    utopia_store::jobs::enqueue_unless_queued(
        pool,
        "align_types",
        serde_json::json!({ "kb_id": kb_id }),
    )
    .await?;
    // 推送来的文档(0054):短语正好是一条声明了的关系的签名,本体当场判(#1242);再排一份
    // 物化,绑着的签名下新来的陈述不等对齐器跑完就成类型化行——没配对话模型的库,对齐器
    // 一次也不会跑完。排着就不重复:连推一百篇只算一遍
    if pushed {
        let bound = crate::phrase_alignment::bind_declared(pool, kb_id, document_id).await?;
        if bound > 0 {
            tracing::info!(%document_id, bound, "推送的短语按声明的关系绑上");
        }
        utopia_store::jobs::enqueue_unless_queued(
            pool,
            utopia_store::phrase_bindings::MATERIALIZE_KIND,
            serde_json::json!({ "kb_id": kb_id }),
        )
        .await?;
    }
    // 队列里多了东西才叫醒人:Review 的计数与对话里那张确认卡都靠这一声
    if pending_count > 0 {
        tracing::info!(%document_id, pending_count, "记忆抽出的陈述进了待确认队列");
        state.emit_pending(kb_id);
        state.emit_review(kb_id);
    }

    // 名字向量:这篇新写的名字事实,向量补上(0041 决定 3 通道 2)。算不出来只记日志——
    // 文档已经抽完了,不能因为召回的辅助数据没算而把它标成 failed
    if let (Some(settings), Some(client)) = (settings, &embed) {
        match embed_pending_names(state, settings, client, kb_id).await {
            Ok(n) if n > 0 => tracing::info!(%document_id, names = n, "名字向量已补"),
            Ok(_) => {}
            Err(e) => tracing::warn!(%document_id, error = %e, "名字向量没补上,下一篇再补"),
        }
    }

    // 灰区对进了审核队列 → 治理 / 裁决任务,同库已排着的不重复。
    // 不排类型消解、不排自动扩本体:开放图谱里没有类也没有关系可扩,那是对齐的事
    if kb.governance {
        if needs_adjudication || human_reviews_found {
            utopia_store::jobs::enqueue_unless_queued(
                pool,
                "govern",
                serde_json::json!({ "kb_id": kb_id }),
            )
            .await?;
        }
    } else if needs_adjudication {
        utopia_store::jobs::enqueue_unless_queued(
            pool,
            "adjudicate_entities",
            serde_json::json!({ "kb_id": kb_id }),
        )
        .await?;
    }
    if needs_adjudication || human_reviews_found {
        state.emit_review(kb_id);
    }
    if let Some(msg) = incomplete {
        return Err(anyhow::anyhow!(msg));
    }
    tracing::info!(%document_id, statements = statement_count, "开放图谱抽取完成");
    Ok(())
}

#[cfg(test)]
#[path = "extraction_open_known_flow_tests.rs"]
mod known_flow_tests;

#[cfg(test)]
#[path = "extraction_open_out_of_credit_tests.rs"]
mod out_of_credit_tests;

#[cfg(test)]
mod tests {
    use super::*;

    /// #729:引文里没有这个数,这条陈述就没有依据。丢弃规则要宁松勿紧
    #[test]
    fn a_value_must_be_in_the_sentence_that_is_quoted() {
        let row = "| Cost of revenue | $46 | $47 | $93 |";
        assert!(shows_value(row, "$46"));
        assert!(shows_value(row, "46"), "货币符号不算数");
        assert!(
            shows_value("| Accounts payable | 1,915 |", "1915"),
            "千分位不算数"
        );
        assert!(shows_value("revenue was $ 1,234", "$1,234"), "空白不算数");
        assert!(
            !shows_value(
                "(A) Acquisition-related costs are comprised of amortization.",
                "$46"
            ),
            "导语里没有这个数——正是压平表格那一行的错法"
        );
        assert!(shows_value("anything", ""), "没有值就没什么可核的");
        assert!(
            shows_value("| a | 4,600 |", "46"),
            "放得松:像是命中就放过,丢错一条比留下一条可疑的更糟"
        );
    }

    #[test]
    fn offsets_are_in_characters_not_bytes() {
        let text = "北京的冬天很冷。去年冬天下了三场雪。";
        assert_eq!(locate(text, "去年冬天"), Some((8, 12)));
        assert_eq!(locate(text, "  去年冬天 "), Some((8, 12)));
        assert_eq!(locate(text, "前年冬天"), None);
        assert_eq!(locate(text, ""), None);
    }

    /// PDF 的文字层在行尾换行,模型抄回来是空格:一段空白对一段空白,偏移照原文算
    #[test]
    fn a_line_break_in_the_passage_matches_a_space_in_the_copy() {
        let text = "1. Succession. Zhang Wei will step down as Chief Technology Officer on December\n   31, 2024. Li Na follows.";
        let quote = "Zhang Wei will step down as Chief Technology Officer on December 31, 2024.";
        let span = locate(text, quote).expect("the sentence is in the passage");
        let found: String = text
            .chars()
            .skip(span.0 as usize)
            .take((span.1 - span.0) as usize)
            .collect();
        assert_eq!(
            found,
            "Zhang Wei will step down as Chief Technology Officer on December\n   31, 2024."
        );
        // 时间词在原文那一句里的位置,不是在引文里的位置:换行加缩进比一个空格长
        let start = locate_time(text, Some((quote, Some(span))), "December 31, 2024").unwrap();
        assert_eq!(
            text.chars()
                .skip(start as usize)
                .take(8)
                .collect::<String>(),
            "December"
        );
        // 反过来也一样:原文是一个空格,抄回来多了空白
        assert_eq!(locate("a b", "a  \t b"), Some((0, 3)));
        // 空白之外逐字比:断在词中间的、换了字的都不算
        assert_eq!(locate("Septem\nber 1, 2024", "September 1, 2024"), None);
        assert_eq!(locate(text, "December 30, 2024"), None);
        assert_eq!(locate("   ", " "), None);
    }

    #[test]
    fn a_time_is_found_inside_its_own_sentence_first() {
        let text = "In 2019 the plant opened. In 2019 it closed again.";
        let quote = ("In 2019 it closed again.", Some((26, 50)));
        assert_eq!(locate_time(text, Some(quote), "2019"), Some(29));
        // 引文里有、引文自己没定位到:起点退回整块里的第一处
        assert_eq!(
            locate_time(text, Some(("it closed in 2019", None)), "2019"),
            Some(3)
        );
        // 不在这句引文里的时间不算这条陈述的,哪怕块里别处有
        assert_eq!(
            locate_time(text, Some(("the plant opened.", Some((8, 25)))), "2019"),
            None
        );
        assert_eq!(locate_time(text, None, "2019"), None);
    }

    #[test]
    fn a_table_row_takes_its_period_from_the_header_in_the_same_chunk() {
        let text = "|  | Q2 FY27 | Q1 FY27 |\n| --- | --- | --- |\n| Revenue | $96,221 | $81,615 |";
        let row = ("| Revenue | $96,221 | $81,615 |", Some((46, 76)));
        // 期数在表头行里,不在这一行里:表格行在整块里找
        assert_eq!(locate_time(text, Some(row), "Q2 FY27"), Some(5));
        // 模型把行首的 "| " 抄掉了:引文定位到的那一行还是表格行
        let bare = ("Revenue | $96,221 | $81,615 |", Some((48, 76)));
        assert_eq!(locate_time(text, Some(bare), "Q1 FY27"), Some(15));
        // 不是表格行的引文还是只认自己那句
        assert_eq!(
            locate_time(text, Some(("Revenue was up.", None)), "Q2 FY27"),
            None
        );
    }

    #[test]
    fn names_match_up_to_whitespace_and_case() {
        assert_eq!(name_key("  Harbor   Bridge "), "harbor bridge");
        assert_eq!(name_key("harbor bridge"), name_key("HARBOR BRIDGE"));
        assert_ne!(name_key("Harbor Bridge"), name_key("Harbour Bridge"));
    }
}