Skip to content

World's first open-source enterprise world model.

22.2 KB
Raw Blame
//! 关系短语按签名绑到属性的账本侧(0044 决定 3 的第二片,见 0066)。
//!
//! 开放陈述的关系是文档自己的短语(`facts.phrase`),不选属性。这里做三件事:数出一个
//! 库里有哪些签名(短语 × 主语的类 × 宾语的类或「值」),各带几条例句;记下每个签名判成
//! 了什么(绑定)并判哪些过期了;给下一片(物化)一张「签名 → 属性、方向」的表。判定
//! 本身——问模型、两票一致才绑——在 server 的 `phrase_alignment` 里,这里不认识模型。
//!
//! 短语的归一同类别词:空白折成一个空格、去两端、小写;[`normalize`] 与 [`PHRASE_SQL`]
//! 必须说同一件事。

use chrono::{DateTime, Utc};
use sqlx::PgPool;
use std::collections::HashMap;
use utopia_core::{AppError, AppResult};
use uuid::Uuid;

/// SQL 侧的归一:与 [`normalize`] 一致。`$col` 由调用处替换成列名。
const PHRASE_SQL: &str = "lower(btrim(regexp_replace($col, '\\s+', ' ', 'g')))";

pub(crate) fn phrase_sql(col: &str) -> String {
    PHRASE_SQL.replace("$col", col)
}

/// 归一一个短语:空白折成一个空格、去两端、小写。"Acquired " 与 "acquired" 是一个短语。
pub fn normalize(phrase: &str) -> String {
    phrase
        .split_whitespace()
        .collect::<Vec<_>>()
        .join(" ")
        .to_lowercase()
}

/// 一条签名:判它该绑到哪个属性时给模型看的全部。
#[derive(Debug, Clone, sqlx::FromRow)]
pub struct PhraseSignature {
    /// 归一过的短语
    pub phrase: String,
    /// 主语的类(类别词还没绑到类时为空)
    pub subject_type_id: Option<Uuid>,
    pub subject_type_key: Option<String>,
    /// 宾语的类;宾语是字面值时为空且 `object_is_value` 为真
    pub object_type_id: Option<Uuid>,
    pub object_type_key: Option<String>,
    pub object_is_value: bool,
    /// 这个签名下活着的开放陈述数
    pub count: i64,
    /// 例句:最多 3 条「主语 —短语→ 宾语」,每条跟着它自己的引文
    pub examples: Vec<String>,
    pub quotes: Vec<String>,
    /// Statement identities in the same order as the live examples and quotes.
    pub example_statement_ids: Vec<Uuid>,
}

/// 库里每条 distinct 的签名:活着的开放陈述,按短语、两端的类、宾语是不是字面值分组。
/// 采纳复用一条属性时在它自己的事务里读(不另占一条连接),所以收任何执行器
pub async fn signatures<'e>(
    pool: impl sqlx::Executor<'e, Database = sqlx::Postgres>,
    kb_id: Uuid,
) -> AppResult<Vec<PhraseSignature>> {
    // 一条陈述可以有多条证据。计数、取例句前先为每条陈述选一条稳定的引用,
    // 优先有完整位置的证据,避免证据多的陈述挤掉其他陈述。
    let sql = format!(
        "WITH live AS (
             SELECT f.id, {phrase} AS phrase,
                    s.type_id AS subject_type_id, o.type_id AS object_type_id,
                    (f.object_id IS NULL) AS object_is_value,
                    s.canonical_name AS subject_name,
                    coalesce(o.canonical_name, f.object_value #>> '{{value}}', '') AS object_name,
                    f.phrase AS spelling, fe.chunk_id, fe.quote_start, fe.quote_end, f.recorded_at
             FROM facts f
             JOIN entities s ON s.id = f.subject_id
             LEFT JOIN entities o ON o.id = f.object_id
             LEFT JOIN LATERAL (
                 SELECT chunk_id, quote_start, quote_end FROM fact_evidence
                 WHERE fact_id = f.id
                 ORDER BY (quote_start IS NULL OR quote_end IS NULL), chunk_id
                 LIMIT 1
             ) fe ON true
             WHERE f.kb_id = $1 AND f.layer = 'open' AND f.invalidated_at IS NULL
               AND f.phrase IS NOT NULL AND btrim(f.phrase) <> ''
         ),
         -- 每组的计数和前 3 条例句在一趟里聚出来。从前每组的例句、引文各是一个关联子查询,
         -- 每个都把全部陈述重扫一遍,耗时随开放陈述数平方增长:3.6 万条时一趟约 200 秒(#1096)。
         -- 计数也在这里取,不另聚一份再按四个键用 IS NOT DISTINCT FROM 接回来——那种连接
         -- Postgres 做不了哈希;分块的原文只为留下的那三条去读
         ranked AS (
             SELECT l.*,
                    row_number() OVER (PARTITION BY l.phrase, l.subject_type_id, l.object_type_id, l.object_is_value
                                       ORDER BY l.recorded_at, l.id) AS rn
             FROM live l
         ),
         grouped AS (
             SELECT r.phrase, r.subject_type_id, r.object_type_id, r.object_is_value, count(*) AS count,
                    array_agg(r.subject_name || ' —' || r.spelling || '→ ' || r.object_name ORDER BY r.rn)
                        FILTER (WHERE r.rn <= 3) AS examples,
                    array_agg(coalesce(substr(c.text, r.quote_start + 1, r.quote_end - r.quote_start), '') ORDER BY r.rn)
                        FILTER (WHERE r.rn <= 3) AS quotes,
                    array_agg(r.id ORDER BY r.rn) FILTER (WHERE r.rn <= 3) AS example_statement_ids
             FROM ranked r
             LEFT JOIN chunks c ON c.id = r.chunk_id AND r.rn <= 3
             GROUP BY r.phrase, r.subject_type_id, r.object_type_id, r.object_is_value
         )
         SELECT g.phrase, g.subject_type_id, st.key AS subject_type_key,
                g.object_type_id, ot.key AS object_type_key, g.object_is_value, g.count,
                coalesce(g.examples, '{{}}') AS examples, coalesce(g.quotes, '{{}}') AS quotes,
                coalesce(g.example_statement_ids, '{{}}'::uuid[]) AS example_statement_ids
         FROM grouped g
         LEFT JOIN entity_types st ON st.id = g.subject_type_id
         LEFT JOIN entity_types ot ON ot.id = g.object_type_id
         -- 计数和短语相同、两端的类不同的签名,顺序也得定下来:对齐按这个顺序分批送模型
         ORDER BY g.count DESC, g.phrase, g.subject_type_id, g.object_type_id, g.object_is_value",
        phrase = phrase_sql("f.phrase")
    );
    Ok(sqlx::query_as(&sql).bind(kb_id).fetch_all(pool).await?)
}

/// 一篇文档里活着的开放陈述落在哪些签名上(键同 [`PhraseSignature::key`])。推送来的
/// 文档抽完,本体只判这些签名(#1242);别的签名还是对齐器的
pub async fn signature_keys_of_document(
    pool: &PgPool,
    kb_id: Uuid,
    document_id: Uuid,
) -> AppResult<Vec<(String, Option<Uuid>, Option<Uuid>, bool)>> {
    let sql = format!(
        "SELECT DISTINCT {phrase}, s.type_id, o.type_id, (f.object_id IS NULL)
           FROM facts f
           JOIN entities s ON s.id = f.subject_id
      LEFT JOIN entities o ON o.id = f.object_id
          WHERE f.kb_id = $1 AND f.layer = 'open' AND f.invalidated_at IS NULL
            AND f.phrase IS NOT NULL AND btrim(f.phrase) <> ''
            AND EXISTS (SELECT 1 FROM fact_evidence fe JOIN chunks c ON c.id = fe.chunk_id
                         WHERE fe.fact_id = f.id AND c.document_id = $2)",
        phrase = phrase_sql("f.phrase")
    );
    Ok(sqlx::query_as(&sql)
        .bind(kb_id)
        .bind(document_id)
        .fetch_all(pool)
        .await?)
}

/// 按 id 取一条绑定的签名(人在队列里定它时用):短语、两端的类、宾语是不是字面值,
/// 例句照判定时记下的,陈述数同。
pub async fn signature_of(
    pool: &PgPool,
    kb_id: Uuid,
    id: Uuid,
) -> AppResult<Option<PhraseSignature>> {
    Ok(sqlx::query_as(
        "SELECT b.phrase, b.subject_type_id, st.key AS subject_type_key,
                b.object_type_id, ot.key AS object_type_key, b.object_is_value,
                b.statement_count::bigint AS count, b.examples, '{}'::text[] AS quotes,
                '{}'::uuid[] AS example_statement_ids
           FROM phrase_bindings b
      LEFT JOIN entity_types st ON st.id = b.subject_type_id
      LEFT JOIN entity_types ot ON ot.id = b.object_type_id
          WHERE b.kb_id = $1 AND b.id = $2",
    )
    .bind(kb_id)
    .bind(id)
    .fetch_optional(pool)
    .await?)
}

/// 一条签名判成了什么。
#[derive(Debug, Clone, sqlx::FromRow)]
pub struct Binding {
    pub phrase: String,
    pub subject_type_id: Option<Uuid>,
    pub object_type_id: Option<Uuid>,
    pub object_is_value: bool,
    /// `status = 'bound'` 时有值
    pub relation_type_id: Option<Uuid>,
    /// forward / reverse,`status = 'bound'` 时有值
    pub direction: Option<String>,
    /// bound / none / undecided
    pub status: String,
    pub decided_at: DateTime<Utc>,
    /// agent / person / ontology(短语正好是一条声明了的关系,#1242)
    pub decided_by: String,
    /// 判定时输入的指纹;NULL = 这一列出现之前的判定
    pub basis: Option<String>,
    /// 一条只说了一刻的陈述,那一刻标的是状态的哪一端:start / end / none(#966)。
    /// NULL = 未知:这一列之前的判定,或属性不是状态
    pub marks: Option<String>,
    /// 对齐器问过 marks 的时刻;NULL = 没问过。问过还没有值的不再问,等人(0053 修订)
    pub marks_asked_at: Option<DateTime<Utc>>,
}

impl Binding {
    /// 和签名对上的键:短语 + 两端的类 + 宾语是不是字面值
    pub fn key(&self) -> (String, Option<Uuid>, Option<Uuid>, bool) {
        (
            self.phrase.clone(),
            self.subject_type_id,
            self.object_type_id,
            self.object_is_value,
        )
    }
}

impl PhraseSignature {
    pub fn key(&self) -> (String, Option<Uuid>, Option<Uuid>, bool) {
        (
            self.phrase.clone(),
            self.subject_type_id,
            self.object_type_id,
            self.object_is_value,
        )
    }
}

/// 库里全部绑定。
pub async fn bindings(pool: &PgPool, kb_id: Uuid) -> AppResult<Vec<Binding>> {
    Ok(sqlx::query_as(
        "SELECT phrase, subject_type_id, object_type_id, object_is_value,
                relation_type_id, direction, status, decided_at, decided_by, basis, marks,
                marks_asked_at
         FROM phrase_bindings WHERE kb_id = $1 ORDER BY phrase",
    )
    .bind(kb_id)
    .fetch_all(pool)
    .await?)
}

/// 一条判定看到的输入的指纹(0053):两端类的祖先闭包(含自己)、宾语是不是字面值、
/// 按继承命中的候选属性与各自的 `updated_at`。worker 每轮对活着的签名重算,与存下的
/// 不一致就是过期——比的是**现在的输入**,不是时刻,于是父边的增删、模型请求途中的
/// 编辑(#795)都看得见,时间戳看不见。
///
/// 只是缓存失效的键,不是安全用途:FNV-1a 64 位够用,也不用为它拉一个哈希依赖。
pub fn basis_of(
    subject_closure: &[Uuid],
    object_closure: &[Uuid],
    object_is_value: bool,
    candidates: &[(Uuid, DateTime<Utc>)],
) -> String {
    let sorted = |ids: &[Uuid]| {
        let mut v: Vec<String> = ids.iter().map(|u| u.to_string()).collect();
        v.sort();
        v.join(",")
    };
    let mut cands: Vec<String> = candidates
        .iter()
        .map(|(id, at)| format!("{id}@{}", at.to_rfc3339()))
        .collect();
    cands.sort();
    let text = format!(
        "s={};o={};v={};c={}",
        sorted(subject_closure),
        sorted(object_closure),
        object_is_value,
        cands.join(",")
    );
    let mut h: u64 = 0xcbf29ce484222325;
    for b in text.as_bytes() {
        h ^= u64::from(*b);
        h = h.wrapping_mul(0x100000001b3);
    }
    format!("{h:016x}")
}

/// 库里每条属性最后一次改动的时刻,给指纹用。一轮读一次,不进视图——视图是给页面的,
/// 页面不需要这个数
pub async fn property_versions(
    pool: &PgPool,
    kb_id: Uuid,
) -> AppResult<HashMap<Uuid, DateTime<Utc>>> {
    let rows: Vec<(Uuid, DateTime<Utc>)> =
        sqlx::query_as("SELECT id, updated_at FROM relation_types WHERE kb_id = $1")
            .bind(kb_id)
            .fetch_all(pool)
            .await?;
    Ok(rows.into_iter().collect())
}

/// 不再成立的绑定:绑到的属性在判定之后改过;或判成 none / undecided 之后库里有属性
/// 新建或修改。负向判定没有选中的属性,已有属性的新定义也可能让它对得上。
/// 属性或类被删了的,行已随级联消失。
pub async fn stale(pool: &PgPool, kb_id: Uuid) -> AppResult<Vec<Binding>> {
    Ok(sqlx::query_as(
        "SELECT b.phrase, b.subject_type_id, b.object_type_id, b.object_is_value,
                b.relation_type_id, b.direction, b.status, b.decided_at, b.decided_by, b.basis,
                b.marks, b.marks_asked_at
         FROM phrase_bindings b
         LEFT JOIN relation_types r ON r.id = b.relation_type_id
         WHERE b.kb_id = $1
           AND ((b.status = 'bound' AND r.updated_at > b.decided_at)
                OR (b.status IN ('none', 'undecided')
                    AND b.decided_at < (SELECT max(updated_at) FROM relation_types
                                        WHERE kb_id = $1)))
         ORDER BY b.phrase",
    )
    .bind(kb_id)
    .fetch_all(pool)
    .await?)
}

/// 一次判定要写的东西。
pub struct Decision<'a> {
    pub relation_type_id: Option<Uuid>,
    /// forward / reverse
    pub direction: Option<&'a str>,
    /// bound / none / undecided
    pub status: &'a str,
    pub votes: &'a serde_json::Value,
    /// agent / person / ontology
    pub decided_by: &'a str,
    /// 判定时输入的指纹([`basis_of`]):代理的判定必带,人的判定不带——人不按指纹重判
    pub basis: Option<&'a str>,
    /// start / end / none:只说了一刻的陈述在状态属性下怎么读(#966)。只有 bound 带;
    /// None = 未知,那样的陈述不算类型化行
    pub marks: Option<&'a str>,
    /// 这次判定问过 marks:对齐器把签名绑到状态属性时随判定一起问(0053 修订)。记下问的
    /// 时刻,没有值的签名就不再问、进对齐队列等人。只有 bound 带
    pub marks_asked: bool,
}

/// 记下一条签名的判定(有则改)。返回是否写入了。
///
/// **人的判定不被代理覆盖**:已有行是人判的而这次是代理或本体,原样留着、返回 false。
/// 反过来人可以改代理的、本体的。
pub async fn decide(
    pool: &PgPool,
    kb_id: Uuid,
    sig: &PhraseSignature,
    d: Decision<'_>,
) -> AppResult<bool> {
    validate_decision(sig, &d)?;
    // The observed regression outcome belongs to this decision, not to a later read that
    // could see a different person's binding. Commit both together, as delivery already does.
    let mut tx = pool.begin().await?;
    let changed = decide_on(&mut tx, kb_id, sig, d).await?;
    tx.commit().await?;
    Ok(changed)
}

fn validate_decision(sig: &PhraseSignature, d: &Decision<'_>) -> AppResult<String> {
    if !matches!(d.status, "bound" | "none" | "undecided") {
        return Err(AppError::Validation(format!(
            "unknown binding status {:?}",
            d.status
        )));
    }
    let bound = d.status == "bound";
    if bound != d.relation_type_id.is_some() || bound != d.direction.is_some() {
        return Err(AppError::Validation(
            "a bound signature needs a property and a direction and an unbound one must not have them".into(),
        ));
    }
    if let Some(dir) = d.direction {
        if !matches!(dir, "forward" | "reverse") {
            return Err(AppError::Validation(format!("unknown direction {dir:?}")));
        }
    }
    if (d.marks.is_some() || d.marks_asked) && !bound {
        return Err(AppError::Validation(
            "only a bound signature says which end a moment marks".into(),
        ));
    }
    if let Some(marks) = d.marks {
        if !matches!(marks, "start" | "end" | "none") {
            return Err(AppError::Validation(format!("unknown marks {marks:?}")));
        }
    }
    if !matches!(d.decided_by, "agent" | "person" | "ontology") {
        return Err(AppError::Validation(format!(
            "unknown decider {:?}",
            d.decided_by
        )));
    }
    let phrase = normalize(&sig.phrase);
    if phrase.is_empty() {
        return Err(AppError::Validation("an empty phrase binds nothing".into()));
    }
    Ok(phrase)
}

/// 人的判定落库时随手排下的重算任务的种类。`main` 按它分发;载荷只有 `kb_id`——
/// job 读的是**当前**的绑定,不回放判定时的属性(0051:回放旧载荷会盖掉后来的人)。
pub const MATERIALIZE_KIND: &str = "materialize_typed";

/// 人的判定与它自己的重算任务**同一事务**提交(0051)。
///
/// 为什么不是「判定落库,然后看有没有 worker 在跑」:正在跑的那次对齐可能已经做完
/// 最后一次读,这条判定就没有任何人替它算类型化行——它被接受了,却永远不投影。
/// 一条判定配一个自己的 job,job 只在判定提交后可见,被谁先处理都读到最新的绑定,
/// 于是最后一次判定总会被算到。代价是 N 次判定 N 次重算,后面的多半是空跑(0051
/// 量过:100 次判定总收敛 593 ms);「有一个在跑就不排」省下的正是那条会丢的投影。
///
/// 返回 job id;代理不能盖人(`decide_on` 的规则)时什么都没写,返回 `None`。
pub async fn decide_with_delivery(
    pool: &PgPool,
    kb_id: Uuid,
    sig: &PhraseSignature,
    d: Decision<'_>,
) -> AppResult<Option<i64>> {
    decide_with_delivery_budget(pool, kb_id, sig, d, 3).await
}

/// 预算单独成参只为了测「排队失败要连判定一起回滚」:0 会被 `enqueue` 拒掉,
/// 那正是一次发生在判定写入之后的真实失败。生产入口固定给 3。
async fn decide_with_delivery_budget(
    pool: &PgPool,
    kb_id: Uuid,
    sig: &PhraseSignature,
    d: Decision<'_>,
    max_attempts: i32,
) -> AppResult<Option<i64>> {
    let mut tx = pool.begin().await?;
    if !decide_on(&mut tx, kb_id, sig, d).await? {
        tx.rollback().await?;
        return Ok(None);
    }
    let id = crate::jobs::enqueue_with_max_attempts_tx(
        &mut tx,
        MATERIALIZE_KIND,
        serde_json::json!({ "kb_id": kb_id }),
        max_attempts,
    )
    .await?;
    tx.commit().await?;
    Ok(Some(id))
}

/// Write on the caller's transaction connection, so related durable work shares its commit.
pub async fn decide_on(
    connection: &mut sqlx::PgConnection,
    kb_id: Uuid,
    sig: &PhraseSignature,
    d: Decision<'_>,
) -> AppResult<bool> {
    let phrase = validate_decision(sig, &d)?;
    let res = sqlx::query(
        "INSERT INTO phrase_bindings
             (id, kb_id, phrase, subject_type_id, object_type_id, object_is_value,
              relation_type_id, direction, status, votes, statement_count, examples,
              decided_at, decided_by, basis, marks, marks_asked_at)
         VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, now(), $13, $14, $15,
                 CASE WHEN $16 THEN now() END)
         ON CONFLICT (kb_id, phrase, subject_type_id, object_type_id, object_is_value) DO UPDATE
            SET relation_type_id = EXCLUDED.relation_type_id,
                direction = EXCLUDED.direction,
                status = EXCLUDED.status,
                votes = EXCLUDED.votes,
                statement_count = EXCLUDED.statement_count,
                examples = EXCLUDED.examples,
                decided_at = now(),
                decided_by = EXCLUDED.decided_by,
                basis = EXCLUDED.basis,
                marks = EXCLUDED.marks,
                marks_asked_at = EXCLUDED.marks_asked_at
          WHERE NOT (phrase_bindings.decided_by = 'person' AND EXCLUDED.decided_by <> 'person')",
    )
    .bind(Uuid::now_v7())
    .bind(kb_id)
    .bind(&phrase)
    .bind(sig.subject_type_id)
    .bind(if sig.object_is_value {
        None
    } else {
        sig.object_type_id
    })
    .bind(sig.object_is_value)
    .bind(d.relation_type_id)
    .bind(d.direction)
    .bind(d.status)
    .bind(d.votes)
    .bind(i32::try_from(sig.count).unwrap_or(i32::MAX))
    .bind(&sig.examples)
    .bind(d.decided_by)
    .bind(d.basis)
    .bind(d.marks)
    .bind(d.marks_asked)
    .execute(&mut *connection)
    .await?;
    let changed = res.rows_affected() > 0;
    if changed {
        crate::ontology_regressions::record_for_signature(connection, kb_id, sig).await?;
    }
    Ok(changed)
}

/// 给这一列之前绑上的签名补问 marks 的结果(0053 修订 2026-09-27)。这一问只问一刻标哪一端,
/// 不重判绑定:`value` 是两票都认这条绑定的属性与方向、又说了同一个值时的那个值,写下它;
/// 否则是 None,绑定原样。两种都记下问过的时刻——问过的不再问,没有值的列进对齐队列等人。
///
/// 只写问的时候那条绑定:还是代理绑的、属性与方向没变、还没有值。途中人判过或重判过的,
/// 这一问的答案不算数,返回 false
pub async fn record_marks(
    pool: &PgPool,
    kb_id: Uuid,
    asked: &Binding,
    value: Option<&str>,
) -> AppResult<bool> {
    if let Some(marks) = value {
        if !matches!(marks, "start" | "end" | "none") {
            return Err(AppError::Validation(format!("unknown marks {marks:?}")));
        }
    }
    let res = sqlx::query(
        "UPDATE phrase_bindings
            SET marks = $8, marks_asked_at = now()
          WHERE kb_id = $1 AND phrase = $2
            AND subject_type_id IS NOT DISTINCT FROM $3
            AND object_type_id IS NOT DISTINCT FROM $4
            AND object_is_value = $5
            AND status = 'bound' AND decided_by = 'agent' AND marks IS NULL
            AND relation_type_id = $6 AND direction = $7",
    )
    .bind(kb_id)
    .bind(&asked.phrase)
    .bind(asked.subject_type_id)
    .bind(asked.object_type_id)
    .bind(asked.object_is_value)
    .bind(asked.relation_type_id)
    .bind(asked.direction.as_deref())
    .bind(value)
    .execute(pool)
    .await?;
    Ok(res.rows_affected() > 0)
}

#[cfg(test)]
#[path = "phrase_bindings_delivery_tests.rs"]
mod delivery_tests;

#[cfg(test)]
mod tests {
    use super::normalize;

    #[test]
    fn a_phrase_is_one_phrase_however_spaced_or_cased() {
        assert_eq!(normalize("  Was   Designed By "), "was designed by");
        assert_eq!(normalize("Revenue"), "revenue");
        assert_eq!(normalize("细化解读"), "细化解读");
        assert_eq!(normalize("   "), "");
    }
}