Skip to content

World's first open-source enterprise world model.

37.8 KB
Raw Blame
//! 来源管理 API + 文本推送摄入(ingest)。

use axum::extract::{Path, State};
use axum::http::HeaderMap;
use axum::Json;
use chrono::{DateTime, Utc};
use serde::Deserialize;
use serde_json::json;
use utopia_core::models::{Role, SOURCE_SECRET_KEYS};
use uuid::Uuid;

use super::graph_routes::require_kb;
use crate::auth::{AuthUser, SessionOrToken};
use crate::error::ApiResult;
use crate::state::AppState;

/// 生成 api 来源的推送密钥。
pub(crate) fn new_ingest_token() -> String {
    format!("utp_{}{}", Uuid::new_v4().simple(), Uuid::new_v4().simple())
}

/// 推送来的密钥对不对。**常量时间比对。**
///
/// 挡的不是计时攻击——这条路上它打不动:密钥是服务端生成的 244 位随机串
/// (`new_ingest_token`,两个 v4 拼起来),比对又发生在一次数据库往返之后,
/// 毫秒级的网络与查询抖动盖住的是纳秒级的差异。真正在保护这把密钥的是熵。
///
/// 这样写是为了**读代码的人不必重新推一遍上面那段**:仓库里另外两条令牌路径
/// 都不做明文比对(个人令牌存 sha256 按哈希查,会话走 JWT 验签),这里是唯一
/// 要拿明文对明文的地方,那就让它自己看起来是想过的。
///
/// 长度不等直接返回——`ct_eq` 只对等长切片有意义。密钥长度固定(`utp_` + 64 位
/// 十六进制),不是秘密的一部分,在这里短路不泄漏任何东西。
fn push_key_matches(stored: Option<&str>, offered: &str) -> bool {
    let Some(stored) = stored else {
        return false;
    };
    if stored.len() != offered.len() {
        return false;
    }
    use subtle::ConstantTimeEq;
    stored.as_bytes().ct_eq(offered.as_bytes()).into()
}

/// 取一条来源,并确认它属于路径上的这个库。
///
/// `require_kb` 只查人对库的权限;来源 id 是另一个维度——不比对的话,A 库的
/// Editor 拿着 B 库来源的 id 就能同步、清理、删除它。不属于就当不存在(404),
/// 与 `get_token` 一直以来的做法一致
async fn source_in_kb(
    state: &AppState,
    kb_id: Uuid,
    source_id: Uuid,
) -> ApiResult<utopia_core::models::Source> {
    let source = utopia_store::sources::get(&state.pool, source_id).await?;
    if source.kb_id != kb_id {
        return Err(utopia_core::AppError::NotFound.into());
    }
    Ok(source)
}

pub async fn list(
    State(state): State<AppState>,
    AuthUser(user): AuthUser,
    Path(kb_id): Path<Uuid>,
) -> ApiResult<Json<serde_json::Value>> {
    require_kb(&state, &user, kb_id, Role::Viewer).await?;
    let sources = utopia_store::sources::list(&state.pool, kb_id).await?;
    Ok(Json(json!({ "sources": sources })))
}

#[derive(Deserialize)]
pub struct CreateBody {
    pub kind: String,
    pub name: String,
    #[serde(default)]
    pub config: serde_json::Value,
    #[serde(default)]
    pub icon: Option<String>,
    #[serde(default)]
    pub sync_interval_minutes: Option<i32>,
    /// 标准 5 段 cron(与 interval 互斥,二者传其一)
    #[serde(default)]
    pub sync_cron: Option<String>,
}

/// 写令牌也能建(0014 决定 6):给一个镜像仓库配 `statements` 或 `api` 来源,不该要人
/// 一个个手点。
///
/// 令牌只建推送型(`statements`/`api`,服务器不替它们拉任何东西):`url`/`rss`/`custom`
/// 一建好服务器就按时去拉(`custom` 还带着存下的凭据),漏出去的 CI 令牌不该能让服务器定时去敲
/// 任意地址。会话照旧建什么都行
pub async fn create(
    State(state): State<AppState>,
    caller: SessionOrToken,
    Path(kb_id): Path<Uuid>,
    Json(body): Json<CreateBody>,
) -> ApiResult<Json<serde_json::Value>> {
    caller.require_write(Some(kb_id))?;
    if caller.pat.is_some() && !is_push_kind(&body.kind) {
        return Err(utopia_core::AppError::CodedForbidden {
            code: "token_source_kind",
            message: "A token creates only statements or api sources; other kinds take a session"
                .into(),
        }
        .into());
    }
    let token_id = caller.token_id();
    let user = caller.user;
    require_kb(&state, &user, kb_id, Role::Editor).await?;
    validate_rss_config(&body.kind, &body.config)?;
    let source = utopia_store::sources::create(
        &state.pool,
        kb_id,
        &body.kind,
        &body.name,
        &body.config,
        body.icon.as_deref(),
        body.sync_interval_minutes,
        body.sync_cron.as_deref(),
    )
    .await?;
    // 拉取型来源建好立即同步一次(有配置即产出,无需等下一个调度周期)
    if matches!(source.kind.as_str(), "url" | "rss" | "custom") {
        let _ = utopia_store::sources::mark_queued(&state.pool, source.id).await;
        utopia_store::jobs::enqueue(
            &state.pool,
            "sync_source",
            json!({ "source_id": source.id }),
        )
        .await?;
    }
    // api 来源:生成专属推送密钥(此后可随时经 get_token 查看)
    let mut ingest_token: Option<String> = None;
    if is_push_kind(&source.kind) {
        let token = new_ingest_token();
        utopia_store::sources::set_ingest_token(&state.pool, source.id, &token).await?;
        ingest_token = Some(token);
    }
    state.emit_source(kb_id);
    let _ = utopia_store::audit::record(
        &state.pool,
        Some(kb_id),
        user.id,
        "source.created",
        "source",
        Some(source.id),
        utopia_store::audit::via_token(
            json!({ "kind": source.kind, "name": source.name }),
            token_id,
        ),
    )
    .await;
    Ok(Json(
        json!({ "source": mask_secrets(source), "ingest_token": ingest_token }),
    ))
}

/// 有推送密钥的来源:`api` 推文档,`statements` 推陈述(0054)。两种的密钥同一套
/// 生成、查看、轮换
fn is_push_kind(kind: &str) -> bool {
    matches!(kind, "api" | "statements")
}

/// 查看推送密钥(Editor;列表响应从不携带,查看走这里)。
///
/// 令牌要 write:密钥能往库里推东西,读令牌拿到它就等于绕过了自己的 scope
pub async fn get_token(
    State(state): State<AppState>,
    caller: SessionOrToken,
    Path((kb_id, source_id)): Path<(Uuid, Uuid)>,
) -> ApiResult<Json<serde_json::Value>> {
    caller.require_write(Some(kb_id))?;
    require_kb(&state, &caller.user, kb_id, Role::Editor).await?;
    let source = source_in_kb(&state, kb_id, source_id).await?;
    if !is_push_kind(&source.kind) {
        return Err(utopia_core::AppError::NotFound.into());
    }
    Ok(Json(json!({ "ingest_token": source.ingest_token })))
}

/// 轮换推送密钥:旧密钥立即失效。
pub async fn rotate_token(
    State(state): State<AppState>,
    AuthUser(user): AuthUser,
    Path((kb_id, source_id)): Path<(Uuid, Uuid)>,
) -> ApiResult<Json<serde_json::Value>> {
    require_kb(&state, &user, kb_id, Role::Editor).await?;
    let source = source_in_kb(&state, kb_id, source_id).await?;
    if !is_push_kind(&source.kind) {
        return Err(utopia_core::AppError::NotFound.into());
    }
    let token = new_ingest_token();
    utopia_store::sources::set_ingest_token(&state.pool, source_id, &token).await?;
    Ok(Json(json!({ "ingest_token": token })))
}

/// 响应前剔除凭据(只进不出;键见 `SOURCE_SECRET_KEYS`)。
fn mask_secrets(source: utopia_core::models::Source) -> utopia_core::models::Source {
    source.without_secrets()
}

fn validate_rss_config(kind: &str, config: &serde_json::Value) -> utopia_core::AppResult<()> {
    if kind == "rss" {
        utopia_store::sources::rss_content_mode(config)?;
    }
    Ok(())
}

/// 更新时凭据的合并规则,每个 `SOURCE_SECRET_KEYS` 里的键一样:新配置里**没有**这个键
/// 或值是空串 → 保留库里的原值(表单留空就是「别动」);显式 `null` → 删掉;
/// 其余照新值。响应从不回显,所以客户端没有办法把旧值原样送回来,规则只能长在这里
fn keep_secrets(next: &mut serde_json::Value, existing: &serde_json::Value) {
    let Some(obj) = next.as_object_mut() else {
        return;
    };
    for key in SOURCE_SECRET_KEYS {
        let keep = match obj.get(*key) {
            None => true,
            Some(serde_json::Value::Null) => {
                obj.remove(*key);
                false
            }
            Some(v) => v.as_str().is_some_and(|s| s.trim().is_empty()),
        };
        if keep {
            obj.remove(*key);
            if let Some(prev) = existing.get(*key) {
                obj.insert((*key).to_string(), prev.clone());
            }
        }
    }
}

#[derive(Deserialize)]
pub struct UpdateBody {
    pub name: Option<String>,
    pub config: Option<serde_json::Value>,
    pub icon: Option<String>,
    /// 出现 schedule 字段即整体覆盖调度(interval 与 cron 互斥;两者皆 null = 关闭定时)
    #[serde(default)]
    pub schedule: Option<ScheduleBody>,
}

#[derive(Deserialize)]
pub struct ScheduleBody {
    #[serde(default)]
    pub sync_interval_minutes: Option<i32>,
    #[serde(default)]
    pub sync_cron: Option<String>,
}

pub async fn update(
    State(state): State<AppState>,
    AuthUser(user): AuthUser,
    Path((kb_id, source_id)): Path<(Uuid, Uuid)>,
    Json(body): Json<UpdateBody>,
) -> ApiResult<Json<serde_json::Value>> {
    require_kb(&state, &user, kb_id, Role::Editor).await?;
    let existing = source_in_kb(&state, kb_id, source_id).await?;
    // 凭据只进不出:响应从不回显,表单留空 / 没传 = 保留库里原值
    let mut config = body.config;
    if let Some(cfg) = config.as_mut() {
        keep_secrets(cfg, &existing.config);
    }
    if let Some(cfg) = config.as_ref() {
        validate_rss_config(&existing.kind, cfg)?;
    }
    let source = utopia_store::sources::update(
        &state.pool,
        source_id,
        body.name.as_deref(),
        config.as_ref(),
        body.icon.as_deref(),
        body.schedule
            .map(|s| (s.sync_interval_minutes, s.sync_cron)),
    )
    .await?;
    state.emit_source(kb_id);
    // 审计不落凭据:config 只记「改没改」
    let _ = utopia_store::audit::record(
        &state.pool,
        Some(kb_id),
        user.id,
        "source.updated",
        "source",
        Some(source_id),
        json!({ "name": body.name, "config_changed": config.is_some() }),
    )
    .await;
    Ok(Json(json!({ "source": mask_secrets(source) })))
}

/// 批量删除该来源下所有"不在来源中"的文档(url 对账 / custom 墓碑标出的)。
pub async fn cleanup_missing(
    State(state): State<AppState>,
    AuthUser(user): AuthUser,
    Path((kb_id, source_id)): Path<(Uuid, Uuid)>,
) -> ApiResult<Json<serde_json::Value>> {
    require_kb(&state, &user, kb_id, Role::Editor).await?;
    source_in_kb(&state, kb_id, source_id).await?;
    let ids = utopia_store::documents::list_missing(&state.pool, source_id).await?;
    for id in &ids {
        utopia_store::documents::delete(&state.pool, kb_id, *id, Some(user.id)).await?;
        let search = state.search.clone();
        let did = id.to_string();
        tokio::task::spawn_blocking(move || search.delete_document(&did))
            .await
            .map_err(|e| utopia_core::AppError::Other(e.into()))?
            .map_err(utopia_core::AppError::Other)?;
    }
    // 与单篇删除同一个收尾:前提没了,派生当场跟上,而不是挂到下一轮定时推导(#875)。
    // 一批只推一遍
    if !ids.is_empty() {
        super::documents_routes::settle_derivations(&state, kb_id).await?;
    }
    state.emit_source(kb_id);
    Ok(Json(json!({ "deleted": ids.len() })))
}

#[cfg(test)]
#[path = "sources_cleanup_tests.rs"]
mod cleanup_tests;

pub async fn delete(
    State(state): State<AppState>,
    AuthUser(user): AuthUser,
    Path((kb_id, source_id)): Path<(Uuid, Uuid)>,
) -> ApiResult<Json<serde_json::Value>> {
    require_kb(&state, &user, kb_id, Role::Editor).await?;
    // Memory 来源常驻:记忆空间不因来源整理而蒸发(记忆文档本身可在 Library 删除)
    let source = source_in_kb(&state, kb_id, source_id).await?;
    if source.kind == utopia_store::memory::MEMORY_SOURCE_KIND {
        return Err(utopia_core::AppError::invalid(
            "memory_source_permanent",
            "The Memory source is permanent.",
        )
        .into());
    }
    utopia_store::sources::delete(&state.pool, source_id).await?;
    state.emit_source(kb_id);
    let _ = utopia_store::audit::record(
        &state.pool,
        Some(kb_id),
        user.id,
        "source.deleted",
        "source",
        Some(source_id),
        json!({}),
    )
    .await;
    Ok(Json(json!({ "ok": true })))
}

/// 同步运行历史(渠道审计)。
pub async fn runs(
    State(state): State<AppState>,
    AuthUser(user): AuthUser,
    Path((kb_id, source_id)): Path<(Uuid, Uuid)>,
) -> ApiResult<Json<serde_json::Value>> {
    require_kb(&state, &user, kb_id, Role::Viewer).await?;
    source_in_kb(&state, kb_id, source_id).await?;
    let runs = utopia_store::sources::list_runs(&state.pool, source_id, 20).await?;
    Ok(Json(json!({ "runs": runs })))
}

pub async fn sync_now(
    State(state): State<AppState>,
    AuthUser(user): AuthUser,
    Path((kb_id, source_id)): Path<(Uuid, Uuid)>,
) -> ApiResult<Json<serde_json::Value>> {
    require_kb(&state, &user, kb_id, Role::Editor).await?;
    source_in_kb(&state, kb_id, source_id).await?;
    let queued = utopia_store::sources::mark_queued(&state.pool, source_id).await?;
    if queued {
        utopia_store::jobs::enqueue(
            &state.pool,
            "sync_source",
            json!({ "source_id": source_id }),
        )
        .await?;
        state.emit_source(kb_id);
    }
    Ok(Json(json!({ "queued": queued })))
}

#[derive(Deserialize)]
pub struct IngestBody {
    pub filename: String,
    /// 墓碑推送(`deleted: true`)可以不带 content——指南一直这么写,而字段
    /// 从前是必填,缺了就在反序列化那一步被拒。其余情况仍然必填:空串在
    /// 下面的校验里挡
    #[serde(default)]
    pub content: String,
    #[serde(default)]
    pub doc_time: Option<DateTime<Utc>>,
    /// 调用方的逻辑文档 ID:同 ID 再推 = 更新同一文档(原地替换 + 版本记录)。
    /// 不传则以 filename 为身份。
    #[serde(default)]
    pub external_id: Option<String>,
    /// 墓碑:true = 按身份标记"不在来源中"(不删除;仅 api 来源推送支持)。
    /// 此时 content 可省略;再次正常推送同身份会摘掉标记。
    #[serde(default)]
    pub deleted: bool,
}

/// 推送的回执:动作,重复时再加上占着这份内容的文档
fn action_reply(action: crate::ingest_sources::IngestAction) -> serde_json::Value {
    match action {
        crate::ingest_sources::IngestAction::Duplicate(id) => {
            json!({ "action": action_str(action), "duplicate_of": id })
        }
        _ => json!({ "action": action_str(action) }),
    }
}

fn action_str(action: crate::ingest_sources::IngestAction) -> &'static str {
    match action {
        crate::ingest_sources::IngestAction::Created => "created",
        crate::ingest_sources::IngestAction::Updated => "updated",
        crate::ingest_sources::IngestAction::Moved => "moved",
        crate::ingest_sources::IngestAction::Unchanged => "unchanged",
        crate::ingest_sources::IngestAction::Tombstoned => "marked_missing",
        crate::ingest_sources::IngestAction::Duplicate(_) => "duplicate",
    }
}

/// KB 级文本推送(会话认证):普通上传语义——落入 Uploads,无身份追踪。
/// 需要"同 ID 再推 = 更新"语义时,建一个 api 来源用它的密钥推送。
pub async fn ingest(
    State(state): State<AppState>,
    AuthUser(user): AuthUser,
    Path(kb_id): Path<Uuid>,
    Json(body): Json<IngestBody>,
) -> ApiResult<Json<serde_json::Value>> {
    require_kb(&state, &user, kb_id, Role::Editor).await?;
    if body.deleted {
        return Err(utopia_core::AppError::Validation(
            "tombstones need identity tracking — push to an api source instead".into(),
        )
        .into());
    }
    if body.filename.trim().is_empty() || body.content.trim().is_empty() {
        return Err(
            utopia_core::AppError::Validation("filename and content are required".into()).into(),
        );
    }
    let action = crate::ingest_sources::ingest_upload(
        &state,
        kb_id,
        body.filename.trim(),
        "text/plain",
        body.content.as_bytes(),
        body.doc_time,
    )
    .await
    .map_err(utopia_core::AppError::Other)?;
    Ok(Json(action_reply(action)))
}

/// 推送失败的两种性质。**调用方发错了**和**我们这边没接住**得分开:前者记进
/// run 供集成调试,但不算来源同步失败——来源没坏,是那一次请求不合格;
/// 后者才该把来源标成 failed 并进告警中心。从前两者都走 `finish_sync(error)`,
/// 一次格式错误就让铃铛说"来源同步失败,没有新内容进来"
enum PushError {
    /// 4xx:负载不合格(JSON 解析、缺字段)
    Rejected(String),
    /// 摄入本身失败
    Failed(String),
}

impl PushError {
    fn message(&self) -> &str {
        match self {
            PushError::Rejected(m) | PushError::Failed(m) => m,
        }
    }
}

/// 认证之后的推送处理:解析 + 校验 + 摄入/墓碑。错误一律返回文字(记进 run)。
async fn handle_push(
    state: &AppState,
    source: &utopia_core::models::Source,
    bytes: &[u8],
) -> Result<crate::ingest_sources::IngestAction, PushError> {
    let body: IngestBody = serde_json::from_slice(bytes)
        .map_err(|e| PushError::Rejected(format!("Invalid JSON payload: {e}")))?;
    let declared = body
        .external_id
        .as_deref()
        .map(str::trim)
        .filter(|s| !s.is_empty());
    let identity = declared.unwrap_or_else(|| body.filename.trim()).to_string();
    if identity.is_empty() {
        return Err(PushError::Rejected(
            "external_id or filename is required".into(),
        ));
    }
    let key = format!("api:{identity}");

    // 墓碑:标记"不在来源中"(与 custom 的 deleted[] 同一条路径),content 可省略
    if body.deleted {
        utopia_store::documents::mark_missing_keys(&state.pool, source.id, &[key])
            .await
            .map_err(|e| PushError::Failed(e.to_string()))?;
        return Ok(crate::ingest_sources::IngestAction::Tombstoned);
    }

    if body.filename.trim().is_empty() || body.content.trim().is_empty() {
        return Err(PushError::Rejected(
            "filename and content are required".into(),
        ));
    }
    // 带了 external_id 身份就是它:不走「同内容 = 移动」,不把另一个 ID 的文档改成它
    let action = crate::ingest_sources::ingest_item_with_outcome(
        state,
        source.kb_id,
        source.id,
        &key,
        if declared.is_some() {
            crate::ingest_sources::Identity::Declared
        } else {
            crate::ingest_sources::Identity::Path
        },
        body.filename.trim(),
        "text/plain",
        body.content.as_bytes(),
        body.doc_time,
    )
    .await
    .map_err(|e| PushError::Failed(e.to_string()))?
    .action;
    // 失而复得:曾被墓碑标记的身份再次正常推送,摘掉 missing 标记;重复的推送什么也没落,不摘
    if !matches!(action, crate::ingest_sources::IngestAction::Duplicate(_)) {
        utopia_store::documents::clear_missing_keys(&state.pool, source.id, &[key])
            .await
            .map_err(|e| PushError::Failed(e.to_string()))?;
    }
    Ok(action)
}

/// api 来源推送(来源专属密钥认证,无会话):三路身份语义——
/// 新 external_id → 新增;同 ID 同内容 → 无操作;同 ID 新内容 → 原地更新 + 版本记录。
/// 认证通过后每次推送都记一条 run(含格式错误——集成调试全靠它);
/// 未认证请求不写任何记录(不给匿名流量制造落库路径)。
pub async fn push(
    State(state): State<AppState>,
    Path(source_id): Path<Uuid>,
    headers: HeaderMap,
    bytes: axum::body::Bytes,
) -> ApiResult<Json<serde_json::Value>> {
    let source = utopia_store::sources::get(&state.pool, source_id).await?;
    if source.kind != "api" {
        return Err(utopia_core::AppError::NotFound.into());
    }
    // Bearer 密钥校验
    let token = headers
        .get("authorization")
        .and_then(|v| v.to_str().ok())
        .and_then(|v| v.strip_prefix("Bearer "))
        .map(str::trim)
        .filter(|s| !s.is_empty())
        .ok_or(utopia_core::AppError::Unauthorized)?;
    if !push_key_matches(source.ingest_token.as_deref(), token) {
        return Err(utopia_core::AppError::Unauthorized.into());
    }

    let run = utopia_store::sources::start_run(&state.pool, source.id).await?;
    match handle_push(&state, &source, &bytes).await {
        Ok(action) => {
            let (created, updated) = match action {
                crate::ingest_sources::IngestAction::Created => (1, 0),
                crate::ingest_sources::IngestAction::Updated
                | crate::ingest_sources::IngestAction::Moved => (0, 1),
                crate::ingest_sources::IngestAction::Unchanged
                | crate::ingest_sources::IngestAction::Tombstoned
                | crate::ingest_sources::IngestAction::Duplicate(_) => (0, 0),
            };
            utopia_store::sources::finish_run(&state.pool, run, source.id, None, created, updated)
                .await?;
            // 来源行同步反映"上一次推送":操作条/左栏状态点直接可用
            utopia_store::sources::finish_sync(&state.pool, source.id, None, created).await?;
            state.emit_source(source.kb_id);
            Ok(Json(action_reply(action)))
        }
        Err(err) => {
            let msg = err.message().to_string();
            utopia_store::sources::finish_run(&state.pool, run, source.id, Some(&msg), 0, 0)
                .await?;
            // 只有我们这边没接住才算来源失败;调用方发错了留在 run 历史里就够
            if let PushError::Failed(_) = err {
                utopia_store::sources::finish_sync(&state.pool, source.id, Some(&msg), 0).await?;
            }
            state.emit_source(source.kb_id);
            Err(utopia_core::AppError::Validation(msg).into())
        }
    }
}

/// 推陈述(0054):请求体就是开放抽取契约(`e` / `s` / `n`),外加 `api` 推送的那层信封。
///
/// 和 `push` 同一把钥匙、同一套身份语义(`statements:{external_id}`)、同一份 run 记录;
/// 不同的只有两点,都在门口定死:**载荷照原样成为文档,一整块**,抽取时按契约解析而不问
/// 模型;**信封与契约之外的任何键都拒收**——契约里本来就没有属性、类或谓词的格子,
/// 一个写了 `predicate` 的调用方应当在这里得到 422,而不是在图里找不到它以为写进去的类型事实。
pub async fn push_statements(
    State(state): State<AppState>,
    Path(source_id): Path<Uuid>,
    headers: HeaderMap,
    bytes: axum::body::Bytes,
) -> ApiResult<Json<serde_json::Value>> {
    let source = utopia_store::sources::get(&state.pool, source_id).await?;
    if source.kind != "statements" {
        return Err(utopia_core::AppError::NotFound.into());
    }
    let token = headers
        .get("authorization")
        .and_then(|v| v.to_str().ok())
        .and_then(|v| v.strip_prefix("Bearer "))
        .map(str::trim)
        .filter(|s| !s.is_empty())
        .ok_or(utopia_core::AppError::Unauthorized)?;
    if !push_key_matches(source.ingest_token.as_deref(), token) {
        return Err(utopia_core::AppError::Unauthorized.into());
    }

    let run = utopia_store::sources::start_run(&state.pool, source.id).await?;
    match handle_statements_push(&state, &source, &bytes).await {
        Ok(action) => {
            let (created, updated) = match action {
                crate::ingest_sources::IngestAction::Created => (1, 0),
                crate::ingest_sources::IngestAction::Updated
                | crate::ingest_sources::IngestAction::Moved => (0, 1),
                crate::ingest_sources::IngestAction::Unchanged
                | crate::ingest_sources::IngestAction::Tombstoned
                | crate::ingest_sources::IngestAction::Duplicate(_) => (0, 0),
            };
            utopia_store::sources::finish_run(&state.pool, run, source.id, None, created, updated)
                .await?;
            utopia_store::sources::finish_sync(&state.pool, source.id, None, created).await?;
            state.emit_source(source.kb_id);
            Ok(Json(action_reply(action)))
        }
        Err(err) => {
            let msg = err.message().to_string();
            utopia_store::sources::finish_run(&state.pool, run, source.id, Some(&msg), 0, 0)
                .await?;
            if let PushError::Failed(_) = err {
                utopia_store::sources::finish_sync(&state.pool, source.id, Some(&msg), 0).await?;
            }
            state.emit_source(source.kb_id);
            Err(utopia_core::AppError::Validation(msg).into())
        }
    }
}

/// 一次推送最多多少字节、多少条陈述。第一刀的上限,不是契约:一块就是一份载荷,
/// 这两个数只是不让一份载荷大到审核卡片和证据视图没法呈现
pub(crate) const STATEMENTS_MAX_BYTES: usize = 64 * 1024;
const STATEMENTS_MAX_ITEMS: usize = 200;
/// 信封里允许的键。契约的三个数组之外,只有 `api` 推送也有的那三个
const STATEMENTS_ENVELOPE: [&str; 6] = ["external_id", "doc_time", "deleted", "e", "s", "n"];

#[derive(serde::Deserialize)]
struct StatementsBody {
    external_id: String,
    #[serde(default)]
    doc_time: Option<DateTime<Utc>>,
    #[serde(default)]
    deleted: bool,
    #[serde(default)]
    e: serde_json::Value,
    #[serde(default)]
    s: serde_json::Value,
    #[serde(default)]
    n: serde_json::Value,
}

/// 门口的校验:形状对不对、有没有契约之外的键、每条陈述的引文格是不是空的。
/// 通过就按契约重新序列化成文档正文——存的是我们自己写出来的那份,不是调用方发来的
/// 字节:身份、日期(有的话)和 `{e, s, n}`,键序固定、没有多余空白,块就是契约本身
fn validate_statements_payload(raw: &[u8]) -> Result<(StatementsBody, Option<String>), String> {
    if raw.len() > STATEMENTS_MAX_BYTES {
        return Err(format!(
            "payload is {} bytes; the limit is {STATEMENTS_MAX_BYTES}",
            raw.len()
        ));
    }
    let value: serde_json::Value =
        serde_json::from_slice(raw).map_err(|e| format!("Invalid JSON payload: {e}"))?;
    let Some(map) = value.as_object() else {
        return Err("the payload must be a JSON object".into());
    };
    if let Some(extra) = map
        .keys()
        .find(|k| !STATEMENTS_ENVELOPE.contains(&k.as_str()))
    {
        return Err(format!(
            "unknown key {extra:?}: the contract has no slot for it (allowed: external_id, doc_time, deleted, e, s, n)"
        ));
    }
    let body: StatementsBody =
        serde_json::from_value(value).map_err(|e| format!("Invalid payload: {e}"))?;
    if body.external_id.trim().is_empty() {
        return Err("external_id is required".into());
    }
    if body.deleted {
        return Ok((body, None));
    }
    let arrays = [("e", &body.e), ("s", &body.s), ("n", &body.n)];
    for (key, v) in arrays {
        if !v.is_array() {
            return Err(format!("{key} must be an array"));
        }
    }
    let statements = body.s.as_array().expect("checked above");
    if statements.is_empty() {
        return Err("s must hold at least one statement".into());
    }
    if statements.len() > STATEMENTS_MAX_ITEMS {
        return Err(format!(
            "{} statements; the limit is {STATEMENTS_MAX_ITEMS} per push",
            statements.len()
        ));
    }
    for (i, item) in statements.iter().enumerate() {
        let Some(arr) = item.as_array() else {
            return Err(format!("s[{i}] must be an array of eight slots"));
        };
        if arr.len() != 8 {
            return Err(format!(
                "s[{i}] has {} slots; the contract has eight",
                arr.len()
            ));
        }
        if !arr[0].is_null() {
            return Err(format!(
                "s[{i}][0] (quote) must be null: the item is its own evidence"
            ));
        }
        if !arr[5].is_null() && !arr[5].is_object() {
            return Err(format!("s[{i}][5] (qualifiers) must be an object or null"));
        }
    }
    for (i, item) in body.n.as_array().expect("checked above").iter().enumerate() {
        let Some(arr) = item.as_array() else {
            return Err(format!("n[{i}] must be an array"));
        };
        if arr.len() > 2 && !arr[2].is_null() {
            return Err(format!("n[{i}][2] (quote) must be null"));
        }
    }
    // 存下的文档带着这次观测的身份和日期,再是契约的三个数组(#900)。身份在正文里,
    // 两次看到同一件事就是两篇正文不同的文档:库里一份内容只能有一篇(`documents_kb_sha_idx`),
    // 文件型来源那条「同内容出现在新路径 = 改名」的识别也就永远碰不到它。解析器只读
    // `e` / `s` / `n`,多出的两个键它不看
    // 只有契约能通过 parse_open_response:这一步在门口跑一遍,抽取时不会再有别的答案
    let mut stored = serde_json::Map::new();
    stored.insert("external_id".into(), json!(body.external_id.trim()));
    if let Some(at) = body.doc_time {
        stored.insert("doc_time".into(), json!(at));
    }
    stored.insert("e".into(), body.e.clone());
    stored.insert("s".into(), body.s.clone());
    stored.insert("n".into(), body.n.clone());
    let content = serde_json::to_string_pretty(&serde_json::Value::Object(stored))
        .map_err(|e| format!("cannot serialise the contract: {e}"))?;
    let parsed = utopia_extract::open::parse_open_response(&content)
        .map_err(|e| format!("the contract does not parse: {e}"))?;
    if parsed.skipped > 0 {
        return Err(format!(
            "{} item(s) are malformed for the contract (e: [name, kind, named]; s: [null, subject, phrase, object, value, qualifiers, when, ended]; n: [entity, name, null])",
            parsed.skipped
        ));
    }
    if parsed.statements.is_empty() {
        return Err("no statement survived parsing".into());
    }
    // 主语和别名所属的东西必须在 `e` 里(0054 决定 5):抽取时找不到的会作为 UNKNOWN_REF
    // 静默丢掉,而门口的职责就是不让调用方以为它写进去了。折叠规则与抽取的 `name_key`
    // 同一条:只许空白和大小写不同。宾语不在此列——它落成字面值,陈述照落
    let listed: std::collections::HashSet<String> = parsed
        .entities
        .iter()
        .map(|e| crate::extraction_open::name_key(&e.name))
        .collect();
    for (i, st) in parsed.statements.iter().enumerate() {
        if !listed.contains(&crate::extraction_open::name_key(&st.subject)) {
            return Err(format!(
                "s[{i}][1] (subject) {:?} is not a thing listed in e",
                st.subject
            ));
        }
    }
    for (i, n) in parsed.names.iter().enumerate() {
        if !listed.contains(&crate::extraction_open::name_key(&n.entity)) {
            return Err(format!(
                "n[{i}][0] (entity) {:?} is not a thing listed in e",
                n.entity
            ));
        }
    }
    Ok((body, Some(content)))
}

async fn handle_statements_push(
    state: &AppState,
    source: &utopia_core::models::Source,
    bytes: &[u8],
) -> Result<crate::ingest_sources::IngestAction, PushError> {
    let (body, content) = validate_statements_payload(bytes).map_err(PushError::Rejected)?;
    let identity = body.external_id.trim().to_string();
    let key = format!("statements:{identity}");
    let Some(content) = content else {
        utopia_store::documents::mark_missing_keys(&state.pool, source.id, &[key])
            .await
            .map_err(|e| PushError::Failed(e.to_string()))?;
        return Ok(crate::ingest_sources::IngestAction::Tombstoned);
    };
    let filename = format!("{identity}.json");
    let action = crate::ingest_sources::ingest_item_with_outcome(
        state,
        source.kb_id,
        source.id,
        &key,
        crate::ingest_sources::Identity::Declared,
        &filename,
        "application/json",
        content.as_bytes(),
        body.doc_time,
    )
    .await
    .map_err(|e| PushError::Failed(e.to_string()))?
    .action;
    if !matches!(action, crate::ingest_sources::IngestAction::Duplicate(_)) {
        utopia_store::documents::clear_missing_keys(&state.pool, source.id, &[key])
            .await
            .map_err(|e| PushError::Failed(e.to_string()))?;
    }
    Ok(action)
}

/// 来源级全量重抽(增量语义):该来源下所有 ready 文档重新过一遍抽取。
/// 走正常管道——实体消解、事实去重、时态冲突照常,既有人工决策全部保留。
pub async fn re_extract(
    State(state): State<AppState>,
    AuthUser(user): AuthUser,
    Path((kb_id, source_id)): Path<(Uuid, Uuid)>,
) -> ApiResult<Json<serde_json::Value>> {
    require_kb(&state, &user, kb_id, Role::Editor).await?;
    let source = utopia_store::sources::get(&state.pool, source_id).await?;
    if source.kb_id != kb_id {
        return Err(utopia_core::AppError::NotFound.into());
    }
    // `queue_extraction` 自己也会把这种来源的文档滤掉,那样这里回的是「排了 0 篇」。
    // 点按钮的人该听到的是为什么
    if !source.extracts() {
        return Err(utopia_core::AppError::invalid(
            "source_not_extracted",
            "Documents under this source are searched, not extracted",
        )
        .into());
    }
    // 任务由 queue_extraction 与状态同事务建好,这里只负责推送
    let ids =
        utopia_store::documents::queue_extraction(&state.pool, kb_id, Some(source_id)).await?;
    for id in &ids {
        state.emit_document(kb_id, *id);
    }
    let _ = utopia_store::audit::record(
        &state.pool,
        Some(kb_id),
        user.id,
        "source.re_extract",
        "source",
        Some(source_id),
        json!({ "name": source.name, "documents": ids.len() }),
    )
    .await;
    Ok(Json(json!({ "queued": ids.len() })))
}

#[cfg(test)]
#[path = "sources_statements_tests.rs"]
mod statements_tests;

#[cfg(test)]
mod tests {
    use super::{keep_secrets, validate_rss_config};
    use serde_json::json;

    /// 认证判断写反是这类改动唯一值得担心的事(`ct_eq` 给的是 `Choice` 不是
    /// `bool`,判反一次就是全放行且一切看起来正常),所以两个方向都钉住
    #[test]
    fn only_the_key_itself_gets_in() {
        let key = super::new_ingest_token();
        assert!(super::push_key_matches(Some(&key), &key));
        let mut off_by_one = key.clone();
        off_by_one.pop();
        off_by_one.push(if key.ends_with('a') { 'b' } else { 'a' });
        assert!(
            !super::push_key_matches(Some(&key), &off_by_one),
            "one character off is not the key"
        );
        assert!(
            !super::push_key_matches(Some(&key), &key[..key.len() - 1]),
            "a prefix is not the key"
        );
        assert!(
            !super::push_key_matches(Some(&key), &format!("{key}x")),
            "the key with something after it is not the key"
        );
        assert!(
            !super::push_key_matches(None, &key),
            "a source that never had a key takes nobody's word"
        );
    }

    #[test]
    fn a_blank_or_missing_secret_keeps_the_stored_one() {
        let existing = json!({ "bucket": "old", "secret_access_key": "s", "password": "p" });
        // 没传 → 留;空串 → 留;有值 → 换;null → 删
        let mut next = json!({ "bucket": "new", "password": "  ", "token": null });
        keep_secrets(&mut next, &existing);
        assert_eq!(next["bucket"], "new");
        assert_eq!(
            next["secret_access_key"], "s",
            "missing keeps the stored value"
        );
        assert_eq!(next["password"], "p", "blank keeps the stored value");
        assert!(next.get("token").is_none(), "an explicit null removes it");
        let mut next = json!({ "secret_access_key": "fresh" });
        keep_secrets(&mut next, &existing);
        assert_eq!(next["secret_access_key"], "fresh");
        assert_eq!(next["password"], "p");
    }

    #[test]
    fn no_secret_reaches_a_response() {
        let source = utopia_core::models::Source {
            id: uuid::Uuid::nil(),
            kb_id: uuid::Uuid::nil(),
            kind: "s3".into(),
            name: "s".into(),
            config: json!({ "bucket": "b", "access_key_id": "AKIA", "secret_access_key": "x",
                            "account_key": "y", "service_account_key": "z", "password": "w",
                            "token": "t", "auth_header": "h" }),
            icon: None,
            sync_interval_minutes: None,
            sync_cron: None,
            last_sync_at: None,
            last_sync_status: "never".into(),
            last_sync_error: None,
            last_sync_added: 0,
            ingest_token: Some("utp_x".into()),
            created_at: chrono::Utc::now(),
        };
        let masked = super::mask_secrets(source);
        let obj = masked.config.as_object().unwrap();
        for key in utopia_core::models::SOURCE_SECRET_KEYS {
            assert!(!obj.contains_key(*key), "{key} leaked");
        }
        assert_eq!(obj["bucket"], "b");
        assert_eq!(
            obj["access_key_id"], "AKIA",
            "an identifier is not a secret"
        );
    }

    #[test]
    fn invalid_rss_content_mode_is_rejected_at_the_route_boundary() {
        assert!(validate_rss_config("rss", &json!({ "content_mode": "all" })).is_err());
        assert!(validate_rss_config("url", &json!({ "content_mode": "all" })).is_ok());
    }
}