Skip to content

World's first open-source enterprise world model.

20.4 KB
Raw Blame
//! 摄入来源仓储:"来源即文件夹"——source 是容器,挂着它摄入的文档,可定时同步。

use chrono::{DateTime, Utc};
use sqlx::PgPool;
use utopia_core::models::{Role, Source, SourceKind, SourceView, SyncRun, SOURCE_SECRET_KEYS};
use utopia_core::{secrets, AppError, AppResult};
use uuid::Uuid;

/// folder = 纯容器(上传/拖拽入内,无同步语义);url/rss = 拉取型;api = 推送型。
/// 本机目录监听(watch_folder)已否决——自部署用户看不到服务器磁盘;
/// 对象存储 / WebDAV / Notion 是它的替代形态(0013)。
/// custom = 自定义拉取器:任何实现 Utopia ingest 接口的 URL(返回 items JSON)即可定时摄取。
/// github_issues / jira_issues = 工单:一张工单连同它的状态变更史成为一篇文档。
///
/// 种类的清单**不在这里写**:`SourceKind`(utopia-core)一个枚举出全部——创建的白名单、
/// 同步的分派、前端的下拉框(有测试对表)。从前这里有一张手写的 `KINDS`,五种连接器
/// 加了同步却没进这张表,界面上选得到、建不出来(#247)
pub fn creatable_kinds() -> Vec<&'static str> {
    SourceKind::creatable().map(|k| k.as_str()).collect()
}

/// 校验并规范化标准 5 段 cron 表达式(内部用 cron crate 的 6 段:补秒位)。
pub fn validate_cron(expr: &str) -> AppResult<String> {
    let normalized = expr.split_whitespace().collect::<Vec<_>>().join(" ");
    let fields = normalized.split(' ').count();
    if fields != 5 {
        return Err(AppError::invalid_detail(
            "bad_cron_fields",
            "Cron expression must have 5 fields (minute hour day month weekday)",
            format!("got {fields}"),
        ));
    }
    use std::str::FromStr;
    cron::Schedule::from_str(&format!("0 {normalized}")).map_err(|e| {
        AppError::invalid_detail("bad_cron", "Invalid cron expression", e.to_string())
    })?;
    Ok(normalized)
}

pub const RSS_CONTENT_MODE_KEY: &str = "content_mode";
pub const RSS_FULL_CONTENT_MODE: &str = "full_new_items";

/// RSS mode is deliberately a small enum in the persisted source config. Old
/// rows without the key retain the legacy feed behavior.
pub fn rss_content_mode(config: &serde_json::Value) -> AppResult<&'static str> {
    let Some(value) = config.get(RSS_CONTENT_MODE_KEY) else {
        return Ok("feed");
    };
    let Some(value) = value.as_str() else {
        return Err(AppError::invalid(
            "rss_content_mode_invalid",
            "RSS content_mode must be feed or full_new_items",
        ));
    };
    match value {
        "feed" => Ok("feed"),
        RSS_FULL_CONTENT_MODE => Ok(RSS_FULL_CONTENT_MODE),
        _ => Err(AppError::invalid(
            "rss_content_mode_invalid",
            "RSS content_mode must be feed or full_new_items",
        )),
    }
}

pub fn rss_full_content_enabled(kind: &str, config: &serde_json::Value) -> AppResult<bool> {
    if SourceKind::parse(kind) != Some(SourceKind::Rss) {
        return Ok(false);
    }
    Ok(rss_content_mode(config)? == RSS_FULL_CONTENT_MODE)
}

/// cron 的下一次触发时刻(服务器本地时区)。
fn cron_next_after(expr: &str, after: DateTime<Utc>) -> Option<DateTime<Utc>> {
    use std::str::FromStr;
    let schedule = cron::Schedule::from_str(&format!("0 {expr}")).ok()?;
    let local_after = after.with_timezone(&chrono::Local);
    schedule
        .after(&local_after)
        .next()
        .map(|t| t.with_timezone(&Utc))
}

pub async fn list(pool: &PgPool, kb_id: Uuid) -> AppResult<Vec<SourceView>> {
    // config 剔掉凭据:列表给 Viewer 看,哪一种连接器的密钥都不下发。
    // 键在 `SOURCE_SECRET_KEYS` 一张表上——从前这里只减 `auth_header`,五种连接器
    // 的密钥就这么漏出去的(#246)
    // 外层别名不能与 ENTRY_SELECT 的来源别名重名;来源和当前代谓词必须留在投影内部,
    // 否则 PostgreSQL 会先计算部署中的全部 observation,再丢弃与当前来源无关的行。
    let rows: Vec<SourceView> = sqlx::query_as(&format!(
        "SELECT listed_source.id, listed_source.kind, listed_source.name,
                listed_source.config - $2::text[] AS config, listed_source.icon,
                listed_source.sync_interval_minutes, listed_source.sync_cron,
                listed_source.last_sync_at, listed_source.last_sync_status,
                listed_source.last_sync_error, listed_source.last_sync_added,
                (SELECT count(*) FROM documents d
                 WHERE d.source_id = listed_source.id AND d.deleted_at IS NULL) AS doc_count,
                (SELECT count(*) FROM documents d
                 WHERE d.source_id = listed_source.id AND d.missing_since IS NOT NULL
                   AND d.deleted_at IS NULL) AS missing_count,
                -- 整块出去,不是八个平铺的列(0026 / #417)。**不是 RSS 的来源
                -- 这一格就是 NULL**:从前「不适用」在 state 上写作 NULL、在五个
                -- 计数上写作 0,同一件事两套说法,而 `queued_count: 0` 读起来
                -- 像「队列空着」。计数在块里仍然 COALESCE 成 0——这时候它真是
                -- 「这一档现在没有」,不是「不适用」。
                CASE WHEN listed_source.kind = 'rss' THEN jsonb_build_object(
                    'state', CASE
                        WHEN listed_source.config->>'content_mode' IS DISTINCT FROM 'full_new_items'
                            THEN 'disabled'
                        WHEN listed_source.rss_baselined_at IS NULL THEN 'pending'
                        ELSE 'active' END,
                    'pending', COALESCE(hydration.pending, 0),
                    'queued', COALESCE(hydration.queued, 0),
                    'retrying', COALESCE(hydration.retrying, 0),
                    'complete', COALESCE(hydration.complete, 0),
                    'terminal', COALESCE(hydration.terminal, 0)
                ) END AS rss_full_content
         FROM sources listed_source
         LEFT JOIN LATERAL (
             SELECT
                 (count(*) FILTER (WHERE projected.state = 'baseline'))::int AS baseline_count,
                 count(*) FILTER (WHERE projected.state = 'pending') AS pending,
                 count(*) FILTER (WHERE projected.state IN ('queued', 'hydrating')) AS queued,
                 count(*) FILTER (WHERE projected.state = 'retry_wait') AS retrying,
                 count(*) FILTER (WHERE projected.state = 'complete') AS complete,
                 count(*) FILTER (
                     WHERE projected.state IN ('terminal', 'deleted', 'superseded')
                 ) AS terminal
             FROM (
                 {}
                 WHERE listed_source.kind = 'rss'
                   AND e.source_id = listed_source.id
                   AND e.activation_generation = listed_source.rss_generation
             ) projected
         ) hydration ON listed_source.kind = 'rss'
         WHERE listed_source.kb_id = $1 ORDER BY listed_source.created_at",
        crate::rss_full_content::ENTRY_SELECT
    ))
    .bind(kb_id)
    .bind(SOURCE_SECRET_KEYS)
    .fetch_all(pool)
    .await?;
    Ok(rows)
}

/// 出库即开封:配置里的凭据键与推送密钥在库里是封印的(`utopia_core::secrets`)。
/// 任何返回 `Source` 的查询都从这里过——`list` 不用,它在 SQL 里就把凭据键剔了
fn opened(mut s: Source) -> AppResult<Source> {
    secrets::open_json_keys(&mut s.config, SOURCE_SECRET_KEYS).map_err(AppError::Other)?;
    s.ingest_token = secrets::open_opt(s.ingest_token.as_deref()).map_err(AppError::Other)?;
    Ok(s)
}

pub async fn get(pool: &PgPool, id: Uuid) -> AppResult<Source> {
    let row: Option<Source> = sqlx::query_as("SELECT * FROM sources WHERE id = $1")
        .bind(id)
        .fetch_optional(pool)
        .await?;
    opened(row.ok_or(AppError::NotFound)?)
}

#[allow(clippy::too_many_arguments)]
pub async fn create(
    pool: &PgPool,
    kb_id: Uuid,
    kind: &str,
    name: &str,
    config: &serde_json::Value,
    icon: Option<&str>,
    sync_interval_minutes: Option<i32>,
    sync_cron: Option<&str>,
) -> AppResult<Source> {
    if !SourceKind::parse(kind).is_some_and(|k| k.creatable_by_hand()) {
        return Err(AppError::Validation(format!(
            "kind must be one of: {}",
            creatable_kinds().join(", ")
        )));
    }
    if name.trim().is_empty() {
        return Err(AppError::invalid(
            "source_name_required",
            "Source name is required",
        ));
    }
    // 互斥:cron 优先(UI 只会传其一)
    let cron_norm = sync_cron.map(validate_cron).transpose()?;
    let interval = if cron_norm.is_some() {
        None
    } else {
        sync_interval_minutes
    };
    // serde 缺省的 Value::Null 会以 jsonb null 落库,前端读 config.x 直接炸——规范化为空对象
    let mut config = if config.is_null() {
        serde_json::json!({})
    } else {
        config.clone()
    };
    secrets::seal_json_keys(&mut config, SOURCE_SECRET_KEYS);
    let full_content = rss_full_content_enabled(kind, &config)?;
    let source_id = Uuid::now_v7();
    let mut tx = pool.begin().await?;
    let source: Source = sqlx::query_as(
        "INSERT INTO sources (id, kb_id, kind, name, config, icon, sync_interval_minutes, sync_cron)
         VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING *",
    )
    .bind(source_id)
    .bind(kb_id)
    .bind(kind)
    .bind(name.trim())
    .bind(config)
    .bind(icon)
    .bind(interval)
    .bind(cron_norm)
    .fetch_one(&mut *tx)
    .await?;
    if full_content {
        crate::rss_full_content::initialize_source(&mut tx, source.id).await?;
    }
    tx.commit().await?;
    opened(source)
}

/// 设置 api 来源的推送密钥(创建 / 轮换时)。
pub async fn set_ingest_token(pool: &PgPool, source_id: Uuid, token: &str) -> AppResult<()> {
    let res = sqlx::query("UPDATE sources SET ingest_token = $2 WHERE id = $1")
        .bind(source_id)
        .bind(secrets::seal(token))
        .execute(pool)
        .await?;
    if res.rows_affected() == 0 {
        return Err(AppError::NotFound);
    }
    Ok(())
}

/// 更新调度:interval 与 cron 互斥,任一被显式设置时都会覆盖两者。
fn rss_feed_url(config: &serde_json::Value) -> Option<&str> {
    config
        .get("feed_url")
        .and_then(|value| value.as_str())
        .map(str::trim)
        .filter(|value| !value.is_empty())
}

#[allow(clippy::too_many_arguments)]
pub async fn update(
    pool: &PgPool,
    id: Uuid,
    name: Option<&str>,
    config: Option<&serde_json::Value>,
    icon: Option<&str>,
    schedule: Option<(Option<i32>, Option<String>)>,
) -> AppResult<Source> {
    let schedule = match schedule {
        Some((interval, cron)) => {
            let cron_norm = cron.as_deref().map(validate_cron).transpose()?;
            let interval = if cron_norm.is_some() { None } else { interval };
            Some((interval, cron_norm))
        }
        None => None,
    };

    let mut tx = pool.begin().await?;
    let previous: Source = sqlx::query_as("SELECT * FROM sources WHERE id = $1 FOR UPDATE")
        .bind(id)
        .fetch_optional(&mut *tx)
        .await?
        .ok_or(AppError::NotFound)?;
    let mut next_config = config
        .map(|value| {
            if value.is_null() {
                serde_json::json!({})
            } else {
                value.clone()
            }
        })
        .unwrap_or_else(|| previous.config.clone());
    secrets::seal_json_keys(&mut next_config, SOURCE_SECRET_KEYS);
    let old_full = rss_full_content_enabled(&previous.kind, &previous.config)?;
    let new_full = rss_full_content_enabled(&previous.kind, &next_config)?;
    let feed_url_changed = previous.kind == "rss"
        && old_full
        && new_full
        && rss_feed_url(&previous.config) != rss_feed_url(&next_config);
    let source = sqlx::query_as(
        "UPDATE sources SET
            name = COALESCE($2, name),
            config = COALESCE($3, config),
            icon = COALESCE($4, icon),
            sync_interval_minutes = CASE WHEN $5 THEN $6 ELSE sync_interval_minutes END,
            sync_cron = CASE WHEN $5 THEN $7 ELSE sync_cron END
         WHERE id = $1 RETURNING *",
    )
    .bind(id)
    .bind(name)
    .bind(Some(next_config))
    .bind(icon)
    .bind(schedule.is_some())
    .bind(schedule.as_ref().and_then(|(i, _)| *i))
    .bind(schedule.as_ref().and_then(|(_, c)| c.clone()))
    .fetch_one(&mut *tx)
    .await?;

    if (!old_full && new_full) || feed_url_changed {
        crate::rss_full_content::enable_source(&mut tx, id).await?;
    }
    tx.commit().await?;
    opened(source)
}

/// 删除来源;其文档保留(source_id 置 NULL,落回 Uploads 组)。
pub async fn delete(pool: &PgPool, id: Uuid) -> AppResult<()> {
    let res = sqlx::query("DELETE FROM sources WHERE id = $1")
        .bind(id)
        .execute(pool)
        .await?;
    if res.rows_affected() == 0 {
        return Err(AppError::NotFound);
    }
    Ok(())
}

/// 到期待同步的来源(调度器每分钟扫)。
/// interval 型在 SQL 里判定;cron 型取回 Rust 侧求下一次触发时刻再过滤。
pub async fn due_sources(pool: &PgPool) -> AppResult<Vec<Source>> {
    let rows: Vec<Source> = sqlx::query_as(
        "SELECT * FROM sources
         WHERE last_sync_status NOT IN ('queued', 'running')
           AND ((sync_interval_minutes IS NOT NULL
                 AND (last_sync_at IS NULL
                      OR last_sync_at + make_interval(mins => sync_interval_minutes) <= now()))
                OR sync_cron IS NOT NULL)",
    )
    .fetch_all(pool)
    .await?;

    let now = chrono::Utc::now();
    let rows = rows
        .into_iter()
        .map(opened)
        .collect::<AppResult<Vec<_>>>()?;
    Ok(rows
        .into_iter()
        .filter(|s| match &s.sync_cron {
            None => true, // interval 型已在 SQL 判定
            Some(expr) => {
                // 基准取上次同步时刻(没同步过取创建时刻):错过的触发点在下一轮扫描补上
                let anchor = s.last_sync_at.unwrap_or(s.created_at);
                cron_next_after(expr, anchor).is_some_and(|next| next <= now)
            }
        })
        .collect())
}

/// 标记入队(幂等:已在队列/运行中则返回 false,避免重复入队)。
pub async fn mark_queued(pool: &PgPool, id: Uuid) -> AppResult<bool> {
    let res = sqlx::query(
        "UPDATE sources SET last_sync_status = 'queued'
         WHERE id = $1 AND last_sync_status NOT IN ('queued', 'running')",
    )
    .bind(id)
    .execute(pool)
    .await?;
    Ok(res.rows_affected() > 0)
}

pub async fn mark_running(pool: &PgPool, id: Uuid) -> AppResult<()> {
    sqlx::query("UPDATE sources SET last_sync_status = 'running' WHERE id = $1")
        .bind(id)
        .execute(pool)
        .await?;
    Ok(())
}

/// 一次同步收尾。失败时记一条告警,成功时什么都不做——
/// **"现在好了没有"不是告警中心该回答的问题**,来源页面上就写着。
///
/// 返回值是"记了没有",调用方据此决定要不要推事件。
pub async fn finish_sync(
    pool: &PgPool,
    id: Uuid,
    error: Option<&str>,
    added: i32,
) -> AppResult<bool> {
    let row: Option<(Uuid, String)> = sqlx::query_as(
        "UPDATE sources SET last_sync_status = $2, last_sync_error = $3,
                last_sync_added = $4, last_sync_at = now()
         WHERE id = $1
         RETURNING kb_id, name",
    )
    .bind(id)
    .bind(if error.is_some() { "failed" } else { "ok" })
    .bind(error)
    .bind(added)
    .fetch_optional(pool)
    .await?;
    let Some((kb_id, name)) = row else {
        return Ok(false);
    };
    let Some(msg) = error else {
        return Ok(false);
    };
    crate::alerts::raise(
        pool,
        crate::alerts::NewAlert {
            kb_id: Some(kb_id),
            severity: "error",
            kind: crate::alerts::kind::SOURCE_SYNC_FAILED,
            // 内容类给 editor,不只给 admin:管理员需要知道该修连接了,
            // 但**配这个源的人**更需要知道你的东西没进来
            min_role: Role::Editor,
            subject_type: Some("source"),
            subject_id: Some(id),
            // 名字存一份:源被删之后 subject_id 解析不出名字,而告警该留得住
            detail: serde_json::json!({ "name": name, "error": msg }),
        },
    )
    .await?;
    Ok(true)
}

/// 记录同步时刻(避免调度器在长同步过程中重复触发后又立刻到期)。
pub async fn touch_sync_time(pool: &PgPool, id: Uuid, at: DateTime<Utc>) -> AppResult<()> {
    sqlx::query("UPDATE sources SET last_sync_at = $2 WHERE id = $1")
        .bind(id)
        .bind(at)
        .execute(pool)
        .await?;
    Ok(())
}

// ---------------------------------------------------------------------------
// 同步运行记录(渠道审计历史)
// ---------------------------------------------------------------------------

/// 调度时钟包含失败尝试,不能作增量游标。回到上次成功运行的开始,
/// 让那次拉取期间发生的更新也能在下一轮读到;没有成功记录就重新全量读取。
pub async fn last_successful_sync_start(
    pool: &PgPool,
    source_id: Uuid,
) -> AppResult<Option<DateTime<Utc>>> {
    Ok(sqlx::query_scalar(
        "SELECT max(started_at) FROM source_sync_runs WHERE source_id = $1 AND status = 'ok'",
    )
    .bind(source_id)
    .fetch_one(pool)
    .await?)
}

pub async fn start_run(pool: &PgPool, source_id: Uuid) -> AppResult<Uuid> {
    let id = Uuid::now_v7();
    sqlx::query("INSERT INTO source_sync_runs (id, source_id) VALUES ($1, $2)")
        .bind(id)
        .bind(source_id)
        .execute(pool)
        .await?;
    Ok(id)
}

pub async fn finish_run(
    pool: &PgPool,
    run_id: Uuid,
    source_id: Uuid,
    error: Option<&str>,
    created_docs: i32,
    updated_docs: i32,
) -> AppResult<()> {
    sqlx::query(
        "UPDATE source_sync_runs SET finished_at = now(), status = $2, error = $3,
                created_docs = $4, updated_docs = $5
         WHERE id = $1",
    )
    .bind(run_id)
    .bind(if error.is_some() { "failed" } else { "ok" })
    .bind(error)
    .bind(created_docs)
    .bind(updated_docs)
    .execute(pool)
    .await?;
    // 每来源只留最近 50 条
    sqlx::query(
        "DELETE FROM source_sync_runs WHERE source_id = $1 AND id NOT IN
         (SELECT id FROM source_sync_runs WHERE source_id = $1
          ORDER BY started_at DESC LIMIT 50)",
    )
    .bind(source_id)
    .execute(pool)
    .await?;
    Ok(())
}

pub async fn list_runs(pool: &PgPool, source_id: Uuid, limit: i64) -> AppResult<Vec<SyncRun>> {
    let rows: Vec<SyncRun> = sqlx::query_as(
        "SELECT id, started_at, finished_at, status, created_docs, updated_docs, error
         FROM source_sync_runs WHERE source_id = $1
         ORDER BY started_at DESC LIMIT $2",
    )
    .bind(source_id)
    .bind(limit)
    .fetch_all(pool)
    .await?;
    Ok(rows)
}

#[cfg(test)]
mod tests {
    use super::*;

    #[test]
    fn cron_validation() {
        assert_eq!(validate_cron("30 9 * * *").unwrap(), "30 9 * * *");
        assert_eq!(
            validate_cron("  0  9 * * Mon,Thu ").unwrap(),
            "0 9 * * Mon,Thu"
        );
        assert!(validate_cron("9 * * *").is_err()); // 4 段
        assert!(validate_cron("99 9 * * *").is_err()); // 分钟越界
        assert!(validate_cron("0 0 0 0 0 0").is_err()); // 6 段
    }

    #[test]
    fn cron_next_computes() {
        let after = chrono::Utc::now();
        let next = cron_next_after("*/5 * * * *", after).unwrap();
        assert!(next > after);
        assert!((next - after).num_minutes() <= 5);
    }

    #[test]
    fn rss_content_mode_defaults_and_validates() {
        assert_eq!(rss_content_mode(&serde_json::json!({})).unwrap(), "feed");
        assert_eq!(
            rss_content_mode(&serde_json::json!({ "content_mode": "feed" })).unwrap(),
            "feed"
        );
        assert_eq!(
            rss_content_mode(&serde_json::json!({ "content_mode": "full_new_items" })).unwrap(),
            "full_new_items"
        );
        assert!(rss_content_mode(&serde_json::json!({ "content_mode": "all" })).is_err());
        assert!(rss_content_mode(&serde_json::json!({ "content_mode": true })).is_err());
        assert!(!rss_full_content_enabled(
            "url",
            &serde_json::json!({
                "content_mode": "full_new_items"
            })
        )
        .unwrap());
    }
}