World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
//! 问数数据源:系统层注册(admin,凭据只进不出)+ 知识库层挂载(库 admin)。
//! 挂载/手动刷新时把目标库的 schema 生成 markdown 摄入 KB(同 key 原地更新),
//! Chat 写 SQL 前可检索到表结构。查询执行的安全闸在刀 2(query_data 工具)。
use axum::extract::{Path, State};
use axum::Json;
use serde::Deserialize;
use serde_json::json;
use utopia_core::models::Role;
use utopia_core::AppError;
use uuid::Uuid;
use super::graph_routes::require_kb;
use crate::auth::AuthUser;
use crate::error::ApiResult;
use crate::state::AppState;
fn require_admin(user: &utopia_core::models::User) -> Result<(), AppError> {
if user.is_admin {
Ok(())
} else {
Err(AppError::Forbidden)
}
}
// ---------------------------------------------------------------------------
// 系统层:注册/测试/删除
// ---------------------------------------------------------------------------
pub async fn list(
State(state): State<AppState>,
AuthUser(user): AuthUser,
) -> ApiResult<Json<serde_json::Value>> {
require_admin(&user)?;
let sources = utopia_store::datasources::list(&state.pool).await?;
Ok(Json(json!({ "data_sources": sources })))
}
#[derive(Deserialize)]
pub struct CreateBody {
pub name: String,
#[serde(default = "default_engine")]
pub engine: String,
pub conn_string: String,
}
fn default_engine() -> String {
"postgres".into()
}
pub async fn create(
State(state): State<AppState>,
AuthUser(user): AuthUser,
Json(body): Json<CreateBody>,
) -> ApiResult<Json<serde_json::Value>> {
require_admin(&user)?;
// 引擎跟着 scheme 走,界面只有一个连接串输入框;body.engine 只为兼容旧调用留着
let engine = crate::query_engine::engine_from_conn(&body.conn_string).ok_or_else(|| {
utopia_core::AppError::invalid(
"unsupported_conn_scheme",
format!(
"Connection string must start with one of: postgres://, mysql://, trino://, databricks://, snowflake://, clickhouse:// (engines: {})",
crate::query_engine::ENGINES.join(", ")
),
)
})?;
let _ = &body.engine;
// 连接串的形状在登记时就校验(缺令牌、缺 warehouse……),错误信息里带写法;
// 否则要等到「测试」才知道,而那一步只回 ok:false
crate::query_engine::engine_for(engine, &body.conn_string)
.map_err(|e| utopia_core::AppError::invalid("bad_conn_string", e.to_string()))?;
let id = utopia_store::datasources::create(
&state.pool,
&body.name,
engine,
&body.conn_string,
user.id,
)
.await?;
// **登记完就能挂。** 授权是按工作区的(0014),可工作区在界面上已经隐形——
// 单租户部署只有一个,谁也没见过它的名字。从前登记完还要在卡片上先
// 「授权给工作区」,等于让人授权一件从没见过的东西,挂载时只得到一句
// 「未授权」。所以登记人所在的每个工作区一并授权;要收窄,卡片上仍能撤销
for ws in utopia_store::workspaces::list_for_user(&state.pool, user.id).await? {
utopia_store::datasources::grant(&state.pool, id, ws.id, user.id).await?;
}
Ok(Json(json!({ "id": id })))
}
pub async fn delete(
State(state): State<AppState>,
AuthUser(user): AuthUser,
Path(id): Path<Uuid>,
) -> ApiResult<Json<serde_json::Value>> {
require_admin(&user)?;
utopia_store::datasources::delete(&state.pool, id).await?;
Ok(Json(json!({ "ok": true })))
}
#[derive(Deserialize)]
pub struct TestConnBody {
pub conn_string: String,
}
/// 存之前先试一次。**这一步不落库**:登记表单里那个「测试连接」按的是它。
///
/// 从前只能先存下来再测,测不通还得回头把这条删掉;而 [`test`] 只回 `ok: false`,
/// 连为什么不通都不说。这里把引擎的报错原样带回去——密码错、库名拼错、
/// 端口不通,是三件不同的事,人得知道是哪一件。
pub async fn test_conn(
AuthUser(user): AuthUser,
Json(body): Json<TestConnBody>,
) -> ApiResult<Json<serde_json::Value>> {
require_admin(&user)?;
let engine = crate::query_engine::engine_from_conn(&body.conn_string).ok_or_else(|| {
AppError::invalid(
"unsupported_conn_scheme",
format!(
"Connection string must start with one of: postgres://, mysql://, trino://, databricks://, snowflake://, clickhouse:// (engines: {})",
crate::query_engine::ENGINES.join(", ")
),
)
})?;
// 连接串的形状(缺令牌、缺 warehouse……)与连得上连不上分开报:前者是写错了,
// 后者是环境问题,两句话对人的下一步动作不一样
let eng = crate::query_engine::engine_for(engine, &body.conn_string)
.map_err(|e| AppError::invalid("bad_conn_string", e.to_string()))?;
match eng.test().await {
Ok(()) => Ok(Json(json!({ "ok": true, "engine": engine }))),
Err(e) => Ok(Json(
json!({ "ok": false, "engine": engine, "error": e.to_string() }),
)),
}
}
pub async fn test(
State(state): State<AppState>,
AuthUser(user): AuthUser,
Path(id): Path<Uuid>,
) -> ApiResult<Json<serde_json::Value>> {
require_admin(&user)?;
let (engine, conn) = utopia_store::datasources::engine_and_conn(&state.pool, id).await?;
let ok = match crate::query_engine::engine_for(&engine, &conn) {
Ok(eng) => eng.test().await.is_ok(),
Err(_) => false,
};
utopia_store::datasources::record_test(&state.pool, id, ok).await?;
Ok(Json(json!({ "ok": ok })))
}
// ---------------------------------------------------------------------------
// 知识库层:挂载/卸载/schema 刷新
// ---------------------------------------------------------------------------
pub async fn mounted(
State(state): State<AppState>,
AuthUser(user): AuthUser,
Path(kb_id): Path<Uuid>,
) -> ApiResult<Json<serde_json::Value>> {
require_kb(&state, &user, kb_id, Role::Viewer).await?;
let mounted = utopia_store::datasources::mounted(&state.pool, kb_id).await?;
Ok(Json(json!({ "data_sources": mounted })))
}
/// 库 admin 能挂哪些(列表给 name/summary,不含凭据)。
///
/// **只列授权给本工作区的(0014)。** 从前这里返回 `datasources::list`——
/// 全部署每一个源,于是任何库的管理员都看得见并挂得上任意生产库。
pub async fn mountable(
State(state): State<AppState>,
AuthUser(user): AuthUser,
Path(kb_id): Path<Uuid>,
) -> ApiResult<Json<serde_json::Value>> {
let kb = require_kb(&state, &user, kb_id, Role::Admin).await?;
let granted =
utopia_store::datasources::granted_to_workspace(&state.pool, kb.workspace_id).await?;
Ok(Json(json!({ "data_sources": granted })))
}
pub async fn mount(
State(state): State<AppState>,
AuthUser(user): AuthUser,
Path((kb_id, ds_id)): Path<(Uuid, Uuid)>,
) -> ApiResult<Json<serde_json::Value>> {
require_kb(&state, &user, kb_id, Role::Admin).await?;
// **列表过滤不是守卫。** 那只挡「看得见」,而这个端点是照着 id 调的——
// 谁都能自己拼一个 uuid 打过来。授权在这里再查一次
if !utopia_store::datasources::is_granted(&state.pool, kb_id, ds_id).await? {
return Err(AppError::invalid(
"source_not_granted",
"This data source is not available to this workspace",
)
.into());
}
utopia_store::datasources::mount(&state.pool, kb_id, ds_id).await?;
// 挂载即摄取 schema:Chat 写 SQL 前能检索到表结构
//
// **这一步失败不能回报成挂载失败。** 上面那行已经写进 kb_data_sources 了,
// 源是真挂着的;从前这里 `?` 出去回 500,人以为没挂上,实际挂上了——
// 而问数看不见它有哪些表。改成:照实说挂载成了,schema 没成,并且
// 报进告警中心,因为那之后就是一个静默的缺失状态(0009)
match sync_schema_doc(&state, kb_id, ds_id).await {
Ok(synced) => Ok(Json(json!({ "ok": true, "schema_tables": synced }))),
Err(e) => {
let name = source_name(&state, ds_id).await;
crate::alerting::observe_schema_sync_failure(&state, kb_id, ds_id, &name, &e).await;
Ok(Json(json!({
"ok": true,
"schema_tables": 0,
"schema_error": e.to_string(),
})))
}
}
}
/// 告警要留住名字:源被删之后 `subject_id` 就解析不出名字了。查不到时给占位符
/// 而不是让告警本身失败——**报警路径上的失败不该淹掉它要报的那件事**。
async fn source_name(state: &AppState, ds_id: Uuid) -> String {
utopia_store::datasources::list(&state.pool)
.await
.ok()
.and_then(|all| all.into_iter().find(|d| d.id == ds_id).map(|d| d.name))
.unwrap_or_else(|| ds_id.to_string())
}
pub async fn unmount(
State(state): State<AppState>,
AuthUser(user): AuthUser,
Path((kb_id, ds_id)): Path<(Uuid, Uuid)>,
) -> ApiResult<Json<serde_json::Value>> {
require_kb(&state, &user, kb_id, Role::Admin).await?;
utopia_store::datasources::unmount(&state.pool, kb_id, ds_id).await?;
Ok(Json(json!({ "ok": true })))
}
/// 手动刷新结构。
///
/// **这里照旧把错误回给调用方**——点按钮的人正看着,而且什么都没半途发生。
/// 但同样报一条告警:留下的后果与挂载失败时一模一样(源挂着、表结构是旧的
/// 或空的),而点按钮的人未必是需要知道这件事的人。
pub async fn sync_schema(
State(state): State<AppState>,
AuthUser(user): AuthUser,
Path((kb_id, ds_id)): Path<(Uuid, Uuid)>,
) -> ApiResult<Json<serde_json::Value>> {
require_kb(&state, &user, kb_id, Role::Admin).await?;
match sync_schema_doc(&state, kb_id, ds_id).await {
Ok(synced) => Ok(Json(json!({ "ok": true, "schema_tables": synced }))),
Err(e) => {
let name = source_name(&state, ds_id).await;
crate::alerting::observe_schema_sync_failure(&state, kb_id, ds_id, &name, &e).await;
// **读不出结构回 422 带引擎原话**;我们自己这边出的错(库、写文档)照旧 500。
// 从前一律 AppError::Other,界面只有「Internal server error」,原因只在服务端
// 日志里——而 Invalid 不打日志,所以这里自己记一行
match e.downcast_ref::<SchemaUnreadable>() {
Some(unreadable) => {
tracing::warn!(%kb_id, %ds_id, error = %unreadable, "数据源结构读取失败");
Err(AppError::invalid_detail(
"schema_sync_failed",
"The data source's schema could not be read",
bounded(&unreadable.to_string(), 600),
)
.into())
}
None => Err(AppError::Other(e).into()),
}
}
}
}
/// Agentic 探索:后台任务读挂载源 schema,提议 指标/维度→字段 映射(低置信入 Review)。
pub async fn explore(
State(state): State<AppState>,
AuthUser(user): AuthUser,
Path(kb_id): Path<Uuid>,
) -> ApiResult<Json<serde_json::Value>> {
require_kb(&state, &user, kb_id, Role::Admin).await?;
if utopia_store::datasources::mounted(&state.pool, kb_id)
.await?
.is_empty()
{
return Err(AppError::invalid("no_data_sources", "No data sources mounted").into());
}
utopia_store::jobs::enqueue(&state.pool, "explore_mappings", json!({ "kb_id": kb_id })).await?;
Ok(Json(json!({ "ok": true })))
}
/// 拉 information_schema 生成 markdown,走三路判定摄入(同 key 原地更新)。
/// 文档挂在 per-KB 的 "Data schemas" folder 来源下。
/// 引擎读不出库表结构(连接、权限、catalog 不存在……),与我们自己这边出错分开:
/// 前者是连接串或权限的事,回给点按钮的人;后者照旧是 500
#[derive(Debug)]
struct SchemaUnreadable(String);
impl std::fmt::Display for SchemaUnreadable {
fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
f.write_str(&self.0)
}
}
impl std::error::Error for SchemaUnreadable {}
/// 截到 `max` 个字符以内(按字符边界),引擎的报错可能带整段堆栈
fn bounded(text: &str, max: usize) -> String {
match text.char_indices().nth(max) {
Some((cut, _)) => format!("{}…", &text[..cut]),
None => text.to_string(),
}
}
async fn sync_schema_doc(state: &AppState, kb_id: Uuid, ds_id: Uuid) -> anyhow::Result<usize> {
const MAX_TABLES: usize = 200;
let name = utopia_store::datasources::list(&state.pool)
.await?
.into_iter()
.find(|d| d.id == ds_id)
.map(|d| d.name)
.ok_or_else(|| anyhow::anyhow!("Data source not found"))?;
let (engine, conn) = utopia_store::datasources::engine_and_conn(&state.pool, ds_id).await?;
let cols = crate::query_engine::engine_for(&engine, &conn)?
.fetch_schema()
.await
.map_err(|e| anyhow::Error::new(SchemaUnreadable(e.to_string())))?;
let mut md = format!(
"# Data source: {name}\n\nEngine: {engine}. Tables and columns available for SQL queries against this source; write SQL in this engine's dialect.\n"
);
let mut current = String::new();
let mut tables = 0usize;
for c in &cols {
let key = format!("{}.{}", c.schema, c.table);
if key != current {
if tables >= MAX_TABLES {
md.push_str("\n(further tables omitted)\n");
break;
}
current = key.clone();
tables += 1;
md.push_str(&format!("\n## {key}\n"));
}
md.push_str(&format!(
"- {} ({}){}\n",
c.column,
c.data_type,
c.comment
.as_deref()
.map(|x| format!(" — {x}"))
.unwrap_or_default()
));
}
// per-KB "Data schemas" 容器来源(folder:纯容器语义)。
//
// **这份文档只做检索语料,不进抽取**(0035 决定 7)。它跟别的文档一样进抽取的
// 时候,抽取器把每个列名都当成了实体——宽表语料上四十个概念实体里二十八个是
// 列名(#553)。「不抽取」记在来源的 config 上,流水线读它
let folder = match sqlx::query_as::<_, (Uuid,)>(
"SELECT id FROM sources WHERE kb_id = $1 AND kind = 'folder' AND name = 'Data schemas'",
)
.bind(kb_id)
.fetch_optional(&state.pool)
.await?
{
Some((id,)) => id,
None => {
utopia_store::sources::create(
&state.pool,
kb_id,
"folder",
"Data schemas",
&serde_json::json!({ "extract": false }),
Some("database"),
None,
None,
)
.await?
.id
}
};
crate::ingest_sources::ingest_item(
state,
kb_id,
folder,
&format!("datasource:{ds_id}:schema"),
&format!("{name}-schema.md"),
"text/markdown",
md.as_bytes(),
None,
)
.await?;
state.emit_source(kb_id);
Ok(tables)
}
// ---------------------------------------------------------------------------
// 系统层:授权(0014)
//
// 授权与挂载是两层,各有各的主人:
// 授权 = 系统管理员说「这个源可以给哪些工作区用」 ← 这里
// 挂载 = KB 管理员说「我这个库挂哪几个」 ← 上面那组
// 两层都是多对多。挂载只能在授权过的集合里挑。
// ---------------------------------------------------------------------------
/// 这个源授权给了哪些工作区。
pub async fn grants(
State(state): State<AppState>,
AuthUser(user): AuthUser,
Path(id): Path<Uuid>,
) -> ApiResult<Json<serde_json::Value>> {
require_admin(&user)?;
let rows = utopia_store::datasources::grants_for_source(&state.pool, id).await?;
let workspaces: Vec<serde_json::Value> = rows
.into_iter()
.map(|(id, name)| json!({ "id": id, "name": name }))
.collect();
Ok(Json(json!({ "workspaces": workspaces })))
}
pub async fn grant(
State(state): State<AppState>,
AuthUser(user): AuthUser,
Path((id, workspace_id)): Path<(Uuid, Uuid)>,
) -> ApiResult<Json<serde_json::Value>> {
require_admin(&user)?;
utopia_store::datasources::grant(&state.pool, id, workspace_id, user.id).await?;
let _ = utopia_store::audit::record(
&state.pool,
None,
user.id,
"data_source.granted",
"data_source",
Some(id),
json!({ "workspace_id": workspace_id }),
)
.await;
Ok(Json(json!({ "ok": true })))
}
/// 收回授权。**连同该工作区里已挂上的一起卸掉**——只删授权行的话,
/// 问数读的还是 `kb_data_sources`,撤销就不生效。返回卸掉了几个,
/// 好让界面说得出「顺带卸了 3 个库」而不是悄悄断人家的连接。
pub async fn revoke(
State(state): State<AppState>,
AuthUser(user): AuthUser,
Path((id, workspace_id)): Path<(Uuid, Uuid)>,
) -> ApiResult<Json<serde_json::Value>> {
require_admin(&user)?;
let unmounted = utopia_store::datasources::revoke(&state.pool, id, workspace_id).await?;
let _ = utopia_store::audit::record(
&state.pool,
None,
user.id,
"data_source.revoked",
"data_source",
Some(id),
json!({ "workspace_id": workspace_id, "unmounted": unmounted }),
)
.await;
Ok(Json(json!({ "ok": true, "unmounted": unmounted })))
}