World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Credentials are sealed before they reach the database (#284)
WaylandYangb9ba34b934e3
diff --git a/.env.example b/.env.exampleindex 7d2c5304..77f5ad10 100644--- a/.env.example+++ b/.env.example@@ -4,6 +4,9 @@ UTOPIA_BIND_ADDR=0.0.0.0:1516# JWT 签名密钥。留空即可——首次启动会自动生成一条存进数据库。# 只有轮换密钥、或要让多个实例显式对齐时才需要填。# UTOPIA_JWT_SECRET=+# 凭据封印钥匙(LLM API Key、连接串、来源 token 落库前用它加密;32 字节,64 位十六进制或 base64)。+# 留空即可——首次启动会生成到数据目录的 secret.key。钥匙不进库;备份数据目录时一起带走。+# UTOPIA_SECRET_KEY=# 前端构建产物目录(存在则由服务端托管 SPA)UTOPIA_WEB_DIST=web/dist# 数据目录:原始文件(files/)与全文索引(index/)diff --git a/Cargo.lock b/Cargo.lockindex d17a6e9c..e46fdc39 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -23,6 +23,16 @@ dependencies = ["pom",]+[[package]]+name = "aead"+version = "0.5.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0"+dependencies = [+ "crypto-common 0.1.7",+ "generic-array",+]+[[package]]name = "aes"version = "0.8.4"@@ -34,6 +44,20 @@ dependencies = ["cpufeatures 0.2.17",]+[[package]]+name = "aes-gcm"+version = "0.10.3"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "831010a0f742e1209b3bcea8fab6a8e149051ba6099432c8cb2cc117dec3ead1"+dependencies = [+ "aead",+ "aes",+ "cipher",+ "ctr",+ "ghash",+ "subtle",+]+[[package]]name = "ahash"version = "0.8.12"@@ -821,6 +845,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a"dependencies = ["generic-array",+ "rand_core 0.6.4","typenum",]@@ -877,6 +902,15 @@ dependencies = ["memchr",]+[[package]]+name = "ctr"+version = "0.9.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "0369ee1ad671834580515889b80f2ea915f23b8be8d0daa4bbaf2ac5c7590835"+dependencies = [+ "cipher",+]+[[package]]name = "curve25519-dalek"version = "4.1.3"@@ -1546,6 +1580,16 @@ dependencies = ["wasm-bindgen",]+[[package]]+name = "ghash"+version = "0.5.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "f0d8a4362ccb29cb0b265253fb0a2728f592895ee6854fd9bc13f2ffda266ff1"+dependencies = [+ "opaque-debug",+ "polyval",+]+[[package]]name = "group"version = "0.13.0"@@ -2685,6 +2729,12 @@ version = "0.1.13"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "269bca4c2591a28585d6bf10d9ed0332b7d76900a1b02bec41bdc3a2cdcda107"+[[package]]+name = "opaque-debug"+version = "0.3.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "c08d65885ee38876c4f86fa503fb49d7b507c2b62552df7c70b2fce627e06381"+[[package]]name = "openssl-probe"version = "0.2.1"@@ -2992,6 +3042,18 @@ version = "0.2.3"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "b4596b6d070b27117e987119b4dac604f3c58cfb0b191112e24771b2faeac1a6"+[[package]]+name = "polyval"+version = "0.6.2"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25"+dependencies = [+ "cfg-if",+ "cpufeatures 0.2.17",+ "opaque-debug",+ "universal-hash",+]+[[package]]name = "pom"version = "1.1.0"@@ -4887,6 +4949,16 @@ version = "0.2.2"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254"+[[package]]+name = "universal-hash"+version = "0.5.1"+source = "registry+https://github.com/rust-lang/crates.io-index"+checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea"+dependencies = [+ "crypto-common 0.1.7",+ "subtle",+]+[[package]]name = "untrusted"version = "0.9.0"@@ -4922,7 +4994,9 @@ checksum = "b6c140620e7ffbb22c2dee59cafe6084a59b5ffc27a8859a5f0d494b5d52b6be"name = "utopia-core"version = "0.1.0"dependencies = [+ "aes-gcm","anyhow",+ "base64 0.23.1","chrono","figment","pgvector",diff --git a/Cargo.toml b/Cargo.tomlindex 890444ca..6bff6381 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -35,6 +35,9 @@ anyhow = "1"uuid = { version = "1", features = ["v4", "v7", "serde"] }chrono = { version = "0.4", features = ["serde"] }argon2 = { version = "0.5", features = ["std"] }+# 凭据静态加密(core::secrets):AES-256-GCM,钥匙不进库+aes-gcm = "0.10"+base64 = "0.23"jsonwebtoken = { version = "10", features = ["rust_crypto"] }figment = { version = "0.10", features = ["env"] }dotenvy = "0.15"diff --git a/SECURITY.md b/SECURITY.mdindex f242746f..274cff38 100644--- a/SECURITY.md+++ b/SECURITY.md@@ -7,11 +7,6 @@ but the places the design has not reached yet.## Before you put this on a public network-**Credentials are stored in the clear.** LLM API keys and Ask-the-Data connection strings-are plain text in Postgres (`llm_settings.chat_api_key`, `data_sources.conn_string`). Anyone-who can read the database can read them. Encryption at rest is a 1.0 item; until then, keep-the system and its database inside a trusted network.-**The default database password is `utopia`.** By default the port binds to loopback(`127.0.0.1:1517`), so nothing outside the host can connect. If you change `UTOPIA_DB_BIND`to expose it, change `UTOPIA_DB_PASSWORD` in `.env` first.@@ -24,6 +19,11 @@ source.## What is in place+- **Credentials sealed at rest** — LLM API keys, Ask-the-Data connection strings, source+ tokens and push keys are AES-256-GCM encrypted before they reach Postgres. The key never+ enters the database: `UTOPIA_SECRET_KEY`, or `secret.key` generated in the data directory+ on first start. Back the key up together with the data directory — without it those values+ cannot be read. Rows written by earlier versions are sealed on the next start.- **JWT signing key generated on first start** — 32 bytes from a CSPRNG, stored in thedatabase. No deployment shares a default key.- **`Secure` on session cookies behind TLS** — decided from `X-Forwarded-Proto`, so localdiff --git a/SECURITY.zh-CN.md b/SECURITY.zh-CN.mdindex 3eedc25f..9e573af1 100644--- a/SECURITY.zh-CN.md+++ b/SECURITY.zh-CN.md@@ -6,14 +6,13 @@ Utopia 目前是 v0.1。下面是**已知的、尚未解决的**限制 ——## 部署到公网之前-**凭据在数据库里是明文的。** LLM API Key 与问数的数据库连接串以明文存在 Postgres 里(`llm_settings.chat_api_key`、`data_sources.conn_string`)。能读到库的人就能拿到它们。静态加密是 1.0 之前的硬化项,在那之前请把这套系统和它的数据库放在可信网络内。-**数据库默认口令是 `utopia`。** 默认配置下端口只绑回环(`127.0.0.1:1517`),外网连不上。如果你改了 `UTOPIA_DB_BIND` 把它暴露出去,先换掉 `.env` 里的 `UTOPIA_DB_PASSWORD`。**数据源的安全上限是它的授权。** 注册数据源是部署级动作,而它携带的连接串会到达每一个被授权的工作区。只把源授权给该看见那个库的工作区,并且在连接串里就用只读的数据库角色 —— 下面那道 SQL 闸是纵深防御,不能替代源头的最小权限。## 已经做了的+- **凭据静态加密** —— LLM API Key、问数连接串、来源配置里的 token 与推送密钥在进 Postgres 之前用 AES-256-GCM 封印。钥匙不进库:`UTOPIA_SECRET_KEY`,或首次启动时生成在数据目录下的 `secret.key`。备份数据目录时把钥匙一起带走——没有它这些值读不出来。旧版本写入的明文行在下一次启动时补封。- **JWT 签名密钥首次启动生成** —— 32 字节 CSPRNG 存入数据库,没有「所有部署共用默认密钥」这回事。- **TLS 后面自动给会话 cookie 打 `Secure`** —— 按 `X-Forwarded-Proto` 判定,本地 HTTP 开发照常。代理不发那个头时用 `UTOPIA_COOKIE_SECURE=true` 强制打开。- **数据库端口只绑回环** —— `127.0.0.1:1517`,应用走 compose 内网连库。diff --git a/crates/utopia-core/Cargo.toml b/crates/utopia-core/Cargo.tomlindex 2d2e8e10..7df0f5cb 100644--- a/crates/utopia-core/Cargo.toml+++ b/crates/utopia-core/Cargo.toml@@ -15,3 +15,5 @@ sqlx.workspace = truefigment.workspace = truepgvector.workspace = truestrum.workspace = true+aes-gcm.workspace = true+base64.workspace = truediff --git a/crates/utopia-core/src/config.rs b/crates/utopia-core/src/config.rsindex c3e225d0..f0d26385 100644--- a/crates/utopia-core/src/config.rs+++ b/crates/utopia-core/src/config.rs@@ -18,6 +18,10 @@ pub struct AppConfig {/// 要求部署者手填一个随机串,现实中的结果是默认值原样上生产。/// 显式给出时优先于库里那条:密钥轮换与多实例显式对齐走这条路。pub jwt_secret: Option<String>,+ /// 凭据封印钥匙(32 字节,64 位十六进制或 base64)。留空则用数据目录下的+ /// `secret.key`,首次启动生成。**钥匙不进库**:库泄漏不等于凭据泄漏,是静态加密+ /// 的全部意义;备份数据目录时把它一起带走,没有它库里的凭据读不出来。+ pub secret_key: Option<String>,/// 前端构建产物目录;存在时由服务端托管 SPA(history fallback)。pub web_dist: String,/// 数据目录:原始文件(files/)与 Tantivy 索引(index/)。@@ -39,6 +43,7 @@ impl Default for AppConfig {migration_url: None,bind_addr: "0.0.0.0:1516".into(),jwt_secret: None,+ secret_key: None,web_dist: "web/dist".into(),data_dir: "data".into(),db_max_connections: None,diff --git a/crates/utopia-core/src/lib.rs b/crates/utopia-core/src/lib.rsindex d55bdcdc..789d21cb 100644--- a/crates/utopia-core/src/lib.rs+++ b/crates/utopia-core/src/lib.rs@@ -3,5 +3,6 @@pub mod config;pub mod error;pub mod models;+pub mod secrets;pub use error::{is_terminal, AppError, AppResult, Terminal};diff --git a/crates/utopia-core/src/secrets.rs b/crates/utopia-core/src/secrets.rsnew file mode 100644index 00000000..2e882e5d--- /dev/null+++ b/crates/utopia-core/src/secrets.rs@@ -0,0 +1,198 @@+//! 凭据静态加密(封印)。+//!+//! 库里存的 LLM API key、问数连接串、来源配置里的 token、api 来源的推送密钥,从前都是+//! 明文——能读到库(或一份备份)的人就拿到了全部外部凭据。现在这些值在落库前用+//! AES-256-GCM 封印,密钥不在库里:环境变量 `UTOPIA_SECRET_KEY`,或数据目录下+//! 首次启动生成的 `secret.key`。威胁模型是「库泄漏 ≠ 凭据泄漏」:一份 pg_dump、一个+//! 只读库账号,拿到的是密文。能同时读到数据目录和库的人(登上了服务器)不在此列——+//! 那是服务器访问控制的事。+//!+//! **格式**:`enc:v1:` + base64(nonce(12) ‖ 密文+tag)。没有前缀的值按明文读(升级前+//! 落库的旧行),启动时 [`crate::secrets`] 的调用方(store 的 backfill)把它们补封。+//! 封印幂等:已封印的值再封一次原样返回,读路径与写路径都不必先判断。+//!+//! **一个进程一把钥匙**:服务启动时 [`init`] 一次;没初始化时 [`seal`] 原样返回、+//! [`open`] 只认明文——单元测试与没有服务的工具走这条。+//!+//! 密钥轮换不在这一版:换钥匙 = 用旧钥匙读出、新钥匙写回,是一次显式的迁移。++use aes_gcm::aead::{Aead, KeyInit, OsRng};+use aes_gcm::{AeadCore, Aes256Gcm, Key, Nonce};+use base64::engine::general_purpose::STANDARD as B64;+use base64::Engine;+use std::sync::OnceLock;++pub const PREFIX: &str = "enc:v1:";+const NONCE_LEN: usize = 12;++static SEALER: OnceLock<Aes256Gcm> = OnceLock::new();++/// 装钥匙。只认第一次;再调返回 false,钥匙不变+pub fn init(key: [u8; 32]) -> bool {+ SEALER+ .set(Aes256Gcm::new(Key::<Aes256Gcm>::from_slice(&key)))+ .is_ok()+}++pub fn is_ready() -> bool {+ SEALER.get().is_some()+}++/// 生一把新钥匙(CSPRNG 32 字节)+pub fn generate_key() -> [u8; 32] {+ Aes256Gcm::generate_key(OsRng).into()+}++/// 钥匙的文本形态:64 位十六进制或 44 位 base64,都收+pub fn parse_key(text: &str) -> Option<[u8; 32]> {+ let t = text.trim();+ let bytes: Vec<u8> = if t.len() == 64 && t.bytes().all(|b| b.is_ascii_hexdigit()) {+ (0..64)+ .step_by(2)+ .map(|i| u8::from_str_radix(&t[i..i + 2], 16).ok())+ .collect::<Option<Vec<u8>>>()?+ } else {+ B64.decode(t).ok()?+ };+ bytes.try_into().ok()+}++pub fn key_to_hex(key: &[u8; 32]) -> String {+ key.iter().map(|b| format!("{b:02x}")).collect()+}++pub fn is_sealed(stored: &str) -> bool {+ stored.starts_with(PREFIX)+}++/// 封印。幂等:已封印的原样返回;没装钥匙时原样返回(明文落库,和从前一样)+pub fn seal(plain: &str) -> String {+ if is_sealed(plain) {+ return plain.to_string();+ }+ let Some(cipher) = SEALER.get() else {+ return plain.to_string();+ };+ let nonce = Aes256Gcm::generate_nonce(&mut OsRng);+ let mut out = cipher+ .encrypt(&nonce, plain.as_bytes())+ .expect("AES-GCM encryption does not fail on in-memory input");+ let mut bytes = nonce.to_vec();+ bytes.append(&mut out);+ format!("{PREFIX}{}", B64.encode(bytes))+}++/// 开封。没有前缀的按明文原样返回(升级前的旧行);有前缀而钥匙不对或被改过,报错——+/// 静默回一段乱码会让下游拿它去调外部接口,错得更远+pub fn open(stored: &str) -> anyhow::Result<String> {+ let Some(body) = stored.strip_prefix(PREFIX) else {+ return Ok(stored.to_string());+ };+ let Some(cipher) = SEALER.get() else {+ anyhow::bail!("a sealed secret was read before the secret key was loaded");+ };+ let bytes = B64+ .decode(body)+ .map_err(|_| anyhow::anyhow!("a sealed secret is not valid base64"))?;+ if bytes.len() < NONCE_LEN {+ anyhow::bail!("a sealed secret is too short");+ }+ let (nonce, ct) = bytes.split_at(NONCE_LEN);+ let plain = cipher+ .decrypt(Nonce::from_slice(nonce), ct)+ .map_err(|_| anyhow::anyhow!("a sealed secret does not open with this key (wrong UTOPIA_SECRET_KEY, or the value was tampered with)"))?;+ String::from_utf8(plain).map_err(|_| anyhow::anyhow!("a sealed secret is not UTF-8"))+}++pub fn seal_opt(plain: Option<&str>) -> Option<String> {+ plain.map(seal)+}++pub fn open_opt(stored: Option<&str>) -> anyhow::Result<Option<String>> {+ stored.map(open).transpose()+}++/// 把 JSON 对象里给定键的字符串值封印(就地)。非字符串值不动+pub fn seal_json_keys(value: &mut serde_json::Value, keys: &[&str]) {+ if let Some(obj) = value.as_object_mut() {+ for key in keys {+ if let Some(serde_json::Value::String(s)) = obj.get(*key) {+ let sealed = seal(s);+ obj.insert((*key).to_string(), serde_json::Value::String(sealed));+ }+ }+ }+}++/// [`seal_json_keys`] 的反向+pub fn open_json_keys(value: &mut serde_json::Value, keys: &[&str]) -> anyhow::Result<()> {+ if let Some(obj) = value.as_object_mut() {+ for key in keys {+ if let Some(serde_json::Value::String(s)) = obj.get(*key) {+ let opened = open(s)?;+ obj.insert((*key).to_string(), serde_json::Value::String(opened));+ }+ }+ }+ Ok(())+}++#[cfg(test)]+mod tests {+ use super::*;++ fn ready() {+ // 测试二进制里只装一次;后面的测试拿同一把+ let _ = init([7u8; 32]);+ }++ #[test]+ fn a_secret_round_trips_and_never_repeats() {+ ready();+ let a = seal("sk-live-123");+ let b = seal("sk-live-123");+ assert!(is_sealed(&a));+ assert_ne!(a, b, "a fresh nonce every time");+ assert_eq!(open(&a).unwrap(), "sk-live-123");+ assert_eq!(open(&b).unwrap(), "sk-live-123");+ }++ #[test]+ fn sealing_is_idempotent_and_plaintext_passes_through() {+ ready();+ let once = seal("token");+ assert_eq!(seal(&once), once, "already sealed stays as it is");+ assert_eq!(open("legacy-plain").unwrap(), "legacy-plain");+ }++ #[test]+ fn a_tampered_value_does_not_open() {+ ready();+ let sealed = seal("secret");+ let mut broken = sealed.clone();+ broken.pop();+ broken.push(if sealed.ends_with('A') { 'B' } else { 'A' });+ assert!(open(&broken).is_err());+ assert!(open("enc:v1:not-base64!").is_err());+ }++ #[test]+ fn json_keys_are_sealed_in_place() {+ ready();+ let mut cfg = serde_json::json!({ "token": "t0k", "url": "https://x", "n": 3 });+ seal_json_keys(&mut cfg, &["token", "url_missing", "n"]);+ assert!(is_sealed(cfg["token"].as_str().unwrap()));+ assert_eq!(cfg["url"], "https://x");+ assert_eq!(cfg["n"], 3);+ open_json_keys(&mut cfg, &["token"]).unwrap();+ assert_eq!(cfg["token"], "t0k");+ }++ #[test]+ fn keys_parse_from_hex_and_base64() {+ let key = generate_key();+ assert_eq!(parse_key(&key_to_hex(&key)), Some(key));+ assert_eq!(parse_key(&B64.encode(key)), Some(key));+ assert_eq!(parse_key("too-short"), None);+ }+}diff --git a/crates/utopia-server/Cargo.toml b/crates/utopia-server/Cargo.tomlindex 628e03db..7f6e3524 100644--- a/crates/utopia-server/Cargo.toml+++ b/crates/utopia-server/Cargo.toml@@ -43,7 +43,7 @@ object_store = { version = "0.14.1", features = ["aws", "azure", "gcp"] }quick-xml.workspace = trueurl = "2.5.8"percent-encoding = "2.3.2"-base64 = "0.23.1"+base64.workspace = true[dev-dependencies]wiremock = "0.6.5"diff --git a/crates/utopia-server/src/main.rs b/crates/utopia-server/src/main.rsindex 56a1a500..fb1ac961 100644--- a/crates/utopia-server/src/main.rs+++ b/crates/utopia-server/src/main.rs@@ -80,6 +80,22 @@ async fn main() -> anyhow::Result<()> {let pool = utopia_store::db::connect(&cfg.database_url, cfg.db_max_connections).await?;+ // 凭据封印钥匙:环境变量优先,否则数据目录下的 secret.key(首次启动生成)。+ // **钥匙不进库**——库泄漏不等于凭据泄漏,是这一层的全部意义。空串按未设置处理,+ // 理由同下面的 JWT 密钥+ let secret_key = match cfg.secret_key.clone().filter(|s| !s.trim().is_empty()) {+ Some(text) => utopia_core::secrets::parse_key(&text).ok_or_else(|| {+ anyhow::anyhow!("UTOPIA_SECRET_KEY must be 32 bytes, as 64 hex characters or base64")+ })?,+ None => secret_key_file(std::path::Path::new(&cfg.data_dir))?,+ };+ utopia_core::secrets::init(secret_key);+ // 升级前落库的明文凭据在这里补封+ let sealed = utopia_store::sealing::backfill(&pool).await?;+ if sealed > 0 {+ tracing::info!(rows = sealed, "凭据补封完成");+ }+let index_dir = std::path::Path::new(&cfg.data_dir).join("index");let search = Arc::new(SearchIndex::open(&index_dir)?);tracing::info!("全文索引就绪: {}", index_dir.display());@@ -404,3 +420,31 @@ async fn dispatch(st: &state::AppState, job: &utopia_store::jobs::Job) -> anyhowother => anyhow::bail!("未知任务类型: {other}"),}}++/// 数据目录下的封印钥匙:有就读,没有就生成一把写进去(unix 下 0600)。+/// 读出来解析不了就报错停下——静默生成一把新的会让库里已封的凭据全部打不开+fn secret_key_file(data_dir: &std::path::Path) -> anyhow::Result<[u8; 32]> {+ let path = data_dir.join("secret.key");+ if path.exists() {+ let text = std::fs::read_to_string(&path)?;+ return utopia_core::secrets::parse_key(&text).ok_or_else(|| {+ anyhow::anyhow!(+ "{} does not hold a 32-byte key (64 hex characters or base64)",+ path.display()+ )+ });+ }+ std::fs::create_dir_all(data_dir)?;+ let key = utopia_core::secrets::generate_key();+ std::fs::write(&path, utopia_core::secrets::key_to_hex(&key))?;+ #[cfg(unix)]+ {+ use std::os::unix::fs::PermissionsExt;+ std::fs::set_permissions(&path, std::fs::Permissions::from_mode(0o600))?;+ }+ tracing::info!(+ "生成了凭据封印钥匙: {}(备份数据目录时一起带走;UTOPIA_SECRET_KEY 可覆盖)",+ path.display()+ );+ Ok(key)+}diff --git a/crates/utopia-store/src/datasources.rs b/crates/utopia-store/src/datasources.rsindex 6e3ec3a9..11404ed8 100644--- a/crates/utopia-store/src/datasources.rs+++ b/crates/utopia-store/src/datasources.rs@@ -4,7 +4,7 @@use chrono::{DateTime, Utc};use sqlx::PgPool;use utopia_core::models::DataSourceView;-use utopia_core::{AppError, AppResult};+use utopia_core::{secrets, AppError, AppResult};use uuid::Uuid;/// data_sources 的行投影,list 与 mounted 共用。@@ -39,11 +39,16 @@ pub fn conn_summary(conn: &str) -> String {fn row_to_view((id, name, engine, conn, created_at, last_test_at, last_test_ok): DataSourceRow,) -> DataSourceView {+ // 连接串在库里是封印的;开不了(换过钥匙)就说开不了,别把密文当 URL 去解析+ let summary = match secrets::open(&conn) {+ Ok(plain) => conn_summary(&plain),+ Err(_) => "(sealed with another key)".into(),+ };DataSourceView {id,name,engine,- summary: conn_summary(&conn),+ summary,created_at,last_test_at,last_test_ok,@@ -92,7 +97,7 @@ pub async fn create(.bind(id).bind(name.trim()).bind(engine)- .bind(conn_string.trim())+ .bind(secrets::seal(conn_string.trim())).bind(created_by).execute(pool).await?;@@ -117,7 +122,8 @@ pub async fn conn_string(pool: &PgPool, id: Uuid) -> AppResult<String> {.bind(id).fetch_optional(pool).await?;- row.map(|(c,)| c).ok_or(AppError::NotFound)+ let (conn,) = row.ok_or(AppError::NotFound)?;+ secrets::open(&conn).map_err(AppError::Other)}pub async fn record_test(pool: &PgPool, id: Uuid, ok: bool) -> AppResult<()> {@@ -174,7 +180,8 @@ pub async fn engine_and_conn(pool: &PgPool, id: Uuid) -> AppResult<(String, Stri.bind(id).fetch_optional(pool).await?;- row.ok_or(AppError::NotFound)+ let (engine, conn) = row.ok_or(AppError::NotFound)?;+ Ok((engine, secrets::open(&conn).map_err(AppError::Other)?))}/// 这个工作区被授权用哪些源(0014)。diff --git a/crates/utopia-store/src/lib.rs b/crates/utopia-store/src/lib.rsindex cccaeb26..3daa6f6b 100644--- a/crates/utopia-store/src/lib.rs+++ b/crates/utopia-store/src/lib.rs@@ -23,6 +23,7 @@ pub mod pending;pub mod reasoning;pub mod resolution;pub mod review;+pub mod sealing;pub mod settings;pub mod sources;pub mod temporal;diff --git a/crates/utopia-store/src/sealing.rs b/crates/utopia-store/src/sealing.rsnew file mode 100644index 00000000..38bcb643--- /dev/null+++ b/crates/utopia-store/src/sealing.rs@@ -0,0 +1,98 @@+//! 凭据补封:升级前落库的明文凭据,在钥匙装好之后的第一次启动补成密文。+//!+//! 幂等,每次启动都跑:判据是「值没有 `enc:v1:` 前缀」,已封的不动。四处凭据+//! (`llm_settings` 两把 key、`data_sources.conn_string`、`sources.config` 的凭据键、+//! `sources.ingest_token`)在这里各扫一遍——加了新的存放处要同时加到这里,否则那一处+//! 会一直是明文而没有任何地方报警。++use sqlx::PgPool;+use utopia_core::models::SOURCE_SECRET_KEYS;+use utopia_core::{secrets, AppResult};+use uuid::Uuid;++/// 返回补封的行数。没装钥匙时什么都不做+pub async fn backfill(pool: &PgPool) -> AppResult<usize> {+ if !secrets::is_ready() {+ return Ok(0);+ }+ Ok(seal_llm_settings(pool, None).await?+ + seal_data_sources(pool, None).await?+ + seal_sources(pool, None).await?)+}++/// `only` = 只补这一个工作区(测试用;启动时传 None 扫全部)+pub async fn seal_llm_settings(pool: &PgPool, only: Option<Uuid>) -> AppResult<usize> {+ let rows: Vec<(Uuid, Option<String>, Option<String>)> = sqlx::query_as(+ "SELECT workspace_id, chat_api_key, embed_api_key FROM llm_settings+ WHERE ($1::uuid IS NULL OR workspace_id = $1)+ AND ((chat_api_key IS NOT NULL AND chat_api_key NOT LIKE 'enc:v1:%')+ OR (embed_api_key IS NOT NULL AND embed_api_key NOT LIKE 'enc:v1:%'))",+ )+ .bind(only)+ .fetch_all(pool)+ .await?;+ let n = rows.len();+ for (ws, chat, embed) in rows {+ sqlx::query(+ "UPDATE llm_settings SET chat_api_key = $2, embed_api_key = $3 WHERE workspace_id = $1",+ )+ .bind(ws)+ .bind(secrets::seal_opt(chat.as_deref()))+ .bind(secrets::seal_opt(embed.as_deref()))+ .execute(pool)+ .await?;+ }+ Ok(n)+}++pub async fn seal_data_sources(pool: &PgPool, only: Option<Uuid>) -> AppResult<usize> {+ let rows: Vec<(Uuid, String)> = sqlx::query_as(+ "SELECT id, conn_string FROM data_sources+ WHERE ($1::uuid IS NULL OR id = $1) AND conn_string NOT LIKE 'enc:v1:%'",+ )+ .bind(only)+ .fetch_all(pool)+ .await?;+ let n = rows.len();+ for (id, conn) in rows {+ sqlx::query("UPDATE data_sources SET conn_string = $2 WHERE id = $1")+ .bind(id)+ .bind(secrets::seal(&conn))+ .execute(pool)+ .await?;+ }+ Ok(n)+}++/// 来源表小,整张读回来在 Rust 里判:凭据键藏在 JSON 里,SQL 判起来反而绕+pub async fn seal_sources(pool: &PgPool, only: Option<Uuid>) -> AppResult<usize> {+ let rows: Vec<(Uuid, serde_json::Value, Option<String>)> = sqlx::query_as(+ "SELECT id, config, ingest_token FROM sources WHERE ($1::uuid IS NULL OR id = $1)",+ )+ .bind(only)+ .fetch_all(pool)+ .await?;+ let mut n = 0;+ for (id, mut config, token) in rows {+ let plain_key = config.as_object().is_some_and(|o| {+ SOURCE_SECRET_KEYS.iter().any(|k| {+ o.get(*k)+ .and_then(|v| v.as_str())+ .is_some_and(|s| !secrets::is_sealed(s))+ })+ });+ let plain_token = token.as_deref().is_some_and(|t| !secrets::is_sealed(t));+ if !plain_key && !plain_token {+ continue;+ }+ secrets::seal_json_keys(&mut config, SOURCE_SECRET_KEYS);+ sqlx::query("UPDATE sources SET config = $2, ingest_token = $3 WHERE id = $1")+ .bind(id)+ .bind(config)+ .bind(secrets::seal_opt(token.as_deref()))+ .execute(pool)+ .await?;+ n += 1;+ }+ Ok(n)+}diff --git a/crates/utopia-store/src/settings.rs b/crates/utopia-store/src/settings.rsindex 07f9df81..13a268a2 100644--- a/crates/utopia-store/src/settings.rs+++ b/crates/utopia-store/src/settings.rs@@ -1,27 +1,36 @@use sqlx::PgPool;use utopia_core::models::LlmSettings;-use utopia_core::AppResult;+use utopia_core::{secrets, AppError, AppResult};use uuid::Uuid;+/// 出库即开封:两把 API key 在库里是封印的(`utopia_core::secrets`)。+/// 任何返回 `LlmSettings` 的查询都从这里过+fn opened(mut s: LlmSettings) -> AppResult<LlmSettings> {+ s.chat_api_key = secrets::open_opt(s.chat_api_key.as_deref()).map_err(AppError::Other)?;+ s.embed_api_key = secrets::open_opt(s.embed_api_key.as_deref()).map_err(AppError::Other)?;+ Ok(s)+}+pub async fn get(pool: &PgPool, workspace_id: Uuid) -> AppResult<Option<LlmSettings>> {- let row = sqlx::query_as("SELECT * FROM llm_settings WHERE workspace_id = $1")- .bind(workspace_id)- .fetch_optional(pool)- .await?;- Ok(row)+ let row: Option<LlmSettings> =+ sqlx::query_as("SELECT * FROM llm_settings WHERE workspace_id = $1")+ .bind(workspace_id)+ .fetch_optional(pool)+ .await?;+ row.map(opened).transpose()}/// 任取一个配了对话模型的工作区设置。给端点探针用:端点地址是部署共用的,/// 从哪个工作区的配置读到的都是同一个地方,而探针没有"当前工作区"这个上下文。pub async fn any_with_chat(pool: &PgPool) -> AppResult<Option<LlmSettings>> {- let row = sqlx::query_as(+ let row: Option<LlmSettings> = sqlx::query_as("SELECT * FROM llm_settingsWHERE chat_base_url IS NOT NULL AND chat_model IS NOT NULLORDER BY workspace_id LIMIT 1",).fetch_optional(pool).await?;- Ok(row)+ row.map(opened).transpose()}/// upsert;api_key 传 None 表示保留旧值(前端不回传密钥)。@@ -37,7 +46,10 @@ pub async fn upsert(embed_model: Option<&str>,embed_dim: Option<i32>,) -> AppResult<LlmSettings> {- let row = sqlx::query_as(+ // 入库即封印;None 仍是 None(保留旧值)+ let chat_api_key = secrets::seal_opt(chat_api_key);+ let embed_api_key = secrets::seal_opt(embed_api_key);+ let row: LlmSettings = sqlx::query_as("INSERT INTO llm_settings(workspace_id, chat_base_url, chat_api_key, chat_model,embed_base_url, embed_api_key, embed_model, embed_dim, updated_at)@@ -63,5 +75,5 @@ pub async fn upsert(.bind(embed_dim).fetch_one(pool).await?;- Ok(row)+ opened(row)}diff --git a/crates/utopia-store/src/sources.rs b/crates/utopia-store/src/sources.rsindex db00ee6a..901b49ad 100644--- a/crates/utopia-store/src/sources.rs+++ b/crates/utopia-store/src/sources.rs@@ -3,7 +3,7 @@use chrono::{DateTime, Utc};use sqlx::PgPool;use utopia_core::models::{Role, Source, SourceKind, SourceView, SyncRun, SOURCE_SECRET_KEYS};-use utopia_core::{AppError, AppResult};+use utopia_core::{secrets, AppError, AppResult};use uuid::Uuid;/// folder = 纯容器(上传/拖拽入内,无同步语义);url/rss = 拉取型;api = 推送型。@@ -70,12 +70,20 @@ pub async fn list(pool: &PgPool, kb_id: Uuid) -> AppResult<Vec<SourceView>> {Ok(rows)}+/// 出库即开封:配置里的凭据键与推送密钥在库里是封印的(`utopia_core::secrets`)。+/// 任何返回 `Source` 的查询都从这里过——`list` 不用,它在 SQL 里就把凭据键剔了+fn opened(mut s: Source) -> AppResult<Source> {+ secrets::open_json_keys(&mut s.config, SOURCE_SECRET_KEYS).map_err(AppError::Other)?;+ s.ingest_token = secrets::open_opt(s.ingest_token.as_deref()).map_err(AppError::Other)?;+ Ok(s)+}+pub async fn get(pool: &PgPool, id: Uuid) -> AppResult<Source> {- sqlx::query_as("SELECT * FROM sources WHERE id = $1")+ let row: Option<Source> = sqlx::query_as("SELECT * FROM sources WHERE id = $1").bind(id).fetch_optional(pool)- .await?- .ok_or(AppError::NotFound)+ .await?;+ opened(row.ok_or(AppError::NotFound)?)}#[allow(clippy::too_many_arguments)]@@ -109,12 +117,14 @@ pub async fn create(sync_interval_minutes};// serde 缺省的 Value::Null 会以 jsonb null 落库,前端读 config.x 直接炸——规范化为空对象- let config = if config.is_null() {+ let mut config = if config.is_null() {serde_json::json!({})} else {config.clone()};- let source = sqlx::query_as(+ // 入库即封印+ secrets::seal_json_keys(&mut config, SOURCE_SECRET_KEYS);+ let source: Source = sqlx::query_as("INSERT INTO sources (id, kb_id, kind, name, config, icon, sync_interval_minutes, sync_cron)VALUES ($1, $2, $3, $4, $5, $6, $7, $8) RETURNING *",)@@ -128,14 +138,14 @@ pub async fn create(.bind(cron_norm).fetch_one(pool).await?;- Ok(source)+ opened(source)}/// 设置 api 来源的推送密钥(创建 / 轮换时)。pub async fn set_ingest_token(pool: &PgPool, source_id: Uuid, token: &str) -> AppResult<()> {let res = sqlx::query("UPDATE sources SET ingest_token = $2 WHERE id = $1").bind(source_id)- .bind(token)+ .bind(secrets::seal(token)).execute(pool).await?;if res.rows_affected() == 0 {@@ -162,7 +172,11 @@ pub async fn update(}None => None,};- let source = sqlx::query_as(+ let config = config.cloned().map(|mut c| {+ secrets::seal_json_keys(&mut c, SOURCE_SECRET_KEYS);+ c+ });+ let source: Source = sqlx::query_as("UPDATE sources SETname = COALESCE($2, name),config = COALESCE($3, config),@@ -181,7 +195,7 @@ pub async fn update(.fetch_optional(pool).await?.ok_or(AppError::NotFound)?;- Ok(source)+ opened(source)}/// 删除来源;其文档保留(source_id 置 NULL,落回 Uploads 组)。@@ -211,6 +225,10 @@ pub async fn due_sources(pool: &PgPool) -> AppResult<Vec<Source>> {.await?;let now = chrono::Utc::now();+ let rows = rows+ .into_iter()+ .map(opened)+ .collect::<AppResult<Vec<_>>>()?;Ok(rows.into_iter().filter(|s| match &s.sync_cron {diff --git a/crates/utopia-store/tests/a_secret_is_sealed_at_rest.rs b/crates/utopia-store/tests/a_secret_is_sealed_at_rest.rsnew file mode 100644index 00000000..279e8437--- /dev/null+++ b/crates/utopia-store/tests/a_secret_is_sealed_at_rest.rs@@ -0,0 +1,260 @@+//! 凭据静态加密:库里是密文,店里拿到的是明文。+//!+//! 四处凭据——`llm_settings` 的两把 key、`data_sources.conn_string`、来源配置里的+//! 凭据键、api 来源的推送密钥——各守两件事:**写进去是封印的**(裸 SQL 读出来带+//! `enc:v1:` 前缀),**读出来是开封的**(store 函数返回原文)。外加两件:+//! 升级前落库的明文行照常读得出来,补封(`sealing::seal_*`,启动时 `backfill` 扫全部)之后它也封上了;+//! 只传 None 的 upsert 保留旧值、不把它变回明文。+//!+//! 没有 `UTOPIA_DATABASE_URL` 时跳过而不是失败。自建自拆,绝不碰已有的库。++use sqlx::PgPool;+use utopia_core::secrets;+use utopia_store::{datasources, sealing, settings, sources};+use uuid::Uuid;++struct Fx {+ org: Uuid,+ ws: Uuid,+ kb: Uuid,+ user: Uuid,+}++async fn seed(pool: &PgPool) -> anyhow::Result<Fx> {+ let (org, ws, kb, user) = (+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ Uuid::now_v7(),+ );+ sqlx::query("INSERT INTO organizations (id, name) VALUES ($1, 'seal-test')")+ .bind(org)+ .execute(pool)+ .await?;+ sqlx::query("INSERT INTO workspaces (id, org_id, name) VALUES ($1, $2, 'seal-test')")+ .bind(ws)+ .bind(org)+ .execute(pool)+ .await?;+ sqlx::query(+ "INSERT INTO users (id, org_id, email, display_name, password_hash)+ VALUES ($1, $2, $1 || '@seal.test', 's', 'x')",+ )+ .bind(user)+ .bind(org)+ .execute(pool)+ .await?;+ sqlx::query(+ "INSERT INTO knowledge_bases (id, workspace_id, name) VALUES ($1, $2, 'seal-test')",+ )+ .bind(kb)+ .bind(ws)+ .execute(pool)+ .await?;+ Ok(Fx { org, ws, kb, user })+}++async fn raw(pool: &PgPool, sql: &str, id: Uuid) -> anyhow::Result<Option<String>> {+ Ok(sqlx::query_scalar(sql).bind(id).fetch_one(pool).await?)+}++#[tokio::test]+async fn a_secret_is_sealed_at_rest() -> anyhow::Result<()> {+ let Some(url) = utopia_store::test_db::url() else {+ return Ok(());+ };+ // 这个测试二进制自己装一把钥匙;服务里由 main 装+ secrets::init(secrets::generate_key());+ let pool = PgPool::connect(&url).await?;+ let f = seed(&pool).await?;+ let mut ds: Option<Uuid> = None;++ let run = async {+ // 1. LLM key:写进去是密文,读出来是原文;只传 None 保留旧值+ let s = settings::upsert(+ &pool,+ f.ws,+ Some("https://llm.example/v1"),+ Some("sk-chat-1"),+ Some("m"),+ None,+ Some("sk-embed-1"),+ None,+ None,+ )+ .await?;+ assert_eq!(s.chat_api_key.as_deref(), Some("sk-chat-1"));+ let stored = raw(+ &pool,+ "SELECT chat_api_key FROM llm_settings WHERE workspace_id = $1",+ f.ws,+ )+ .await?+ .unwrap();+ assert!(secrets::is_sealed(&stored), "sealed in the database");+ assert_ne!(stored, "sk-chat-1");+ let got = settings::get(&pool, f.ws).await?.unwrap();+ assert_eq!(got.chat_api_key.as_deref(), Some("sk-chat-1"));+ assert_eq!(got.embed_api_key.as_deref(), Some("sk-embed-1"));+ settings::upsert(+ &pool,+ f.ws,+ Some("https://llm.example/v1"),+ None,+ Some("m2"),+ None,+ None,+ None,+ None,+ )+ .await?;+ let got = settings::get(&pool, f.ws).await?.unwrap();+ assert_eq!(+ got.chat_api_key.as_deref(),+ Some("sk-chat-1"),+ "None keeps the old key"+ );++ // 2. 连接串(名字全局唯一,带上随机后缀别和残留撞上)+ let ds_name = format!("warehouse-{}", Uuid::now_v7().simple());+ let id = datasources::create(+ &pool,+ &ds_name,+ "postgres",+ "postgres://analyst:[email protected]:5432/sales",+ f.user,+ )+ .await?;+ ds = Some(id);+ let stored = raw(+ &pool,+ "SELECT conn_string FROM data_sources WHERE id = $1",+ id,+ )+ .await?+ .unwrap();+ assert!(secrets::is_sealed(&stored));+ assert!(!stored.contains("pw-secret"));+ let (engine, conn) = datasources::engine_and_conn(&pool, id).await?;+ assert_eq!(engine, "postgres");+ assert!(conn.contains("pw-secret"));+ let view = datasources::list(&pool)+ .await?+ .into_iter()+ .find(|v| v.id == id)+ .unwrap();+ assert_eq!(+ view.summary, "db.example:5432/sales",+ "the summary opens first"+ );++ // 3. 来源配置里的凭据键 + 推送密钥+ let src = sources::create(+ &pool,+ f.kb,+ "url",+ "feed",+ &serde_json::json!({ "urls": ["https://x.example/a"], "auth_header": "Bearer t0k" }),+ None,+ None,+ None,+ )+ .await?;+ assert_eq!(src.config["auth_header"], "Bearer t0k");+ let stored = raw(+ &pool,+ "SELECT config->>'auth_header' FROM sources WHERE id = $1",+ src.id,+ )+ .await?+ .unwrap();+ assert!(secrets::is_sealed(&stored));+ assert_eq!(+ sources::get(&pool, src.id).await?.config["auth_header"],+ "Bearer t0k"+ );+ sources::set_ingest_token(&pool, src.id, "utp_push_1").await?;+ let stored = raw(+ &pool,+ "SELECT ingest_token FROM sources WHERE id = $1",+ src.id,+ )+ .await?+ .unwrap();+ assert!(secrets::is_sealed(&stored));+ assert_eq!(+ sources::get(&pool, src.id).await?.ingest_token.as_deref(),+ Some("utp_push_1")+ );++ // 4. 旧行:明文照常读,补封之后是密文、读出来还是原文+ sqlx::query(+ "UPDATE llm_settings SET chat_api_key = 'legacy-plain' WHERE workspace_id = $1",+ )+ .bind(f.ws)+ .execute(&pool)+ .await?;+ sqlx::query("UPDATE sources SET ingest_token = 'utp_legacy' WHERE id = $1")+ .bind(src.id)+ .execute(&pool)+ .await?;+ assert_eq!(+ settings::get(&pool, f.ws)+ .await?+ .unwrap()+ .chat_api_key+ .as_deref(),+ Some("legacy-plain")+ );+ // **只补自己的行**:全局 backfill 会拿这把一次性钥匙把共用测试库里别人的+ // 凭据也封上——那些行从此谁也打不开+ let n = sealing::seal_llm_settings(&pool, Some(f.ws)).await?+ + sealing::seal_sources(&pool, Some(src.id)).await?;+ assert_eq!(n, 2, "both legacy rows were sealed");+ let stored = raw(+ &pool,+ "SELECT chat_api_key FROM llm_settings WHERE workspace_id = $1",+ f.ws,+ )+ .await?+ .unwrap();+ assert!(secrets::is_sealed(&stored));+ assert_eq!(+ settings::get(&pool, f.ws)+ .await?+ .unwrap()+ .chat_api_key+ .as_deref(),+ Some("legacy-plain")+ );+ assert_eq!(+ sources::get(&pool, src.id).await?.ingest_token.as_deref(),+ Some("utp_legacy")+ );+ assert_eq!(+ sealing::seal_llm_settings(&pool, Some(f.ws)).await?+ + sealing::seal_sources(&pool, Some(src.id)).await?+ + sealing::seal_data_sources(&pool, Some(id)).await?,+ 0,+ "idempotent"+ );+ Ok::<_, anyhow::Error>(())+ }+ .await;++ if let Some(id) = ds {+ let _ = sqlx::query("DELETE FROM data_sources WHERE id = $1")+ .bind(id)+ .execute(&pool)+ .await;+ }+ let _ = sqlx::query("DELETE FROM knowledge_bases WHERE id = $1")+ .bind(f.kb)+ .execute(&pool)+ .await;+ let _ = sqlx::query("DELETE FROM organizations WHERE id = $1")+ .bind(f.org)+ .execute(&pool)+ .await;+ run+}diff --git a/docker-compose.yml b/docker-compose.ymlindex 828a3f99..119ee288 100644--- a/docker-compose.yml+++ b/docker-compose.yml@@ -49,6 +49,7 @@ services:# 留空:首次启动自动生成一条存进 deployment_settings。这里若给缺省值,# 自动生成就永远走不到,所有部署又会共用同一个人人皆知的密钥。UTOPIA_JWT_SECRET: ${UTOPIA_JWT_SECRET:-}+ UTOPIA_SECRET_KEY: ${UTOPIA_SECRET_KEY:-}UTOPIA_WEB_DIST: /app/web-distUTOPIA_DATA_DIR: /app/data# 连接池上限。缺省 32,与 worker 并发的缺省对齐;池子小于并发时症状是