World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Schema columns carry primary and foreign keys (#671)
Royce61ece8e3a787
diff --git a/crates/utopia-server/src/mappings.rs b/crates/utopia-server/src/mappings.rsindex 2e7163c8..da8c91c7 100644--- a/crates/utopia-server/src/mappings.rs+++ b/crates/utopia-server/src/mappings.rs@@ -16,6 +16,42 @@ use uuid::Uuid;const MAX_SCHEMA_CHARS: usize = 12_000;+/// schema 文本开头那一句图例:列行里的方括号是什么+const COLUMN_MARKERS: &str =+ "Column markers: [PK] single-column primary key; [FK→schema.table] foreign key to that table.+";++/// schema 文本里的一列:` 名字 类型 [PK, FK→schema.table] -- 注释`。+///+/// 键标记放在类型后面、注释前面,用方括号:`-- ` 已经是注释的分隔,写成 `-- PK`+/// 就与一条内容恰好是「PK」的注释分不开。主键又是外键(一对一的扩展表)两个都标。+/// 可空不写进来:宽表上几乎每列都是 NOT NULL,每列多十几个字符会让+/// `MAX_SCHEMA_CHARS` 装下的列明显变少,而两个提示词都用不到它(#502)+fn column_line(c: &crate::query_engine::SchemaColumn) -> String {+ let mut marks = Vec::new();+ if c.is_primary_key {+ marks.push("PK".to_string());+ }+ if let Some(target) = &c.references_table {+ marks.push(format!("FK→{target}"));+ }+ let marks = if marks.is_empty() {+ String::new()+ } else {+ format!(" [{}]", marks.join(", "))+ };+ let comment = c+ .comment+ .as_deref()+ .map(|x| format!(" -- {x}"))+ .unwrap_or_default();+ format!(+ " {} {}{marks}{comment}+",+ c.column, c.data_type+ )+}+/// 一轮允许提几条口径。////// **上限跟着 schema 的大小走。** 写死的 12 对一个三张表的小库绰绰有余,@@ -191,7 +227,7 @@ async fn explore(state: &AppState, kb_id: Uuid, run: Uuid) -> anyhow::Result<()>// **上限是跨源的一个总数。** 从前那个 `break` 只跳出当前源的列循环,// 下一个源接着往同一个字符串里追加——`MAX_SCHEMA_CHARS` 读起来像个上限,// 实际上是「每个源各自超一次」的下限。- let mut schema_txt = String::new();+ let mut schema_txt = String::from(COLUMN_MARKERS);let mut tables_scanned = 0i32;let mut columns_scanned = 0i32;let mut truncated = false;@@ -213,12 +249,7 @@ async fn explore(state: &AppState, kb_id: Uuid, run: Uuid) -> anyhow::Result<()>schema_txt.push_str(&format!("table {key}:\n"));}columns_scanned += 1;- schema_txt.push_str(&format!(- " {} {}{}\n",- c.column,- c.data_type,- c.comment.map(|x| format!(" -- {x}")).unwrap_or_default()- ));+ schema_txt.push_str(&column_line(&c));if schema_txt.len() > MAX_SCHEMA_CHARS {schema_txt.push_str("(truncated)\n");truncated = true;@@ -503,7 +534,51 @@ async fn explore(state: &AppState, kb_id: Uuid, run: Uuid) -> anyhow::Result<()>#[cfg(test)]mod tests {- use super::{parse_description, proposal_cap};+ use super::{column_line, parse_description, proposal_cap};+ use crate::query_engine::SchemaColumn;++ fn column(pk: bool, fk: Option<&str>, comment: Option<&str>) -> SchemaColumn {+ SchemaColumn {+ schema: "a".into(),+ table: "t".into(),+ column: "id".into(),+ data_type: "integer".into(),+ comment: comment.map(str::to_string),+ is_primary_key: pk,+ references_table: fk.map(str::to_string),+ }+ }++ #[test]+ fn a_column_line_marks_its_keys_before_the_comment() {+ assert_eq!(+ column_line(&column(false, None, None)),+ " id integer+"+ );+ assert_eq!(+ column_line(&column(true, None, Some("Order id"))),+ " id integer [PK] -- Order id+"+ );+ assert_eq!(+ column_line(&column(false, Some("a.orders"), None)),+ " id integer [FK→a.orders]+"+ );+ // 一对一的扩展表:主键同时是外键,两个都标+ assert_eq!(+ column_line(&column(true, Some("a.p1"), None)),+ " id integer [PK, FK→a.p1]+"+ );+ // 注释里写着 PK 的普通列,与标记分得开+ assert_eq!(+ column_line(&column(false, None, Some("PK"))),+ " id integer -- PK+"+ );+ }#[test]fn a_description_is_kept_only_when_it_says_something() {diff --git a/crates/utopia-server/src/query_engine/mod.rs b/crates/utopia-server/src/query_engine/mod.rsindex 7e5cb6ce..bf933f9c 100644--- a/crates/utopia-server/src/query_engine/mod.rs+++ b/crates/utopia-server/src/query_engine/mod.rs@@ -54,6 +54,18 @@ pub struct SchemaColumn {pub column: String,pub data_type: String,pub comment: Option<String>,+ /// 该列本身是这张表的主键(单列主键;组合主键里这一位仍为 false)。+ /// 目前只有 Postgres 从 catalog 读出来;MySQL / Trino / Snowflake / Databricks+ /// 恒为 false,意思是「不知道」,不是「不是键」——各自的读法是 #502 的后几刀。+ /// 探索提示词靠它把 ID 与量分开,宽表上一个八十列的 schema 没有这个+ /// 几乎认不出哪一列是键(#502)+ pub is_primary_key: bool,+ /// 该列本身是某张表的单列外键时,它指向的表(`schema.table`);组合外键的成员+ /// 与读不出外键的引擎都是 None。+ ///+ /// 可空没有放进来:两个探索提示词都不用它,宽表上每列多一个 NOT NULL 只是挤掉+ /// 列(#502 真用到时再加)+ pub references_table: Option<String>,}#[async_trait::async_trait]diff --git a/crates/utopia-server/src/query_engine/mysql.rs b/crates/utopia-server/src/query_engine/mysql.rsindex 8eb386f4..a6573a57 100644--- a/crates/utopia-server/src/query_engine/mysql.rs+++ b/crates/utopia-server/src/query_engine/mysql.rs@@ -232,6 +232,12 @@ impl QueryEngine for MysqlEngine {data_type,// 没有注释时这一列是空串而不是 NULL,照抄会让每张表都挂一个空注释comment: comment.filter(|c| !c.trim().is_empty()),+ // MySQL 的 PK / FK 要走 information_schema.statistics 与+ // key_column_usage 才能拿到(PK 在 INDEX_NAME='PRIMARY' 上、FK 在+ // REFERENCED_TABLE_NAME 非空上)。这一刀先给 false,下一刀接 #502+ // 的 MySQL cut 再补+ is_primary_key: false,+ references_table: None,}).collect())}diff --git a/crates/utopia-server/src/query_engine/postgres.rs b/crates/utopia-server/src/query_engine/postgres.rsindex 5bba3a4e..7ea356cd 100644--- a/crates/utopia-server/src/query_engine/postgres.rs+++ b/crates/utopia-server/src/query_engine/postgres.rs@@ -37,7 +37,7 @@ impl QueryEngine for PostgresEngine {async fn fetch_schema(&self) -> anyhow::Result<Vec<SchemaColumn>> {let pool = self.pool().await?;- let rows: Vec<(String, String, String, String, Option<String>)> = sqlx::query_as(+ let cols: Vec<(String, String, String, String, Option<String>)> = sqlx::query_as("SELECT c.table_schema, c.table_name, c.column_name,c.data_type, pgd.descriptionFROM information_schema.columns c@@ -50,15 +50,32 @@ impl QueryEngine for PostgresEngine {).fetch_all(&pool).await?;+ // 键是锦上添花:读不出来就照从前那样只给列,不让整次取 schema 失败(#502)+ let keys = match keys(&pool).await {+ Ok(k) => k,+ Err(e) => {+ tracing::warn!(error = %e, "读不出主键/外键,schema 不带键标记");+ Keys::default()+ }+ };pool.close().await;- Ok(rows++ Ok(cols.into_iter()- .map(|(schema, table, column, data_type, comment)| SchemaColumn {- schema,- table,- column,- data_type,- comment,+ .map(|(schema, table, column, data_type, comment)| {+ let key = (schema, table, column);+ let is_primary_key = keys.primary.contains(&key);+ let references_table = keys.foreign.get(&key).cloned();+ let (schema, table, column) = key;+ SchemaColumn {+ schema,+ table,+ column,+ data_type,+ comment,+ is_primary_key,+ references_table,+ }}).collect())}@@ -91,3 +108,251 @@ impl QueryEngine for PostgresEngine {Ok(QueryResult { rows, truncated })}}++type ColumnKey = (String, String, String);++/// 一个库里的单列主键与单列外键,按 (schema, table, column) 查+#[derive(Default)]+struct Keys {+ primary: std::collections::HashSet<ColumnKey>,+ /// 外键列 → 它指向的 `schema.table`+ foreign: std::collections::HashMap<ColumnKey, String>,+}++/// 从 `pg_constraint` 读键,**不走 `information_schema`**。三个原因,都实测过:+///+/// - `information_schema.table_constraints` / `key_column_usage` 只给表的属主或有+/// SELECT 以外权限的角色看。BI 连接通常只有 SELECT,于是一个键都读不到,也不报错;+/// `pg_catalog` 对所有角色可见。+/// - `information_schema` 的约束视图按约束名连接,千张表时要几秒到几分钟;这里+/// 是按 oid 连接,毫秒级。取 schema 在挂载请求里就会跑,不能挂住。+/// - 约束名只在一张表内唯一。两张表各有一个叫 `fk_ref` 的外键,按 schema + 约束名+/// 连接会把目标串到别的表上;`conrelid` / `confrelid` 是这条约束自己的表。+///+/// 只认**单列**的键:组合主键、组合外键的成员不标。探索提示词拿 PK 当「一行是什么」+/// 的 ID、拿 FK 当关联路径,把组合键的一列单独标出来是误导。一列同时在两个单列外键里+/// (少见)取排序在前的那个,结果稳定+async fn keys(pool: &sqlx::PgPool) -> anyhow::Result<Keys> {+ let rows: Vec<(String, String, String, String, Option<String>)> = sqlx::query_as(+ "SELECT n.nspname, c.relname, a.attname, k.contype::text,+ CASE WHEN k.contype = 'f' THEN rn.nspname || '.' || rc.relname END+ FROM pg_catalog.pg_constraint k+ JOIN pg_catalog.pg_class c ON c.oid = k.conrelid+ JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace+ JOIN pg_catalog.pg_attribute a+ ON a.attrelid = k.conrelid AND a.attnum = k.conkey[1]+ LEFT JOIN pg_catalog.pg_class rc ON rc.oid = k.confrelid+ LEFT JOIN pg_catalog.pg_namespace rn ON rn.oid = rc.relnamespace+ WHERE k.contype IN ('p', 'f')+ AND cardinality(k.conkey) = 1+ AND n.nspname NOT IN ('pg_catalog', 'information_schema')+ ORDER BY 1, 2, 3, 4, 5",+ )+ .fetch_all(pool)+ .await?;+ let mut keys = Keys::default();+ for (schema, table, column, kind, target) in rows {+ let key = (schema, table, column);+ match (kind.as_str(), target) {+ ("p", _) => {+ keys.primary.insert(key);+ }+ ("f", Some(target)) => {+ keys.foreign.entry(key).or_insert(target);+ }+ _ => {}+ }+ }+ Ok(keys)+}++#[cfg(test)]+mod tests {+ use super::{PostgresEngine, QueryEngine, SchemaColumn};+ use sqlx::postgres::PgPoolOptions;+ use uuid::Uuid;++ /// 每个测试自己的 schema,名字带随机后缀:并行跑不撞,也不碰库里已有的表。+ /// 测完 `DROP SCHEMA ... CASCADE`+ struct Fx {+ url: String,+ pool: sqlx::PgPool,+ schemas: Vec<String>,+ }++ impl Fx {+ async fn new(schemas: usize) -> Option<Self> {+ let url = utopia_store::test_db::url()?;+ let pool = PgPoolOptions::new()+ .max_connections(1)+ .connect(&url)+ .await+ .expect("connect");+ let suffix = Uuid::now_v7().simple().to_string();+ let schemas: Vec<String> = (0..schemas)+ .map(|i| format!("schema_keys_{i}_{}", &suffix[suffix.len() - 12..]))+ .collect();+ for s in &schemas {+ sqlx::query(&format!("CREATE SCHEMA {s}"))+ .execute(&pool)+ .await+ .expect("create schema");+ }+ Some(Self { url, pool, schemas })+ }++ async fn exec(&self, sql: &str) {+ sqlx::raw_sql(sql).execute(&self.pool).await.expect(sql);+ }++ async fn columns(&self, url: &str) -> Vec<SchemaColumn> {+ let mut cols = PostgresEngine::new(url)+ .fetch_schema()+ .await+ .expect("schema");+ cols.retain(|c| self.schemas.contains(&c.schema));+ cols+ }++ async fn cleanup(self) {+ for s in &self.schemas {+ sqlx::query(&format!("DROP SCHEMA IF EXISTS {s} CASCADE"))+ .execute(&self.pool)+ .await+ .expect("drop schema");+ }+ self.pool.close().await;+ }+ }++ fn col<'a>(+ cols: &'a [SchemaColumn],+ schema: &str,+ table: &str,+ column: &str,+ ) -> &'a SchemaColumn {+ cols.iter()+ .find(|c| c.schema == schema && c.table == table && c.column == column)+ .unwrap_or_else(|| panic!("{schema}.{table}.{column}"))+ }++ /// 单列主键标出来;组合主键、组合外键的成员都不标+ #[tokio::test]+ async fn a_single_column_key_is_marked_and_a_composite_one_is_not() {+ let Some(fx) = Fx::new(1).await else { return };+ let a = fx.schemas[0].clone();+ fx.exec(&format!(+ "CREATE TABLE {a}.parent (id serial PRIMARY KEY, name text NOT NULL, note text);+ CREATE TABLE {a}.line (+ order_id int NOT NULL,+ ordinal int NOT NULL,+ payload text,+ PRIMARY KEY (order_id, ordinal)+ );+ CREATE TABLE {a}.line_note (+ order_id int,+ ordinal int,+ FOREIGN KEY (order_id, ordinal) REFERENCES {a}.line (order_id, ordinal)+ );"+ ))+ .await;++ let cols = fx.columns(&fx.url).await;+ let id = col(&cols, &a, "parent", "id");+ assert!(id.is_primary_key && id.references_table.is_none());+ let name = col(&cols, &a, "parent", "name");+ assert!(!name.is_primary_key);+ let note = col(&cols, &a, "parent", "note");+ assert!(!note.is_primary_key && note.references_table.is_none());+ for c in ["order_id", "ordinal"] {+ let member = col(&cols, &a, "line", c);+ assert!(!member.is_primary_key, "组合主键的成员 line.{c} 不标");+ let fk_member = col(&cols, &a, "line_note", c);+ assert!(+ fk_member.references_table.is_none(),+ "组合外键的成员 line_note.{c} 不标"+ );+ }+ fx.cleanup().await;+ }++ /// 外键带上它指向的表:自引用、跨 schema,以及两张表上**同名**的约束各指各的。+ /// 约束名只在一张表内唯一,按名字连接会把目标串到别处+ #[tokio::test]+ async fn a_foreign_key_points_at_its_own_target() {+ let Some(fx) = Fx::new(2).await else { return };+ let (a, b) = (fx.schemas[0].clone(), fx.schemas[1].clone());+ fx.exec(&format!(+ "CREATE TABLE {a}.p1 (id int PRIMARY KEY, parent int REFERENCES {a}.p1 (id));+ CREATE TABLE {a}.p2 (id int PRIMARY KEY);+ CREATE TABLE {a}.c1 (x int CONSTRAINT fk_ref REFERENCES {a}.p1 (id));+ CREATE TABLE {a}.c2 (y int CONSTRAINT fk_ref REFERENCES {a}.p2 (id));+ -- 与 p2 的主键约束同名的外键+ CREATE TABLE {a}.c3 (z int CONSTRAINT p2_pkey REFERENCES {a}.p1 (id));+ CREATE TABLE {b}.orders (item_id int NOT NULL REFERENCES {a}.p2 (id));"+ ))+ .await;++ let cols = fx.columns(&fx.url).await;+ let target = |s: &str, t: &str, c: &str| col(&cols, s, t, c).references_table.clone();+ let p1 = format!("{a}.p1");+ let p2 = format!("{a}.p2");+ assert_eq!(target(&a, "p1", "parent"), Some(p1.clone()), "自引用");+ assert_eq!(target(&a, "c1", "x"), Some(p1.clone()));+ assert_eq!(target(&a, "c2", "y"), Some(p2.clone()), "同名约束不串表");+ assert_eq!(target(&a, "c3", "z"), Some(p1.clone()));+ assert_eq!(+ target(&b, "orders", "item_id"),+ Some(p2.clone()),+ "跨 schema"+ );+ let p2_id = col(&cols, &a, "p2", "id");+ assert!(+ p2_id.is_primary_key && p2_id.references_table.is_none(),+ "同名的外键不影响 p2 的主键"+ );+ fx.cleanup().await;+ }++ /// 只有 SELECT 权限的连接照样读得到键。`information_schema` 的约束视图对这种+ /// 角色是空的——BI 连接大多就是这种角色。建不了角色(库用户没有 CREATEROLE)就跳过+ #[tokio::test]+ async fn a_read_only_login_still_sees_the_keys() {+ let Some(fx) = Fx::new(1).await else { return };+ let a = fx.schemas[0].clone();+ fx.exec(&format!(+ "CREATE TABLE {a}.p (id int PRIMARY KEY);+ CREATE TABLE {a}.c (p_id int REFERENCES {a}.p (id));"+ ))+ .await;+ let role = format!("{a}_ro");+ let password = Uuid::now_v7().simple().to_string();+ if let Err(e) = sqlx::query(&format!("CREATE ROLE {role} LOGIN PASSWORD '{password}'"))+ .execute(&fx.pool)+ .await+ {+ eprintln!("跳过:建不了只读角色({e})");+ fx.cleanup().await;+ return;+ }+ fx.exec(&format!(+ "GRANT USAGE ON SCHEMA {a} TO {role};+ GRANT SELECT ON ALL TABLES IN SCHEMA {a} TO {role};"+ ))+ .await;+ let mut ro = url::Url::parse(&fx.url).expect("database url");+ ro.set_username(&role).expect("username");+ ro.set_password(Some(&password)).expect("password");++ let cols = fx.columns(ro.as_str()).await;+ assert!(col(&cols, &a, "p", "id").is_primary_key);+ assert_eq!(+ col(&cols, &a, "c", "p_id").references_table,+ Some(format!("{a}.p"))+ );++ fx.exec(&format!("DROP OWNED BY {role}; DROP ROLE {role};"))+ .await;+ fx.cleanup().await;+ }+}diff --git a/crates/utopia-server/src/query_engine/trino.rs b/crates/utopia-server/src/query_engine/trino.rsindex 337d4d41..386f23eb 100644--- a/crates/utopia-server/src/query_engine/trino.rs+++ b/crates/utopia-server/src/query_engine/trino.rs@@ -179,6 +179,11 @@ pub(crate) fn schema_row(row: Vec<serde_json::Value>) -> SchemaColumn {.and_then(|v| v.as_str()).filter(|s| !s.is_empty()).map(str::to_string),+ // Trino 与 Snowflake 的 `information_schema.table_constraints` / `key_column_usage`+ // 拿 PK 与 FK;Databricks 的 Unity Catalog 不登记主键(PK 恒为 false)。+ // 这一刀先 false,下一刀接 #502 的 Trino/Snowflake cut 再补+ is_primary_key: false,+ references_table: None,}}