Skip to content

World's first open-source enterprise world model.

Schema columns carry primary and foreign keys (#671)

Royce61ece8e3a787
diff --git a/crates/utopia-server/src/mappings.rs b/crates/utopia-server/src/mappings.rs
index 2e7163c8..da8c91c7 100644
--- a/crates/utopia-server/src/mappings.rs
+++ b/crates/utopia-server/src/mappings.rs
@@ -16,6 +16,42 @@ use uuid::Uuid;
const MAX_SCHEMA_CHARS: usize = 12_000;
+/// schema 文本开头那一句图例:列行里的方括号是什么
+const COLUMN_MARKERS: &str =
+ "Column markers: [PK] single-column primary key; [FK→schema.table] foreign key to that table.
+";
+
+/// schema 文本里的一列:` 名字 类型 [PK, FK→schema.table] -- 注释`。
+///
+/// 键标记放在类型后面、注释前面,用方括号:`-- ` 已经是注释的分隔,写成 `-- PK`
+/// 就与一条内容恰好是「PK」的注释分不开。主键又是外键(一对一的扩展表)两个都标。
+/// 可空不写进来:宽表上几乎每列都是 NOT NULL,每列多十几个字符会让
+/// `MAX_SCHEMA_CHARS` 装下的列明显变少,而两个提示词都用不到它(#502)
+fn column_line(c: &crate::query_engine::SchemaColumn) -> String {
+ let mut marks = Vec::new();
+ if c.is_primary_key {
+ marks.push("PK".to_string());
+ }
+ if let Some(target) = &c.references_table {
+ marks.push(format!("FK→{target}"));
+ }
+ let marks = if marks.is_empty() {
+ String::new()
+ } else {
+ format!(" [{}]", marks.join(", "))
+ };
+ let comment = c
+ .comment
+ .as_deref()
+ .map(|x| format!(" -- {x}"))
+ .unwrap_or_default();
+ format!(
+ " {} {}{marks}{comment}
+",
+ c.column, c.data_type
+ )
+}
+
/// 一轮允许提几条口径。
///
/// **上限跟着 schema 的大小走。** 写死的 12 对一个三张表的小库绰绰有余,
@@ -191,7 +227,7 @@ async fn explore(state: &AppState, kb_id: Uuid, run: Uuid) -> anyhow::Result<()>
// **上限是跨源的一个总数。** 从前那个 `break` 只跳出当前源的列循环,
// 下一个源接着往同一个字符串里追加——`MAX_SCHEMA_CHARS` 读起来像个上限,
// 实际上是「每个源各自超一次」的下限。
- let mut schema_txt = String::new();
+ let mut schema_txt = String::from(COLUMN_MARKERS);
let mut tables_scanned = 0i32;
let mut columns_scanned = 0i32;
let mut truncated = false;
@@ -213,12 +249,7 @@ async fn explore(state: &AppState, kb_id: Uuid, run: Uuid) -> anyhow::Result<()>
schema_txt.push_str(&format!("table {key}:\n"));
}
columns_scanned += 1;
- schema_txt.push_str(&format!(
- " {} {}{}\n",
- c.column,
- c.data_type,
- c.comment.map(|x| format!(" -- {x}")).unwrap_or_default()
- ));
+ schema_txt.push_str(&column_line(&c));
if schema_txt.len() > MAX_SCHEMA_CHARS {
schema_txt.push_str("(truncated)\n");
truncated = true;
@@ -503,7 +534,51 @@ async fn explore(state: &AppState, kb_id: Uuid, run: Uuid) -> anyhow::Result<()>
#[cfg(test)]
mod tests {
- use super::{parse_description, proposal_cap};
+ use super::{column_line, parse_description, proposal_cap};
+ use crate::query_engine::SchemaColumn;
+
+ fn column(pk: bool, fk: Option<&str>, comment: Option<&str>) -> SchemaColumn {
+ SchemaColumn {
+ schema: "a".into(),
+ table: "t".into(),
+ column: "id".into(),
+ data_type: "integer".into(),
+ comment: comment.map(str::to_string),
+ is_primary_key: pk,
+ references_table: fk.map(str::to_string),
+ }
+ }
+
+ #[test]
+ fn a_column_line_marks_its_keys_before_the_comment() {
+ assert_eq!(
+ column_line(&column(false, None, None)),
+ " id integer
+"
+ );
+ assert_eq!(
+ column_line(&column(true, None, Some("Order id"))),
+ " id integer [PK] -- Order id
+"
+ );
+ assert_eq!(
+ column_line(&column(false, Some("a.orders"), None)),
+ " id integer [FK→a.orders]
+"
+ );
+ // 一对一的扩展表:主键同时是外键,两个都标
+ assert_eq!(
+ column_line(&column(true, Some("a.p1"), None)),
+ " id integer [PK, FK→a.p1]
+"
+ );
+ // 注释里写着 PK 的普通列,与标记分得开
+ assert_eq!(
+ column_line(&column(false, None, Some("PK"))),
+ " id integer -- PK
+"
+ );
+ }
#[test]
fn a_description_is_kept_only_when_it_says_something() {
diff --git a/crates/utopia-server/src/query_engine/mod.rs b/crates/utopia-server/src/query_engine/mod.rs
index 7e5cb6ce..bf933f9c 100644
--- a/crates/utopia-server/src/query_engine/mod.rs
+++ b/crates/utopia-server/src/query_engine/mod.rs
@@ -54,6 +54,18 @@ pub struct SchemaColumn {
pub column: String,
pub data_type: String,
pub comment: Option<String>,
+ /// 该列本身是这张表的主键(单列主键;组合主键里这一位仍为 false)。
+ /// 目前只有 Postgres 从 catalog 读出来;MySQL / Trino / Snowflake / Databricks
+ /// 恒为 false,意思是「不知道」,不是「不是键」——各自的读法是 #502 的后几刀。
+ /// 探索提示词靠它把 ID 与量分开,宽表上一个八十列的 schema 没有这个
+ /// 几乎认不出哪一列是键(#502)
+ pub is_primary_key: bool,
+ /// 该列本身是某张表的单列外键时,它指向的表(`schema.table`);组合外键的成员
+ /// 与读不出外键的引擎都是 None。
+ ///
+ /// 可空没有放进来:两个探索提示词都不用它,宽表上每列多一个 NOT NULL 只是挤掉
+ /// 列(#502 真用到时再加)
+ pub references_table: Option<String>,
}
#[async_trait::async_trait]
diff --git a/crates/utopia-server/src/query_engine/mysql.rs b/crates/utopia-server/src/query_engine/mysql.rs
index 8eb386f4..a6573a57 100644
--- a/crates/utopia-server/src/query_engine/mysql.rs
+++ b/crates/utopia-server/src/query_engine/mysql.rs
@@ -232,6 +232,12 @@ impl QueryEngine for MysqlEngine {
data_type,
// 没有注释时这一列是空串而不是 NULL,照抄会让每张表都挂一个空注释
comment: comment.filter(|c| !c.trim().is_empty()),
+ // MySQL 的 PK / FK 要走 information_schema.statistics 与
+ // key_column_usage 才能拿到(PK 在 INDEX_NAME='PRIMARY' 上、FK 在
+ // REFERENCED_TABLE_NAME 非空上)。这一刀先给 false,下一刀接 #502
+ // 的 MySQL cut 再补
+ is_primary_key: false,
+ references_table: None,
})
.collect())
}
diff --git a/crates/utopia-server/src/query_engine/postgres.rs b/crates/utopia-server/src/query_engine/postgres.rs
index 5bba3a4e..7ea356cd 100644
--- a/crates/utopia-server/src/query_engine/postgres.rs
+++ b/crates/utopia-server/src/query_engine/postgres.rs
@@ -37,7 +37,7 @@ impl QueryEngine for PostgresEngine {
async fn fetch_schema(&self) -> anyhow::Result<Vec<SchemaColumn>> {
let pool = self.pool().await?;
- let rows: Vec<(String, String, String, String, Option<String>)> = sqlx::query_as(
+ let cols: Vec<(String, String, String, String, Option<String>)> = sqlx::query_as(
"SELECT c.table_schema, c.table_name, c.column_name,
c.data_type, pgd.description
FROM information_schema.columns c
@@ -50,15 +50,32 @@ impl QueryEngine for PostgresEngine {
)
.fetch_all(&pool)
.await?;
+ // 键是锦上添花:读不出来就照从前那样只给列,不让整次取 schema 失败(#502)
+ let keys = match keys(&pool).await {
+ Ok(k) => k,
+ Err(e) => {
+ tracing::warn!(error = %e, "读不出主键/外键,schema 不带键标记");
+ Keys::default()
+ }
+ };
pool.close().await;
- Ok(rows
+
+ Ok(cols
.into_iter()
- .map(|(schema, table, column, data_type, comment)| SchemaColumn {
- schema,
- table,
- column,
- data_type,
- comment,
+ .map(|(schema, table, column, data_type, comment)| {
+ let key = (schema, table, column);
+ let is_primary_key = keys.primary.contains(&key);
+ let references_table = keys.foreign.get(&key).cloned();
+ let (schema, table, column) = key;
+ SchemaColumn {
+ schema,
+ table,
+ column,
+ data_type,
+ comment,
+ is_primary_key,
+ references_table,
+ }
})
.collect())
}
@@ -91,3 +108,251 @@ impl QueryEngine for PostgresEngine {
Ok(QueryResult { rows, truncated })
}
}
+
+type ColumnKey = (String, String, String);
+
+/// 一个库里的单列主键与单列外键,按 (schema, table, column) 查
+#[derive(Default)]
+struct Keys {
+ primary: std::collections::HashSet<ColumnKey>,
+ /// 外键列 → 它指向的 `schema.table`
+ foreign: std::collections::HashMap<ColumnKey, String>,
+}
+
+/// 从 `pg_constraint` 读键,**不走 `information_schema`**。三个原因,都实测过:
+///
+/// - `information_schema.table_constraints` / `key_column_usage` 只给表的属主或有
+/// SELECT 以外权限的角色看。BI 连接通常只有 SELECT,于是一个键都读不到,也不报错;
+/// `pg_catalog` 对所有角色可见。
+/// - `information_schema` 的约束视图按约束名连接,千张表时要几秒到几分钟;这里
+/// 是按 oid 连接,毫秒级。取 schema 在挂载请求里就会跑,不能挂住。
+/// - 约束名只在一张表内唯一。两张表各有一个叫 `fk_ref` 的外键,按 schema + 约束名
+/// 连接会把目标串到别的表上;`conrelid` / `confrelid` 是这条约束自己的表。
+///
+/// 只认**单列**的键:组合主键、组合外键的成员不标。探索提示词拿 PK 当「一行是什么」
+/// 的 ID、拿 FK 当关联路径,把组合键的一列单独标出来是误导。一列同时在两个单列外键里
+/// (少见)取排序在前的那个,结果稳定
+async fn keys(pool: &sqlx::PgPool) -> anyhow::Result<Keys> {
+ let rows: Vec<(String, String, String, String, Option<String>)> = sqlx::query_as(
+ "SELECT n.nspname, c.relname, a.attname, k.contype::text,
+ CASE WHEN k.contype = 'f' THEN rn.nspname || '.' || rc.relname END
+ FROM pg_catalog.pg_constraint k
+ JOIN pg_catalog.pg_class c ON c.oid = k.conrelid
+ JOIN pg_catalog.pg_namespace n ON n.oid = c.relnamespace
+ JOIN pg_catalog.pg_attribute a
+ ON a.attrelid = k.conrelid AND a.attnum = k.conkey[1]
+ LEFT JOIN pg_catalog.pg_class rc ON rc.oid = k.confrelid
+ LEFT JOIN pg_catalog.pg_namespace rn ON rn.oid = rc.relnamespace
+ WHERE k.contype IN ('p', 'f')
+ AND cardinality(k.conkey) = 1
+ AND n.nspname NOT IN ('pg_catalog', 'information_schema')
+ ORDER BY 1, 2, 3, 4, 5",
+ )
+ .fetch_all(pool)
+ .await?;
+ let mut keys = Keys::default();
+ for (schema, table, column, kind, target) in rows {
+ let key = (schema, table, column);
+ match (kind.as_str(), target) {
+ ("p", _) => {
+ keys.primary.insert(key);
+ }
+ ("f", Some(target)) => {
+ keys.foreign.entry(key).or_insert(target);
+ }
+ _ => {}
+ }
+ }
+ Ok(keys)
+}
+
+#[cfg(test)]
+mod tests {
+ use super::{PostgresEngine, QueryEngine, SchemaColumn};
+ use sqlx::postgres::PgPoolOptions;
+ use uuid::Uuid;
+
+ /// 每个测试自己的 schema,名字带随机后缀:并行跑不撞,也不碰库里已有的表。
+ /// 测完 `DROP SCHEMA ... CASCADE`
+ struct Fx {
+ url: String,
+ pool: sqlx::PgPool,
+ schemas: Vec<String>,
+ }
+
+ impl Fx {
+ async fn new(schemas: usize) -> Option<Self> {
+ let url = utopia_store::test_db::url()?;
+ let pool = PgPoolOptions::new()
+ .max_connections(1)
+ .connect(&url)
+ .await
+ .expect("connect");
+ let suffix = Uuid::now_v7().simple().to_string();
+ let schemas: Vec<String> = (0..schemas)
+ .map(|i| format!("schema_keys_{i}_{}", &suffix[suffix.len() - 12..]))
+ .collect();
+ for s in &schemas {
+ sqlx::query(&format!("CREATE SCHEMA {s}"))
+ .execute(&pool)
+ .await
+ .expect("create schema");
+ }
+ Some(Self { url, pool, schemas })
+ }
+
+ async fn exec(&self, sql: &str) {
+ sqlx::raw_sql(sql).execute(&self.pool).await.expect(sql);
+ }
+
+ async fn columns(&self, url: &str) -> Vec<SchemaColumn> {
+ let mut cols = PostgresEngine::new(url)
+ .fetch_schema()
+ .await
+ .expect("schema");
+ cols.retain(|c| self.schemas.contains(&c.schema));
+ cols
+ }
+
+ async fn cleanup(self) {
+ for s in &self.schemas {
+ sqlx::query(&format!("DROP SCHEMA IF EXISTS {s} CASCADE"))
+ .execute(&self.pool)
+ .await
+ .expect("drop schema");
+ }
+ self.pool.close().await;
+ }
+ }
+
+ fn col<'a>(
+ cols: &'a [SchemaColumn],
+ schema: &str,
+ table: &str,
+ column: &str,
+ ) -> &'a SchemaColumn {
+ cols.iter()
+ .find(|c| c.schema == schema && c.table == table && c.column == column)
+ .unwrap_or_else(|| panic!("{schema}.{table}.{column}"))
+ }
+
+ /// 单列主键标出来;组合主键、组合外键的成员都不标
+ #[tokio::test]
+ async fn a_single_column_key_is_marked_and_a_composite_one_is_not() {
+ let Some(fx) = Fx::new(1).await else { return };
+ let a = fx.schemas[0].clone();
+ fx.exec(&format!(
+ "CREATE TABLE {a}.parent (id serial PRIMARY KEY, name text NOT NULL, note text);
+ CREATE TABLE {a}.line (
+ order_id int NOT NULL,
+ ordinal int NOT NULL,
+ payload text,
+ PRIMARY KEY (order_id, ordinal)
+ );
+ CREATE TABLE {a}.line_note (
+ order_id int,
+ ordinal int,
+ FOREIGN KEY (order_id, ordinal) REFERENCES {a}.line (order_id, ordinal)
+ );"
+ ))
+ .await;
+
+ let cols = fx.columns(&fx.url).await;
+ let id = col(&cols, &a, "parent", "id");
+ assert!(id.is_primary_key && id.references_table.is_none());
+ let name = col(&cols, &a, "parent", "name");
+ assert!(!name.is_primary_key);
+ let note = col(&cols, &a, "parent", "note");
+ assert!(!note.is_primary_key && note.references_table.is_none());
+ for c in ["order_id", "ordinal"] {
+ let member = col(&cols, &a, "line", c);
+ assert!(!member.is_primary_key, "组合主键的成员 line.{c} 不标");
+ let fk_member = col(&cols, &a, "line_note", c);
+ assert!(
+ fk_member.references_table.is_none(),
+ "组合外键的成员 line_note.{c} 不标"
+ );
+ }
+ fx.cleanup().await;
+ }
+
+ /// 外键带上它指向的表:自引用、跨 schema,以及两张表上**同名**的约束各指各的。
+ /// 约束名只在一张表内唯一,按名字连接会把目标串到别处
+ #[tokio::test]
+ async fn a_foreign_key_points_at_its_own_target() {
+ let Some(fx) = Fx::new(2).await else { return };
+ let (a, b) = (fx.schemas[0].clone(), fx.schemas[1].clone());
+ fx.exec(&format!(
+ "CREATE TABLE {a}.p1 (id int PRIMARY KEY, parent int REFERENCES {a}.p1 (id));
+ CREATE TABLE {a}.p2 (id int PRIMARY KEY);
+ CREATE TABLE {a}.c1 (x int CONSTRAINT fk_ref REFERENCES {a}.p1 (id));
+ CREATE TABLE {a}.c2 (y int CONSTRAINT fk_ref REFERENCES {a}.p2 (id));
+ -- 与 p2 的主键约束同名的外键
+ CREATE TABLE {a}.c3 (z int CONSTRAINT p2_pkey REFERENCES {a}.p1 (id));
+ CREATE TABLE {b}.orders (item_id int NOT NULL REFERENCES {a}.p2 (id));"
+ ))
+ .await;
+
+ let cols = fx.columns(&fx.url).await;
+ let target = |s: &str, t: &str, c: &str| col(&cols, s, t, c).references_table.clone();
+ let p1 = format!("{a}.p1");
+ let p2 = format!("{a}.p2");
+ assert_eq!(target(&a, "p1", "parent"), Some(p1.clone()), "自引用");
+ assert_eq!(target(&a, "c1", "x"), Some(p1.clone()));
+ assert_eq!(target(&a, "c2", "y"), Some(p2.clone()), "同名约束不串表");
+ assert_eq!(target(&a, "c3", "z"), Some(p1.clone()));
+ assert_eq!(
+ target(&b, "orders", "item_id"),
+ Some(p2.clone()),
+ "跨 schema"
+ );
+ let p2_id = col(&cols, &a, "p2", "id");
+ assert!(
+ p2_id.is_primary_key && p2_id.references_table.is_none(),
+ "同名的外键不影响 p2 的主键"
+ );
+ fx.cleanup().await;
+ }
+
+ /// 只有 SELECT 权限的连接照样读得到键。`information_schema` 的约束视图对这种
+ /// 角色是空的——BI 连接大多就是这种角色。建不了角色(库用户没有 CREATEROLE)就跳过
+ #[tokio::test]
+ async fn a_read_only_login_still_sees_the_keys() {
+ let Some(fx) = Fx::new(1).await else { return };
+ let a = fx.schemas[0].clone();
+ fx.exec(&format!(
+ "CREATE TABLE {a}.p (id int PRIMARY KEY);
+ CREATE TABLE {a}.c (p_id int REFERENCES {a}.p (id));"
+ ))
+ .await;
+ let role = format!("{a}_ro");
+ let password = Uuid::now_v7().simple().to_string();
+ if let Err(e) = sqlx::query(&format!("CREATE ROLE {role} LOGIN PASSWORD '{password}'"))
+ .execute(&fx.pool)
+ .await
+ {
+ eprintln!("跳过:建不了只读角色({e})");
+ fx.cleanup().await;
+ return;
+ }
+ fx.exec(&format!(
+ "GRANT USAGE ON SCHEMA {a} TO {role};
+ GRANT SELECT ON ALL TABLES IN SCHEMA {a} TO {role};"
+ ))
+ .await;
+ let mut ro = url::Url::parse(&fx.url).expect("database url");
+ ro.set_username(&role).expect("username");
+ ro.set_password(Some(&password)).expect("password");
+
+ let cols = fx.columns(ro.as_str()).await;
+ assert!(col(&cols, &a, "p", "id").is_primary_key);
+ assert_eq!(
+ col(&cols, &a, "c", "p_id").references_table,
+ Some(format!("{a}.p"))
+ );
+
+ fx.exec(&format!("DROP OWNED BY {role}; DROP ROLE {role};"))
+ .await;
+ fx.cleanup().await;
+ }
+}
diff --git a/crates/utopia-server/src/query_engine/trino.rs b/crates/utopia-server/src/query_engine/trino.rs
index 337d4d41..386f23eb 100644
--- a/crates/utopia-server/src/query_engine/trino.rs
+++ b/crates/utopia-server/src/query_engine/trino.rs
@@ -179,6 +179,11 @@ pub(crate) fn schema_row(row: Vec<serde_json::Value>) -> SchemaColumn {
.and_then(|v| v.as_str())
.filter(|s| !s.is_empty())
.map(str::to_string),
+ // Trino 与 Snowflake 的 `information_schema.table_constraints` / `key_column_usage`
+ // 拿 PK 与 FK;Databricks 的 Unity Catalog 不登记主键(PK 恒为 false)。
+ // 这一刀先 false,下一刀接 #502 的 Trino/Snowflake cut 再补
+ is_primary_key: false,
+ references_table: None,
}
}