World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Fix Databricks schema read fallback and error handling (#702)
Qishang Zhong55f5477f2e42
diff --git a/crates/utopia-server/src/api/datasource_routes.rs b/crates/utopia-server/src/api/datasource_routes.rsindex c77990ca..b43f914e 100644--- a/crates/utopia-server/src/api/datasource_routes.rs+++ b/crates/utopia-server/src/api/datasource_routes.rs@@ -248,7 +248,21 @@ pub async fn sync_schema(Err(e) => {let name = source_name(&state, ds_id).await;crate::alerting::observe_schema_sync_failure(&state, kb_id, ds_id, &name, &e).await;- Err(AppError::Other(e).into())+ // **读不出结构回 422 带引擎原话**;我们自己这边出的错(库、写文档)照旧 500。+ // 从前一律 AppError::Other,界面只有「Internal server error」,原因只在服务端+ // 日志里——而 Invalid 不打日志,所以这里自己记一行+ match e.downcast_ref::<SchemaUnreadable>() {+ Some(unreadable) => {+ tracing::warn!(%kb_id, %ds_id, error = %unreadable, "数据源结构读取失败");+ Err(AppError::invalid_detail(+ "schema_sync_failed",+ "The data source's schema could not be read",+ bounded(&unreadable.to_string(), 600),+ )+ .into())+ }+ None => Err(AppError::Other(e).into()),+ }}}}@@ -272,6 +286,27 @@ pub async fn explore(/// 拉 information_schema 生成 markdown,走三路判定摄入(同 key 原地更新)。/// 文档挂在 per-KB 的 "Data schemas" folder 来源下。+/// 引擎读不出库表结构(连接、权限、catalog 不存在……),与我们自己这边出错分开:+/// 前者是连接串或权限的事,回给点按钮的人;后者照旧是 500+#[derive(Debug)]+struct SchemaUnreadable(String);++impl std::fmt::Display for SchemaUnreadable {+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {+ f.write_str(&self.0)+ }+}++impl std::error::Error for SchemaUnreadable {}++/// 截到 `max` 个字符以内(按字符边界),引擎的报错可能带整段堆栈+fn bounded(text: &str, max: usize) -> String {+ match text.char_indices().nth(max) {+ Some((cut, _)) => format!("{}…", &text[..cut]),+ None => text.to_string(),+ }+}+async fn sync_schema_doc(state: &AppState, kb_id: Uuid, ds_id: Uuid) -> anyhow::Result<usize> {const MAX_TABLES: usize = 200;let name = utopia_store::datasources::list(&state.pool)@@ -283,7 +318,8 @@ async fn sync_schema_doc(state: &AppState, kb_id: Uuid, ds_id: Uuid) -> anyhow::let (engine, conn) = utopia_store::datasources::engine_and_conn(&state.pool, ds_id).await?;let cols = crate::query_engine::engine_for(&engine, &conn)?.fetch_schema()- .await?;+ .await+ .map_err(|e| anyhow::Error::new(SchemaUnreadable(e.to_string())))?;let mut md = format!("# Data source: {name}\n\nEngine: {engine}. Tables and columns available for SQL queries against this source; write SQL in this engine's dialect.\n"diff --git a/crates/utopia-server/src/query_engine/databricks.rs b/crates/utopia-server/src/query_engine/databricks.rsindex 7f9eb379..34641a83 100644--- a/crates/utopia-server/src/query_engine/databricks.rs+++ b/crates/utopia-server/src/query_engine/databricks.rs@@ -62,6 +62,45 @@ impl DatabricksEngine {Self { conn }}+ /// information_schema.columns 的候选写法 `(说明, SQL)`,说明只用来报错。+ /// 没有 catalog 时只剩一条:会话默认那份。+ fn schema_queries(&self) -> Vec<(&'static str, String)> {+ let schema_filter = self+ .conn+ .schema+ .as_deref()+ .map(|s| format!(" AND table_schema = {}", sql_literal(s)))+ .unwrap_or_default();+ let select = "SELECT table_schema, table_name, column_name, data_type, comment";+ let tail = format!(+ "table_schema <> 'information_schema'{schema_filter} \+ ORDER BY table_schema, table_name, ordinal_position"+ );+ match self.conn.catalog.as_deref() {+ Some(catalog) => vec![+ (+ "catalog information_schema",+ format!(+ "{select} FROM `{}`.information_schema.columns WHERE {tail}",+ catalog.replace('`', "``")+ ),+ ),+ (+ "system information_schema",+ format!(+ "{select} FROM system.information_schema.columns \+ WHERE table_catalog = {} AND {tail}",+ sql_literal(catalog)+ ),+ ),+ ],+ None => vec![(+ "session information_schema",+ format!("{select} FROM information_schema.columns WHERE {tail}"),+ )],+ }+ }+async fn run(&self, sql: &str) -> anyhow::Result<(Vec<String>, Vec<Vec<serde_json::Value>>)> {let client = super::http()?;let mut body = json!({@@ -151,6 +190,19 @@ impl DatabricksEngine {}}+/// 引擎说的是「这个 catalog / 表不存在(或看不见)」。认的是 Databricks 的错误类名,+/// 它们是协议的一部分(`[TABLE_OR_VIEW_NOT_FOUND] …`),不是措辞+fn is_not_found(e: &anyhow::Error) -> bool {+ const CLASSES: [&str; 4] = [+ "TABLE_OR_VIEW_NOT_FOUND",+ "NO_SUCH_CATALOG_EXCEPTION",+ "CATALOG_NOT_FOUND",+ "SCHEMA_NOT_FOUND",+ ];+ let text = e.to_string();+ CLASSES.iter().any(|c| text.contains(c))+}+#[async_trait::async_trait]impl QueryEngine for DatabricksEngine {async fn test(&self) -> anyhow::Result<()> {@@ -158,27 +210,38 @@ impl QueryEngine for DatabricksEngine {}async fn fetch_schema(&self) -> anyhow::Result<Vec<SchemaColumn>> {- // 带 catalog 就查那个 catalog 的 information_schema;不带就是会话默认的- let prefix = self- .conn- .catalog- .as_deref()- .map(|c| format!("`{}`.", c.replace('`', "``")))- .unwrap_or_default();- let schema_filter = self- .conn- .schema- .as_deref()- .map(|s| format!(" AND table_schema = {}", sql_literal(s)))- .unwrap_or_default();- let sql = format!(- "SELECT table_schema, table_name, column_name, data_type, comment \- FROM {prefix}information_schema.columns \- WHERE table_schema <> 'information_schema'{schema_filter} \- ORDER BY table_schema, table_name, ordinal_position"- );- let (_, rows) = self.run(&sql).await?;- Ok(rows.into_iter().map(super::trino::schema_row).collect())+ // 两条候选,先准后全。真实集群上 `main`.information_schema.columns 回过+ // TABLE_OR_VIEW_NOT_FOUND(#241),所以第一条**说找不到**时退到 system 那份。+ //+ // 只有「找不到」才退:令牌不对、仓库停着、超时,换一条语句也还是这样,再跑一遍+ // 只是让点按钮的人多等半分钟。退过去读到 0 行也算失败——两份 information_schema+ // 都只列令牌看得见的对象,system 那份还不含 hive_metastore,看不见的 catalog+ // 在那里是「成功地没有行」;当成功的话,结构文档会被一份空的覆盖掉。+ // 失败时每条试过的写法连同引擎的原话都带上+ let queries = self.schema_queries();+ let mut tried: Vec<String> = Vec::new();+ for (i, (label, sql)) in queries.iter().enumerate() {+ match self.run(sql).await {+ Ok((_, rows)) if rows.is_empty() && i > 0 => {+ tried.push(format!(+ "{label}: no columns for this catalog — the token cannot see it, \+ or it is not managed by Unity Catalog"+ ));+ break;+ }+ Ok((_, rows)) => {+ return Ok(rows.into_iter().map(super::trino::schema_row).collect())+ }+ Err(e) => {+ let missing = is_not_found(&e);+ tried.push(format!("{label}: {e}"));+ if !missing {+ break;+ }+ }+ }+ }+ anyhow::bail!("no readable information_schema ({})", tried.join("; "))}async fn execute(&self, sql: &str) -> anyhow::Result<QueryResult> {@@ -197,7 +260,7 @@ mod tests {use super::super::QueryEngine;use super::DatabricksEngine;use serde_json::json;- use wiremock::matchers::{header, method, path};+ use wiremock::matchers::{body_string_contains, header, method, path};use wiremock::{Mock, MockServer, ResponseTemplate};fn conn(server: &MockServer) -> DatabricksConn {@@ -268,4 +331,148 @@ mod tests {.to_string();assert!(err.contains("TABLE_OR_VIEW_NOT_FOUND"), "{err}");}++ /// catalog 级那份读不到(真实集群回过 TABLE_OR_VIEW_NOT_FOUND)时退到 system 那份。+ #[tokio::test]+ async fn the_schema_read_falls_back_to_the_system_information_schema() {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(path("/api/2.0/sql/statements"))+ .and(body_string_contains("`main`.information_schema.columns"))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({+ "statement_id": "s3",+ "status": { "state": "FAILED", "error": {+ "error_code": "BAD_REQUEST",+ "message": "[TABLE_OR_VIEW_NOT_FOUND] The table or view `main`.`information_schema`.`columns` cannot be found."+ } }+ })))+ .expect(1)+ .mount(&server)+ .await;+ Mock::given(method("POST"))+ .and(path("/api/2.0/sql/statements"))+ .and(body_string_contains("system.information_schema.columns"))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({+ "statement_id": "s4",+ "status": { "state": "SUCCEEDED" },+ "manifest": { "schema": { "columns": [+ { "name": "table_schema", "type_text": "STRING", "position": 0 },+ { "name": "table_name", "type_text": "STRING", "position": 1 },+ { "name": "column_name", "type_text": "STRING", "position": 2 },+ { "name": "data_type", "type_text": "STRING", "position": 3 },+ { "name": "comment", "type_text": "STRING", "position": 4 }+ ] } },+ "result": { "data_array": [+ ["default", "orders", "amount", "DECIMAL(12,2)", "订单金额"]+ ] }+ })))+ .expect(1)+ .mount(&server)+ .await;++ let cols = DatabricksEngine::new(conn(&server))+ .fetch_schema()+ .await+ .unwrap();+ assert_eq!(cols.len(), 1);+ assert_eq!(cols[0].schema, "default");+ assert_eq!(cols[0].table, "orders");+ assert_eq!(cols[0].column, "amount");+ assert_eq!(cols[0].data_type, "DECIMAL(12,2)");+ assert_eq!(cols[0].comment.as_deref(), Some("订单金额"));+ }++ fn failed(code: &str, message: &str) -> ResponseTemplate {+ ResponseTemplate::new(200).set_body_json(json!({+ "status": { "state": "FAILED", "error": { "error_code": code, "message": message } }+ }))+ }++ /// 两条都不通时,错误带上每一条试过的写法和它自己的原话。+ #[tokio::test]+ async fn two_dead_ends_say_which_ones_were_tried() {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(path("/api/2.0/sql/statements"))+ .and(body_string_contains("`main`.information_schema.columns"))+ .respond_with(failed(+ "BAD_REQUEST",+ "[TABLE_OR_VIEW_NOT_FOUND] first words",+ ))+ .mount(&server)+ .await;+ Mock::given(method("POST"))+ .and(path("/api/2.0/sql/statements"))+ .and(body_string_contains("system.information_schema.columns"))+ .respond_with(failed("PERMISSION_DENIED", "second words"))+ .mount(&server)+ .await;++ let err = DatabricksEngine::new(conn(&server))+ .fetch_schema()+ .await+ .unwrap_err()+ .to_string();+ assert!(err.contains("catalog information_schema"), "{err}");+ assert!(err.contains("first words"), "{err}");+ assert!(err.contains("system information_schema"), "{err}");+ assert!(err.contains("second words"), "{err}");+ }++ /// 令牌不对、仓库停着这类错误不是「找不到」:换一条语句也一样,只发一次+ #[tokio::test]+ async fn an_error_that_is_not_a_missing_table_is_not_retried() {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(path("/api/2.0/sql/statements"))+ .respond_with(failed("PERMISSION_DENIED", "token lacks USE CATALOG"))+ .expect(1)+ .mount(&server)+ .await;++ let err = DatabricksEngine::new(conn(&server))+ .fetch_schema()+ .await+ .unwrap_err()+ .to_string();+ assert!(err.contains("token lacks USE CATALOG"), "{err}");+ assert!(!err.contains("system information_schema"), "{err}");+ }++ /// 退到 system 那份读到 0 行:看不见这个 catalog,或它不归 Unity Catalog 管。+ /// 这是失败,不是一份空的结构——否则刷新会把原来的结构文档覆盖成空的+ #[tokio::test]+ async fn a_fallback_that_reads_no_columns_is_a_failure() {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(path("/api/2.0/sql/statements"))+ .and(body_string_contains("`main`.information_schema.columns"))+ .respond_with(failed(+ "BAD_REQUEST",+ "[TABLE_OR_VIEW_NOT_FOUND] cannot be found",+ ))+ .mount(&server)+ .await;+ Mock::given(method("POST"))+ .and(path("/api/2.0/sql/statements"))+ .and(body_string_contains("system.information_schema.columns"))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({+ "statement_id": "s5",+ "status": { "state": "SUCCEEDED" },+ "manifest": { "schema": { "columns": [+ { "name": "table_schema", "type_text": "STRING", "position": 0 }+ ] } },+ "result": { "data_array": [] }+ })))+ .mount(&server)+ .await;++ let err = DatabricksEngine::new(conn(&server))+ .fetch_schema()+ .await+ .unwrap_err()+ .to_string();+ assert!(err.contains("no columns for this catalog"), "{err}");+ assert!(err.contains("cannot be found"), "{err}");+ }}diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.tsindex 4868dff0..79175b48 100644--- a/web/src/i18n/en.ts+++ b/web/src/i18n/en.ts@@ -75,6 +75,10 @@ export const en = {ds_name_required: "Give this data source a name.",only_postgres: "Only PostgreSQL is supported for now.",bad_conn_string: "A connection string starts with postgres://",+ // 刷新结构失败;引擎的原话走 detail(括号里那句)+ schema_sync_failed:+ "The schema could not be read — the engine's own message is in parentheses. Check " ++ "the catalog, schema and account privileges in the connection string, then try Refresh schema again.",concurrency_range: "Pick a number between 1 and 256.",inference_off:"Materialized inference is off for this knowledge base. Turn it on in Settings.",diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.tsindex 8fa8fd8e..c0801312 100644--- a/web/src/i18n/zh.ts+++ b/web/src/i18n/zh.ts@@ -68,6 +68,10 @@ export const zh: Strings = {ds_name_required: "给这个数据源起个名字。",only_postgres: "目前只支持 PostgreSQL。",bad_conn_string: "连接串要以 postgres:// 开头",+ // 刷新结构失败;引擎的原话走 detail(括号里那句)+ schema_sync_failed:+ "库表结构没能读出来——引擎的原话在括号里。确认连接串里的库、schema 与账号权限," ++ "再点「刷新结构」试试。",concurrency_range: "填一个 1 到 256 之间的数。",inference_off: "这个库没有打开物化推理。到设置里开。",bad_resolution: "这不是一个有效的裁决。",