Skip to content

World's first open-source enterprise world model.

Fix Databricks schema read fallback and error handling (#702)

Qishang Zhong55f5477f2e42
diff --git a/crates/utopia-server/src/api/datasource_routes.rs b/crates/utopia-server/src/api/datasource_routes.rs
index c77990ca..b43f914e 100644
--- a/crates/utopia-server/src/api/datasource_routes.rs
+++ b/crates/utopia-server/src/api/datasource_routes.rs
@@ -248,7 +248,21 @@ pub async fn sync_schema(
Err(e) => {
let name = source_name(&state, ds_id).await;
crate::alerting::observe_schema_sync_failure(&state, kb_id, ds_id, &name, &e).await;
- Err(AppError::Other(e).into())
+ // **读不出结构回 422 带引擎原话**;我们自己这边出的错(库、写文档)照旧 500。
+ // 从前一律 AppError::Other,界面只有「Internal server error」,原因只在服务端
+ // 日志里——而 Invalid 不打日志,所以这里自己记一行
+ match e.downcast_ref::<SchemaUnreadable>() {
+ Some(unreadable) => {
+ tracing::warn!(%kb_id, %ds_id, error = %unreadable, "数据源结构读取失败");
+ Err(AppError::invalid_detail(
+ "schema_sync_failed",
+ "The data source's schema could not be read",
+ bounded(&unreadable.to_string(), 600),
+ )
+ .into())
+ }
+ None => Err(AppError::Other(e).into()),
+ }
}
}
}
@@ -272,6 +286,27 @@ pub async fn explore(
/// 拉 information_schema 生成 markdown,走三路判定摄入(同 key 原地更新)。
/// 文档挂在 per-KB 的 "Data schemas" folder 来源下。
+/// 引擎读不出库表结构(连接、权限、catalog 不存在……),与我们自己这边出错分开:
+/// 前者是连接串或权限的事,回给点按钮的人;后者照旧是 500
+#[derive(Debug)]
+struct SchemaUnreadable(String);
+
+impl std::fmt::Display for SchemaUnreadable {
+ fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+ f.write_str(&self.0)
+ }
+}
+
+impl std::error::Error for SchemaUnreadable {}
+
+/// 截到 `max` 个字符以内(按字符边界),引擎的报错可能带整段堆栈
+fn bounded(text: &str, max: usize) -> String {
+ match text.char_indices().nth(max) {
+ Some((cut, _)) => format!("{}…", &text[..cut]),
+ None => text.to_string(),
+ }
+}
+
async fn sync_schema_doc(state: &AppState, kb_id: Uuid, ds_id: Uuid) -> anyhow::Result<usize> {
const MAX_TABLES: usize = 200;
let name = utopia_store::datasources::list(&state.pool)
@@ -283,7 +318,8 @@ async fn sync_schema_doc(state: &AppState, kb_id: Uuid, ds_id: Uuid) -> anyhow::
let (engine, conn) = utopia_store::datasources::engine_and_conn(&state.pool, ds_id).await?;
let cols = crate::query_engine::engine_for(&engine, &conn)?
.fetch_schema()
- .await?;
+ .await
+ .map_err(|e| anyhow::Error::new(SchemaUnreadable(e.to_string())))?;
let mut md = format!(
"# Data source: {name}\n\nEngine: {engine}. Tables and columns available for SQL queries against this source; write SQL in this engine's dialect.\n"
diff --git a/crates/utopia-server/src/query_engine/databricks.rs b/crates/utopia-server/src/query_engine/databricks.rs
index 7f9eb379..34641a83 100644
--- a/crates/utopia-server/src/query_engine/databricks.rs
+++ b/crates/utopia-server/src/query_engine/databricks.rs
@@ -62,6 +62,45 @@ impl DatabricksEngine {
Self { conn }
}
+ /// information_schema.columns 的候选写法 `(说明, SQL)`,说明只用来报错。
+ /// 没有 catalog 时只剩一条:会话默认那份。
+ fn schema_queries(&self) -> Vec<(&'static str, String)> {
+ let schema_filter = self
+ .conn
+ .schema
+ .as_deref()
+ .map(|s| format!(" AND table_schema = {}", sql_literal(s)))
+ .unwrap_or_default();
+ let select = "SELECT table_schema, table_name, column_name, data_type, comment";
+ let tail = format!(
+ "table_schema <> 'information_schema'{schema_filter} \
+ ORDER BY table_schema, table_name, ordinal_position"
+ );
+ match self.conn.catalog.as_deref() {
+ Some(catalog) => vec![
+ (
+ "catalog information_schema",
+ format!(
+ "{select} FROM `{}`.information_schema.columns WHERE {tail}",
+ catalog.replace('`', "``")
+ ),
+ ),
+ (
+ "system information_schema",
+ format!(
+ "{select} FROM system.information_schema.columns \
+ WHERE table_catalog = {} AND {tail}",
+ sql_literal(catalog)
+ ),
+ ),
+ ],
+ None => vec![(
+ "session information_schema",
+ format!("{select} FROM information_schema.columns WHERE {tail}"),
+ )],
+ }
+ }
+
async fn run(&self, sql: &str) -> anyhow::Result<(Vec<String>, Vec<Vec<serde_json::Value>>)> {
let client = super::http()?;
let mut body = json!({
@@ -151,6 +190,19 @@ impl DatabricksEngine {
}
}
+/// 引擎说的是「这个 catalog / 表不存在(或看不见)」。认的是 Databricks 的错误类名,
+/// 它们是协议的一部分(`[TABLE_OR_VIEW_NOT_FOUND] …`),不是措辞
+fn is_not_found(e: &anyhow::Error) -> bool {
+ const CLASSES: [&str; 4] = [
+ "TABLE_OR_VIEW_NOT_FOUND",
+ "NO_SUCH_CATALOG_EXCEPTION",
+ "CATALOG_NOT_FOUND",
+ "SCHEMA_NOT_FOUND",
+ ];
+ let text = e.to_string();
+ CLASSES.iter().any(|c| text.contains(c))
+}
+
#[async_trait::async_trait]
impl QueryEngine for DatabricksEngine {
async fn test(&self) -> anyhow::Result<()> {
@@ -158,27 +210,38 @@ impl QueryEngine for DatabricksEngine {
}
async fn fetch_schema(&self) -> anyhow::Result<Vec<SchemaColumn>> {
- // 带 catalog 就查那个 catalog 的 information_schema;不带就是会话默认的
- let prefix = self
- .conn
- .catalog
- .as_deref()
- .map(|c| format!("`{}`.", c.replace('`', "``")))
- .unwrap_or_default();
- let schema_filter = self
- .conn
- .schema
- .as_deref()
- .map(|s| format!(" AND table_schema = {}", sql_literal(s)))
- .unwrap_or_default();
- let sql = format!(
- "SELECT table_schema, table_name, column_name, data_type, comment \
- FROM {prefix}information_schema.columns \
- WHERE table_schema <> 'information_schema'{schema_filter} \
- ORDER BY table_schema, table_name, ordinal_position"
- );
- let (_, rows) = self.run(&sql).await?;
- Ok(rows.into_iter().map(super::trino::schema_row).collect())
+ // 两条候选,先准后全。真实集群上 `main`.information_schema.columns 回过
+ // TABLE_OR_VIEW_NOT_FOUND(#241),所以第一条**说找不到**时退到 system 那份。
+ //
+ // 只有「找不到」才退:令牌不对、仓库停着、超时,换一条语句也还是这样,再跑一遍
+ // 只是让点按钮的人多等半分钟。退过去读到 0 行也算失败——两份 information_schema
+ // 都只列令牌看得见的对象,system 那份还不含 hive_metastore,看不见的 catalog
+ // 在那里是「成功地没有行」;当成功的话,结构文档会被一份空的覆盖掉。
+ // 失败时每条试过的写法连同引擎的原话都带上
+ let queries = self.schema_queries();
+ let mut tried: Vec<String> = Vec::new();
+ for (i, (label, sql)) in queries.iter().enumerate() {
+ match self.run(sql).await {
+ Ok((_, rows)) if rows.is_empty() && i > 0 => {
+ tried.push(format!(
+ "{label}: no columns for this catalog — the token cannot see it, \
+ or it is not managed by Unity Catalog"
+ ));
+ break;
+ }
+ Ok((_, rows)) => {
+ return Ok(rows.into_iter().map(super::trino::schema_row).collect())
+ }
+ Err(e) => {
+ let missing = is_not_found(&e);
+ tried.push(format!("{label}: {e}"));
+ if !missing {
+ break;
+ }
+ }
+ }
+ }
+ anyhow::bail!("no readable information_schema ({})", tried.join("; "))
}
async fn execute(&self, sql: &str) -> anyhow::Result<QueryResult> {
@@ -197,7 +260,7 @@ mod tests {
use super::super::QueryEngine;
use super::DatabricksEngine;
use serde_json::json;
- use wiremock::matchers::{header, method, path};
+ use wiremock::matchers::{body_string_contains, header, method, path};
use wiremock::{Mock, MockServer, ResponseTemplate};
fn conn(server: &MockServer) -> DatabricksConn {
@@ -268,4 +331,148 @@ mod tests {
.to_string();
assert!(err.contains("TABLE_OR_VIEW_NOT_FOUND"), "{err}");
}
+
+ /// catalog 级那份读不到(真实集群回过 TABLE_OR_VIEW_NOT_FOUND)时退到 system 那份。
+ #[tokio::test]
+ async fn the_schema_read_falls_back_to_the_system_information_schema() {
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(path("/api/2.0/sql/statements"))
+ .and(body_string_contains("`main`.information_schema.columns"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "statement_id": "s3",
+ "status": { "state": "FAILED", "error": {
+ "error_code": "BAD_REQUEST",
+ "message": "[TABLE_OR_VIEW_NOT_FOUND] The table or view `main`.`information_schema`.`columns` cannot be found."
+ } }
+ })))
+ .expect(1)
+ .mount(&server)
+ .await;
+ Mock::given(method("POST"))
+ .and(path("/api/2.0/sql/statements"))
+ .and(body_string_contains("system.information_schema.columns"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "statement_id": "s4",
+ "status": { "state": "SUCCEEDED" },
+ "manifest": { "schema": { "columns": [
+ { "name": "table_schema", "type_text": "STRING", "position": 0 },
+ { "name": "table_name", "type_text": "STRING", "position": 1 },
+ { "name": "column_name", "type_text": "STRING", "position": 2 },
+ { "name": "data_type", "type_text": "STRING", "position": 3 },
+ { "name": "comment", "type_text": "STRING", "position": 4 }
+ ] } },
+ "result": { "data_array": [
+ ["default", "orders", "amount", "DECIMAL(12,2)", "订单金额"]
+ ] }
+ })))
+ .expect(1)
+ .mount(&server)
+ .await;
+
+ let cols = DatabricksEngine::new(conn(&server))
+ .fetch_schema()
+ .await
+ .unwrap();
+ assert_eq!(cols.len(), 1);
+ assert_eq!(cols[0].schema, "default");
+ assert_eq!(cols[0].table, "orders");
+ assert_eq!(cols[0].column, "amount");
+ assert_eq!(cols[0].data_type, "DECIMAL(12,2)");
+ assert_eq!(cols[0].comment.as_deref(), Some("订单金额"));
+ }
+
+ fn failed(code: &str, message: &str) -> ResponseTemplate {
+ ResponseTemplate::new(200).set_body_json(json!({
+ "status": { "state": "FAILED", "error": { "error_code": code, "message": message } }
+ }))
+ }
+
+ /// 两条都不通时,错误带上每一条试过的写法和它自己的原话。
+ #[tokio::test]
+ async fn two_dead_ends_say_which_ones_were_tried() {
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(path("/api/2.0/sql/statements"))
+ .and(body_string_contains("`main`.information_schema.columns"))
+ .respond_with(failed(
+ "BAD_REQUEST",
+ "[TABLE_OR_VIEW_NOT_FOUND] first words",
+ ))
+ .mount(&server)
+ .await;
+ Mock::given(method("POST"))
+ .and(path("/api/2.0/sql/statements"))
+ .and(body_string_contains("system.information_schema.columns"))
+ .respond_with(failed("PERMISSION_DENIED", "second words"))
+ .mount(&server)
+ .await;
+
+ let err = DatabricksEngine::new(conn(&server))
+ .fetch_schema()
+ .await
+ .unwrap_err()
+ .to_string();
+ assert!(err.contains("catalog information_schema"), "{err}");
+ assert!(err.contains("first words"), "{err}");
+ assert!(err.contains("system information_schema"), "{err}");
+ assert!(err.contains("second words"), "{err}");
+ }
+
+ /// 令牌不对、仓库停着这类错误不是「找不到」:换一条语句也一样,只发一次
+ #[tokio::test]
+ async fn an_error_that_is_not_a_missing_table_is_not_retried() {
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(path("/api/2.0/sql/statements"))
+ .respond_with(failed("PERMISSION_DENIED", "token lacks USE CATALOG"))
+ .expect(1)
+ .mount(&server)
+ .await;
+
+ let err = DatabricksEngine::new(conn(&server))
+ .fetch_schema()
+ .await
+ .unwrap_err()
+ .to_string();
+ assert!(err.contains("token lacks USE CATALOG"), "{err}");
+ assert!(!err.contains("system information_schema"), "{err}");
+ }
+
+ /// 退到 system 那份读到 0 行:看不见这个 catalog,或它不归 Unity Catalog 管。
+ /// 这是失败,不是一份空的结构——否则刷新会把原来的结构文档覆盖成空的
+ #[tokio::test]
+ async fn a_fallback_that_reads_no_columns_is_a_failure() {
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(path("/api/2.0/sql/statements"))
+ .and(body_string_contains("`main`.information_schema.columns"))
+ .respond_with(failed(
+ "BAD_REQUEST",
+ "[TABLE_OR_VIEW_NOT_FOUND] cannot be found",
+ ))
+ .mount(&server)
+ .await;
+ Mock::given(method("POST"))
+ .and(path("/api/2.0/sql/statements"))
+ .and(body_string_contains("system.information_schema.columns"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "statement_id": "s5",
+ "status": { "state": "SUCCEEDED" },
+ "manifest": { "schema": { "columns": [
+ { "name": "table_schema", "type_text": "STRING", "position": 0 }
+ ] } },
+ "result": { "data_array": [] }
+ })))
+ .mount(&server)
+ .await;
+
+ let err = DatabricksEngine::new(conn(&server))
+ .fetch_schema()
+ .await
+ .unwrap_err()
+ .to_string();
+ assert!(err.contains("no columns for this catalog"), "{err}");
+ assert!(err.contains("cannot be found"), "{err}");
+ }
}
diff --git a/web/src/i18n/en.ts b/web/src/i18n/en.ts
index 4868dff0..79175b48 100644
--- a/web/src/i18n/en.ts
+++ b/web/src/i18n/en.ts
@@ -75,6 +75,10 @@ export const en = {
ds_name_required: "Give this data source a name.",
only_postgres: "Only PostgreSQL is supported for now.",
bad_conn_string: "A connection string starts with postgres://",
+ // 刷新结构失败;引擎的原话走 detail(括号里那句)
+ schema_sync_failed:
+ "The schema could not be read — the engine's own message is in parentheses. Check " +
+ "the catalog, schema and account privileges in the connection string, then try Refresh schema again.",
concurrency_range: "Pick a number between 1 and 256.",
inference_off:
"Materialized inference is off for this knowledge base. Turn it on in Settings.",
diff --git a/web/src/i18n/zh.ts b/web/src/i18n/zh.ts
index 8fa8fd8e..c0801312 100644
--- a/web/src/i18n/zh.ts
+++ b/web/src/i18n/zh.ts
@@ -68,6 +68,10 @@ export const zh: Strings = {
ds_name_required: "给这个数据源起个名字。",
only_postgres: "目前只支持 PostgreSQL。",
bad_conn_string: "连接串要以 postgres:// 开头",
+ // 刷新结构失败;引擎的原话走 detail(括号里那句)
+ schema_sync_failed:
+ "库表结构没能读出来——引擎的原话在括号里。确认连接串里的库、schema 与账号权限," +
+ "再点「刷新结构」试试。",
concurrency_range: "填一个 1 到 256 之间的数。",
inference_off: "这个库没有打开物化推理。到设置里开。",
bad_resolution: "这不是一个有效的裁决。",