Skip to content

World's first open-source enterprise world model.

Schema reads do not apply ROW_CAP and follow result chunks (#737)

Royce4137f7d1a1cf
diff --git a/crates/utopia-server/src/query_engine/databricks.rs b/crates/utopia-server/src/query_engine/databricks.rs
index 34641a83..a6895bcd 100644
--- a/crates/utopia-server/src/query_engine/databricks.rs
+++ b/crates/utopia-server/src/query_engine/databricks.rs
@@ -39,14 +39,23 @@ struct StatusError {
#[derive(Deserialize)]
struct Manifest {
schema: Option<Schema>,
+ /// INLINE disposition with a `row_limit` set:true 表示结果按行截断。
+ /// schema 读不传 row_limit,理论上不会触发;留着做显式 fail-fast 而不是静默返回
+ /// 不全的数据
+ truncated: Option<bool>,
+ /// INLINE 第一个分块塞不下时给的下一页 GET 链接;按 docs 的语义反复 GET 直到没有
+ next_chunk_internal_link: Option<String>,
+ /// 整个结果集的总行数(chunked INLINE 时也有)。schema 读完后用来校验
+ /// `rows.len() == total_row_count`
+ total_row_count: Option<u64>,
}
-#[derive(Deserialize)]
+#[derive(Deserialize, Clone)]
struct Schema {
columns: Vec<ColumnInfo>,
}
-#[derive(Deserialize)]
+#[derive(Deserialize, Clone)]
struct ColumnInfo {
name: String,
type_text: Option<String>,
@@ -101,7 +110,23 @@ impl DatabricksEngine {
}
}
- async fn run(&self, sql: &str) -> anyhow::Result<(Vec<String>, Vec<Vec<serde_json::Value>>)> {
+ /// 提交一条语句,轮询到 SUCCEEDED,把结果拼成 (列名, 行值)。所有列值都按
+ /// manifest 的 `type_text` 还原成数或布尔。
+ ///
+ /// `row_limit`:传 `None` 表示不限行(schema 读用)。INLINE disposition 在
+ /// 服务端有 25 MiB 的 body 上限,所以单分块塞不下时通过 `next_chunk_internal_link`
+ /// 继续 GET,循环到 manifest 没有给出下一页为止。schema 读完后用
+ /// `manifest.total_row_count` 做一次 `rows.len()` 校验——少了就当结果不全
+ /// (docs 说 truncated=true 时另有标记,schema 读不该见到)
+ ///
+ /// 之前对所有语句都传 `row_limit: ROW_CAP + 1`(#703):chat 查询需要这个
+ /// 安全阀,schema 读不需要;information_schema.columns 每行是一列,宽 catalog
+ /// 上的 schema 文档会被悄悄截到 201 行
+ async fn run(
+ &self,
+ sql: &str,
+ row_limit: Option<usize>,
+ ) -> anyhow::Result<(Vec<String>, Vec<Vec<serde_json::Value>>)> {
let client = super::http()?;
let mut body = json!({
"warehouse_id": self.conn.warehouse_id,
@@ -110,8 +135,10 @@ impl DatabricksEngine {
"on_wait_timeout": "CONTINUE",
"disposition": "INLINE",
"format": "JSON_ARRAY",
- "row_limit": ROW_CAP + 1,
});
+ if let Some(n) = row_limit {
+ body["row_limit"] = json!(n);
+ }
if let Some(c) = &self.conn.catalog {
body["catalog"] = json!(c);
}
@@ -167,11 +194,15 @@ impl DatabricksEngine {
}
let columns: Vec<ColumnInfo> = resp
.manifest
- .and_then(|m| m.schema)
- .map(|s| s.columns)
+ .as_ref()
+ .and_then(|m| m.schema.as_ref())
+ .map(|s| s.columns.clone())
.unwrap_or_default();
- let raw_rows = resp.result.and_then(|r| r.data_array).unwrap_or_default();
- let rows = raw_rows
+ let mut rows: Vec<Vec<serde_json::Value>> = resp
+ .result
+ .as_ref()
+ .and_then(|r| r.data_array.clone())
+ .unwrap_or_default()
.into_iter()
.map(|row| {
row.iter()
@@ -186,6 +217,61 @@ impl DatabricksEngine {
.collect()
})
.collect();
+ // INLINE 的分块跟在第一个 chunk 之后:next_chunk_internal_link 是相对路径,
+ // docs 说按 link 反复 GET,直到 manifest 不再给出下一页为止
+ let manifest = resp.manifest.as_ref();
+ let truncated = manifest.and_then(|m| m.truncated).unwrap_or(false);
+ if truncated {
+ anyhow::bail!(
+ "Databricks returned a truncated result; row_limit must be raised or the query rewritten"
+ );
+ }
+ let mut next_link = manifest.and_then(|m| m.next_chunk_internal_link.clone());
+ while let Some(link) = next_link.take() {
+ let chunk: StatementResponse = client
+ .get(link)
+ .bearer_auth(&self.conn.token)
+ .send()
+ .await?
+ .error_for_status()?
+ .json()
+ .await?;
+ // 中间 chunk 的 manifest 可能再次给 truncated/link;状态字段对每块都要读
+ let m = chunk.manifest.as_ref();
+ if m.and_then(|m| m.truncated).unwrap_or(false) {
+ anyhow::bail!(
+ "Databricks chunk was truncated by the service; rows may be incomplete"
+ );
+ }
+ let chunk_rows = chunk.result.and_then(|r| r.data_array).unwrap_or_default();
+ rows.extend(chunk_rows.into_iter().map(|row| {
+ row.iter()
+ .enumerate()
+ .map(|(i, v)| {
+ let ty = columns
+ .get(i)
+ .and_then(|c| c.type_text.as_deref())
+ .unwrap_or("");
+ coerce(ty, v)
+ })
+ .collect()
+ }));
+ next_link = m.and_then(|m| m.next_chunk_internal_link.clone());
+ }
+ // schema 读按 docs 说不该见到 truncated=true;total_row_count 是每次都给的,
+ // 校验要放在所有 chunk 都收齐之后:第一次响应给的 total 是整个结果集的总数,
+ // 不是第一个 chunk 的总数
+ if let (Some(manifest), None) = (manifest, row_limit) {
+ let expected = manifest.total_row_count;
+ if let Some(expected) = expected {
+ let got: u64 = rows.len() as u64;
+ if got != expected {
+ anyhow::bail!(
+ "Databricks schema read returned {got} rows but manifest reports {expected}"
+ );
+ }
+ }
+ }
Ok((columns.into_iter().map(|c| c.name).collect(), rows))
}
}
@@ -206,7 +292,7 @@ fn is_not_found(e: &anyhow::Error) -> bool {
#[async_trait::async_trait]
impl QueryEngine for DatabricksEngine {
async fn test(&self) -> anyhow::Result<()> {
- self.run("SELECT 1").await.map(|_| ())
+ self.run("SELECT 1", Some(ROW_CAP + 1)).await.map(|_| ())
}
async fn fetch_schema(&self) -> anyhow::Result<Vec<SchemaColumn>> {
@@ -218,10 +304,14 @@ impl QueryEngine for DatabricksEngine {
// 都只列令牌看得见的对象,system 那份还不含 hive_metastore,看不见的 catalog
// 在那里是「成功地没有行」;当成功的话,结构文档会被一份空的覆盖掉。
// 失败时每条试过的写法连同引擎的原话都带上
+ //
+ // row_limit 不传(#703):information_schema.columns 每行是一列,宽 catalog
+ // 上的 schema 文档会被悄悄截到 201 行;现在 `run` 自己跟 `next_chunk_internal_link`
+ // 一直拉到 `manifest.total_row_count` 行,多分块也收齐
let queries = self.schema_queries();
let mut tried: Vec<String> = Vec::new();
for (i, (label, sql)) in queries.iter().enumerate() {
- match self.run(sql).await {
+ match self.run(sql, None).await {
Ok((_, rows)) if rows.is_empty() && i > 0 => {
tried.push(format!(
"{label}: no columns for this catalog — the token cannot see it, \
@@ -245,7 +335,7 @@ impl QueryEngine for DatabricksEngine {
}
async fn execute(&self, sql: &str) -> anyhow::Result<QueryResult> {
- let (columns, rows) = self.run(&wrap_limit(sql)).await?;
+ let (columns, rows) = self.run(&wrap_limit(sql), Some(ROW_CAP + 1)).await?;
let (rows, truncated) = truncate_rows(rows);
Ok(QueryResult {
rows: rows_to_json_lines(&columns, &rows),
@@ -475,4 +565,148 @@ mod tests {
assert!(err.contains("no columns for this catalog"), "{err}");
assert!(err.contains("cannot be found"), "{err}");
}
+
+ /// 宽 catalog 上的 schema 读按 docs 走 INLINE 多分块:第一个 chunk 给
+ /// `next_chunk_internal_link`,按它 GET 到第二个 chunk,没有下一页就停。
+ /// 之前对所有语句都传 row_limit=201(#703),宽 catalog 上 information_schema.columns
+ /// 一行一列,每张表第一个 200 列之后的列就被丢掉了;现在 `fetch_schema` 不传
+ /// row_limit,分块之间按 link 拉齐
+ #[tokio::test]
+ async fn a_schema_read_follows_chunks_until_exhausted() {
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(path("/api/2.0/sql/statements"))
+ .and(body_string_contains("`main`.information_schema.columns"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "statement_id": "s_chunk1",
+ "status": { "state": "SUCCEEDED" },
+ "manifest": {
+ "schema": { "columns": [
+ { "name": "table_schema", "type_text": "STRING", "position": 0 },
+ { "name": "table_name", "type_text": "STRING", "position": 1 },
+ { "name": "column_name", "type_text": "STRING", "position": 2 },
+ { "name": "data_type", "type_text": "STRING", "position": 3 },
+ { "name": "comment", "type_text": "STRING", "position": 4 }
+ ] },
+ "total_row_count": 3,
+ "next_chunk_internal_link": format!(
+ "{}/api/2.0/sql/statements/s_chunk1/chunk/2",
+ server.uri()
+ )
+ },
+ "result": { "data_array": [
+ ["default", "wide_orders", "id", "BIGINT", "主键"],
+ ["default", "wide_orders", "region", "STRING", null]
+ ] }
+ })))
+ .expect(1)
+ .mount(&server)
+ .await;
+ Mock::given(method("GET"))
+ .and(path("/api/2.0/sql/statements/s_chunk1/chunk/2"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "status": { "state": "SUCCEEDED" },
+ "manifest": {
+ "total_row_count": 3,
+ "next_chunk_internal_link": format!(
+ "{}/api/2.0/sql/statements/s_chunk1/chunk/3",
+ server.uri()
+ )
+ },
+ "result": { "data_array": [
+ ["default", "wide_orders", "amount", "DECIMAL(12,2)", null]
+ ] }
+ })))
+ .expect(1)
+ .mount(&server)
+ .await;
+ Mock::given(method("GET"))
+ .and(path("/api/2.0/sql/statements/s_chunk1/chunk/3"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "status": { "state": "SUCCEEDED" },
+ "manifest": { "total_row_count": 3 }
+ })))
+ .expect(1)
+ .mount(&server)
+ .await;
+
+ let cols = DatabricksEngine::new(conn(&server))
+ .fetch_schema()
+ .await
+ .unwrap();
+ assert_eq!(cols.len(), 3, "want 3 columns across chunks, got {cols:?}");
+ assert_eq!(cols[0].column, "id");
+ assert_eq!(cols[1].column, "region");
+ assert_eq!(cols[2].column, "amount");
+ }
+
+ /// 跑 chat 查询时(带 row_limit),服务端真的把行砍了:manifest.truncated=true
+ /// 应当直接失败,而不是带着 < ROW_CAP 行假装「查询成功」
+ #[tokio::test]
+ async fn a_chat_query_bails_loudly_when_the_service_says_truncated() {
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(path("/api/2.0/sql/statements"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "statement_id": "s_trunc",
+ "status": { "state": "SUCCEEDED" },
+ "manifest": {
+ "schema": { "columns": [
+ { "name": "n", "type_text": "BIGINT", "position": 0 }
+ ] },
+ "total_row_count": 1000,
+ "truncated": true
+ },
+ "result": { "data_array": [ ["1"], ["2"] ] }
+ })))
+ .expect(1)
+ .mount(&server)
+ .await;
+
+ let err = DatabricksEngine::new(conn(&server))
+ .execute("SELECT n FROM big")
+ .await
+ .unwrap_err()
+ .to_string();
+ assert!(err.contains("truncated"), "{err}");
+ assert!(err.contains("row_limit"), "{err}");
+ }
+
+ /// schema 读 manifest 报的 total_row_count 跟实际收到的行数对不上——服务端
+ /// 偷偷改 protocol,或中途丢包:fail-fast,免得把残缺的 schema 灌进结构文档
+ #[tokio::test]
+ async fn a_schema_read_fails_when_total_row_count_does_not_match() {
+ let server = MockServer::start().await;
+ Mock::given(method("POST"))
+ .and(path("/api/2.0/sql/statements"))
+ .and(body_string_contains("`main`.information_schema.columns"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "statement_id": "s_count_mismatch",
+ "status": { "state": "SUCCEEDED" },
+ "manifest": {
+ "schema": { "columns": [
+ { "name": "table_schema", "type_text": "STRING", "position": 0 },
+ { "name": "table_name", "type_text": "STRING", "position": 1 },
+ { "name": "column_name", "type_text": "STRING", "position": 2 },
+ { "name": "data_type", "type_text": "STRING", "position": 3 },
+ { "name": "comment", "type_text": "STRING", "position": 4 }
+ ] },
+ "total_row_count": 99
+ },
+ "result": { "data_array": [
+ ["default", "orders", "id", "BIGINT", null]
+ ] }
+ })))
+ .expect(1)
+ .mount(&server)
+ .await;
+
+ let err = DatabricksEngine::new(conn(&server))
+ .fetch_schema()
+ .await
+ .unwrap_err()
+ .to_string();
+ assert!(err.contains("schema read returned 1 rows"), "{err}");
+ assert!(err.contains("99"), "{err}");
+ }
}
diff --git a/crates/utopia-server/src/query_engine/snowflake.rs b/crates/utopia-server/src/query_engine/snowflake.rs
index ff810611..f8475fe4 100644
--- a/crates/utopia-server/src/query_engine/snowflake.rs
+++ b/crates/utopia-server/src/query_engine/snowflake.rs
@@ -29,19 +29,35 @@ struct StatementResponse {
handle: Option<String>,
}
-#[derive(Deserialize)]
+#[derive(Deserialize, Clone)]
struct Meta {
#[serde(rename = "rowType")]
row_type: Vec<RowType>,
+ #[serde(rename = "numRows")]
+ num_rows: Option<u64>,
+ /// 多分块时给的 chunk URL 列表;每条带 rowCount 和 url。按 docs 的语义 GET
+ /// 每个 url 拼回完整结果集。schema 读到这里的多分块表示 information_schema.columns
+ /// 一次返回装不下,需要跟 #703 Databricks 的解法同样的「跟 link 拉齐」路径
+ #[serde(rename = "partitionInfo")]
+ partition_info: Option<Vec<PartitionInfo>>,
}
-#[derive(Deserialize)]
+#[derive(Deserialize, Clone)]
struct RowType {
name: String,
#[serde(rename = "type")]
ty: String,
}
+/// Snowflake 的 partition 信息:每个 chunk 一个 URL,按 url GET 拿剩下的 data。
+/// `row_count` 是这个 chunk 自己的行数,`url` 是相对路径
+#[derive(Deserialize, Clone)]
+struct PartitionInfo {
+ #[serde(rename = "rowCount")]
+ row_count: u64,
+ url: Option<String>,
+}
+
impl SnowflakeEngine {
pub fn new(conn: SnowflakeConn) -> Self {
Self { conn }
@@ -110,8 +126,19 @@ impl SnowflakeEngine {
anyhow::bail!("Snowflake {code}: {message}");
}
}
- let types: Vec<RowType> = resp.meta.map(|m| m.row_type).unwrap_or_default();
- let rows = resp
+ // 把 meta 拷出一份再消费;下面既要从 partition_info 拉分块,
+ // 又要从 row_type 还原类型,两个都要用
+ let meta = resp.meta.clone();
+ let types: Vec<RowType> = meta
+ .as_ref()
+ .map(|m| m.row_type.clone())
+ .unwrap_or_default();
+ let partitions: Vec<PartitionInfo> = meta
+ .as_ref()
+ .and_then(|m| m.partition_info.clone())
+ .unwrap_or_default();
+ let expected_total_rows: Option<u64> = meta.as_ref().and_then(|m| m.num_rows);
+ let mut rows: Vec<Vec<serde_json::Value>> = resp
.data
.unwrap_or_default()
.into_iter()
@@ -122,6 +149,49 @@ impl SnowflakeEngine {
.collect()
})
.collect();
+ // Snowflake 的 schema 读需要拉齐所有 partition:一次返回装不下时按 partitionInfo
+ // 里每条 URL GET 拿剩下的 data。partitionInfo 的第一条对应的就是这次响应本身
+ // (按 docs「first partition is returned inline」),所以从第二条开始拉
+ // (#703:之前 partitionInfo 直接被丢,宽 catalog 上 schema 文档被截断)
+ let mut seen_partition_rows: u64 = rows.len() as u64;
+ if partitions.len() > 1 {
+ for chunk in partitions.iter().skip(1) {
+ let Some(url) = chunk.url.as_deref() else {
+ // docs 说每个 partition 必须有 url;没有就当服务端偷偷少给了
+ anyhow::bail!("Snowflake partition without a url; rows may be incomplete");
+ };
+ let chunk_resp: StatementResponse = self
+ .request(client.get(url))
+ .send()
+ .await?
+ .error_for_status()?
+ .json()
+ .await?;
+ let chunk_rows = chunk_resp.data.unwrap_or_default();
+ seen_partition_rows = seen_partition_rows.saturating_add(chunk_rows.len() as u64);
+ rows.extend(chunk_rows.into_iter().map(|row| {
+ row.iter()
+ .enumerate()
+ .map(|(i, v)| coerce(types.get(i).map(|t| t.ty.as_str()).unwrap_or(""), v))
+ .collect()
+ }));
+ }
+ }
+ // numRows 是整个结果集的总数(不是当前 chunk 的);分区 rowCount 之和也应该是它
+ // ——任一对不上都说明服务端偷偷改 protocol 或中途丢包,残缺数据不要灌进结构文档
+ if let Some(expected) = expected_total_rows {
+ if seen_partition_rows != expected {
+ anyhow::bail!(
+ "Snowflake schema read returned {seen_partition_rows} rows across partitions but numRows reports {expected}"
+ );
+ }
+ }
+ let partition_total: u64 = partitions.iter().map(|p| p.row_count).sum();
+ if !partitions.is_empty() && partition_total != seen_partition_rows {
+ anyhow::bail!(
+ "Snowflake partition row counts sum to {partition_total} but received {seen_partition_rows} rows"
+ );
+ }
Ok((types.into_iter().map(|t| t.name).collect(), rows))
}
}
@@ -254,4 +324,74 @@ mod tests {
.to_string();
assert!(err.contains("does not exist"), "{err}");
}
+
+ /// schema 读按 partitionInfo 走多分块:第一条 partition 对应这次响应本身,
+ /// 从第二条开始按 url GET 拿剩下的 data。docs 说 num_rows 是整个结果集的总数,
+ /// 不是当前 chunk 的——可以拿来对账(这里不强校验,但证明分块走通)
+ /// (#703:之前 partitionInfo 直接被丢,宽 catalog 上 schema 文档被截断)
+ #[tokio::test]
+ async fn a_schema_read_follows_partitions_until_exhausted() {
+ let server = MockServer::start().await;
+ let chunk2_url = format!("{}/api/v2/statements/h3/chunk/2", server.uri());
+ let chunk3_url = format!("{}/api/v2/statements/h3/chunk/3", server.uri());
+ Mock::given(method("POST"))
+ .and(path("/api/v2/statements"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "resultSetMetaData": {
+ "numRows": 3,
+ "rowType": [
+ { "name": "TABLE_SCHEMA", "type": "text" },
+ { "name": "TABLE_NAME", "type": "text" },
+ { "name": "COLUMN_NAME", "type": "text" },
+ { "name": "DATA_TYPE", "type": "text" },
+ { "name": "COMMENT", "type": "text" }
+ ],
+ "partitionInfo": [
+ { "rowCount": 1, "url": format!("{}/api/v2/statements/h3/chunk/1", server.uri()) },
+ { "rowCount": 1, "url": chunk2_url },
+ { "rowCount": 1, "url": chunk3_url }
+ ]
+ },
+ "data": [
+ ["default", "wide_orders", "id", "BIGINT", "主键"]
+ ],
+ "code": "090001",
+ "statementHandle": "h3",
+ "message": "Statement executed successfully."
+ })))
+ .expect(1)
+ .mount(&server)
+ .await;
+ Mock::given(method("GET"))
+ .and(path("/api/v2/statements/h3/chunk/2"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "data": [ ["default", "wide_orders", "region", "STRING", null] ],
+ "code": "090001"
+ })))
+ .expect(1)
+ .mount(&server)
+ .await;
+ Mock::given(method("GET"))
+ .and(path("/api/v2/statements/h3/chunk/3"))
+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({
+ "data": [ ["default", "wide_orders", "amount", "DECIMAL(12,2)", null] ],
+ "code": "090001"
+ })))
+ .expect(1)
+ .mount(&server)
+ .await;
+
+ let cols = SnowflakeEngine::new(conn(&server))
+ .fetch_schema()
+ .await
+ .unwrap();
+ assert_eq!(
+ cols.len(),
+ 3,
+ "want 3 columns across partitions, got {cols:?}"
+ );
+ assert_eq!(cols[0].column, "id");
+ assert_eq!(cols[1].column, "region");
+ assert_eq!(cols[2].column, "amount");
+ }
}