World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Schema reads do not apply ROW_CAP and follow result chunks (#737)
Royce4137f7d1a1cf
diff --git a/crates/utopia-server/src/query_engine/databricks.rs b/crates/utopia-server/src/query_engine/databricks.rsindex 34641a83..a6895bcd 100644--- a/crates/utopia-server/src/query_engine/databricks.rs+++ b/crates/utopia-server/src/query_engine/databricks.rs@@ -39,14 +39,23 @@ struct StatusError {#[derive(Deserialize)]struct Manifest {schema: Option<Schema>,+ /// INLINE disposition with a `row_limit` set:true 表示结果按行截断。+ /// schema 读不传 row_limit,理论上不会触发;留着做显式 fail-fast 而不是静默返回+ /// 不全的数据+ truncated: Option<bool>,+ /// INLINE 第一个分块塞不下时给的下一页 GET 链接;按 docs 的语义反复 GET 直到没有+ next_chunk_internal_link: Option<String>,+ /// 整个结果集的总行数(chunked INLINE 时也有)。schema 读完后用来校验+ /// `rows.len() == total_row_count`+ total_row_count: Option<u64>,}-#[derive(Deserialize)]+#[derive(Deserialize, Clone)]struct Schema {columns: Vec<ColumnInfo>,}-#[derive(Deserialize)]+#[derive(Deserialize, Clone)]struct ColumnInfo {name: String,type_text: Option<String>,@@ -101,7 +110,23 @@ impl DatabricksEngine {}}- async fn run(&self, sql: &str) -> anyhow::Result<(Vec<String>, Vec<Vec<serde_json::Value>>)> {+ /// 提交一条语句,轮询到 SUCCEEDED,把结果拼成 (列名, 行值)。所有列值都按+ /// manifest 的 `type_text` 还原成数或布尔。+ ///+ /// `row_limit`:传 `None` 表示不限行(schema 读用)。INLINE disposition 在+ /// 服务端有 25 MiB 的 body 上限,所以单分块塞不下时通过 `next_chunk_internal_link`+ /// 继续 GET,循环到 manifest 没有给出下一页为止。schema 读完后用+ /// `manifest.total_row_count` 做一次 `rows.len()` 校验——少了就当结果不全+ /// (docs 说 truncated=true 时另有标记,schema 读不该见到)+ ///+ /// 之前对所有语句都传 `row_limit: ROW_CAP + 1`(#703):chat 查询需要这个+ /// 安全阀,schema 读不需要;information_schema.columns 每行是一列,宽 catalog+ /// 上的 schema 文档会被悄悄截到 201 行+ async fn run(+ &self,+ sql: &str,+ row_limit: Option<usize>,+ ) -> anyhow::Result<(Vec<String>, Vec<Vec<serde_json::Value>>)> {let client = super::http()?;let mut body = json!({"warehouse_id": self.conn.warehouse_id,@@ -110,8 +135,10 @@ impl DatabricksEngine {"on_wait_timeout": "CONTINUE","disposition": "INLINE","format": "JSON_ARRAY",- "row_limit": ROW_CAP + 1,});+ if let Some(n) = row_limit {+ body["row_limit"] = json!(n);+ }if let Some(c) = &self.conn.catalog {body["catalog"] = json!(c);}@@ -167,11 +194,15 @@ impl DatabricksEngine {}let columns: Vec<ColumnInfo> = resp.manifest- .and_then(|m| m.schema)- .map(|s| s.columns)+ .as_ref()+ .and_then(|m| m.schema.as_ref())+ .map(|s| s.columns.clone()).unwrap_or_default();- let raw_rows = resp.result.and_then(|r| r.data_array).unwrap_or_default();- let rows = raw_rows+ let mut rows: Vec<Vec<serde_json::Value>> = resp+ .result+ .as_ref()+ .and_then(|r| r.data_array.clone())+ .unwrap_or_default().into_iter().map(|row| {row.iter()@@ -186,6 +217,61 @@ impl DatabricksEngine {.collect()}).collect();+ // INLINE 的分块跟在第一个 chunk 之后:next_chunk_internal_link 是相对路径,+ // docs 说按 link 反复 GET,直到 manifest 不再给出下一页为止+ let manifest = resp.manifest.as_ref();+ let truncated = manifest.and_then(|m| m.truncated).unwrap_or(false);+ if truncated {+ anyhow::bail!(+ "Databricks returned a truncated result; row_limit must be raised or the query rewritten"+ );+ }+ let mut next_link = manifest.and_then(|m| m.next_chunk_internal_link.clone());+ while let Some(link) = next_link.take() {+ let chunk: StatementResponse = client+ .get(link)+ .bearer_auth(&self.conn.token)+ .send()+ .await?+ .error_for_status()?+ .json()+ .await?;+ // 中间 chunk 的 manifest 可能再次给 truncated/link;状态字段对每块都要读+ let m = chunk.manifest.as_ref();+ if m.and_then(|m| m.truncated).unwrap_or(false) {+ anyhow::bail!(+ "Databricks chunk was truncated by the service; rows may be incomplete"+ );+ }+ let chunk_rows = chunk.result.and_then(|r| r.data_array).unwrap_or_default();+ rows.extend(chunk_rows.into_iter().map(|row| {+ row.iter()+ .enumerate()+ .map(|(i, v)| {+ let ty = columns+ .get(i)+ .and_then(|c| c.type_text.as_deref())+ .unwrap_or("");+ coerce(ty, v)+ })+ .collect()+ }));+ next_link = m.and_then(|m| m.next_chunk_internal_link.clone());+ }+ // schema 读按 docs 说不该见到 truncated=true;total_row_count 是每次都给的,+ // 校验要放在所有 chunk 都收齐之后:第一次响应给的 total 是整个结果集的总数,+ // 不是第一个 chunk 的总数+ if let (Some(manifest), None) = (manifest, row_limit) {+ let expected = manifest.total_row_count;+ if let Some(expected) = expected {+ let got: u64 = rows.len() as u64;+ if got != expected {+ anyhow::bail!(+ "Databricks schema read returned {got} rows but manifest reports {expected}"+ );+ }+ }+ }Ok((columns.into_iter().map(|c| c.name).collect(), rows))}}@@ -206,7 +292,7 @@ fn is_not_found(e: &anyhow::Error) -> bool {#[async_trait::async_trait]impl QueryEngine for DatabricksEngine {async fn test(&self) -> anyhow::Result<()> {- self.run("SELECT 1").await.map(|_| ())+ self.run("SELECT 1", Some(ROW_CAP + 1)).await.map(|_| ())}async fn fetch_schema(&self) -> anyhow::Result<Vec<SchemaColumn>> {@@ -218,10 +304,14 @@ impl QueryEngine for DatabricksEngine {// 都只列令牌看得见的对象,system 那份还不含 hive_metastore,看不见的 catalog// 在那里是「成功地没有行」;当成功的话,结构文档会被一份空的覆盖掉。// 失败时每条试过的写法连同引擎的原话都带上+ //+ // row_limit 不传(#703):information_schema.columns 每行是一列,宽 catalog+ // 上的 schema 文档会被悄悄截到 201 行;现在 `run` 自己跟 `next_chunk_internal_link`+ // 一直拉到 `manifest.total_row_count` 行,多分块也收齐let queries = self.schema_queries();let mut tried: Vec<String> = Vec::new();for (i, (label, sql)) in queries.iter().enumerate() {- match self.run(sql).await {+ match self.run(sql, None).await {Ok((_, rows)) if rows.is_empty() && i > 0 => {tried.push(format!("{label}: no columns for this catalog — the token cannot see it, \@@ -245,7 +335,7 @@ impl QueryEngine for DatabricksEngine {}async fn execute(&self, sql: &str) -> anyhow::Result<QueryResult> {- let (columns, rows) = self.run(&wrap_limit(sql)).await?;+ let (columns, rows) = self.run(&wrap_limit(sql), Some(ROW_CAP + 1)).await?;let (rows, truncated) = truncate_rows(rows);Ok(QueryResult {rows: rows_to_json_lines(&columns, &rows),@@ -475,4 +565,148 @@ mod tests {assert!(err.contains("no columns for this catalog"), "{err}");assert!(err.contains("cannot be found"), "{err}");}++ /// 宽 catalog 上的 schema 读按 docs 走 INLINE 多分块:第一个 chunk 给+ /// `next_chunk_internal_link`,按它 GET 到第二个 chunk,没有下一页就停。+ /// 之前对所有语句都传 row_limit=201(#703),宽 catalog 上 information_schema.columns+ /// 一行一列,每张表第一个 200 列之后的列就被丢掉了;现在 `fetch_schema` 不传+ /// row_limit,分块之间按 link 拉齐+ #[tokio::test]+ async fn a_schema_read_follows_chunks_until_exhausted() {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(path("/api/2.0/sql/statements"))+ .and(body_string_contains("`main`.information_schema.columns"))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({+ "statement_id": "s_chunk1",+ "status": { "state": "SUCCEEDED" },+ "manifest": {+ "schema": { "columns": [+ { "name": "table_schema", "type_text": "STRING", "position": 0 },+ { "name": "table_name", "type_text": "STRING", "position": 1 },+ { "name": "column_name", "type_text": "STRING", "position": 2 },+ { "name": "data_type", "type_text": "STRING", "position": 3 },+ { "name": "comment", "type_text": "STRING", "position": 4 }+ ] },+ "total_row_count": 3,+ "next_chunk_internal_link": format!(+ "{}/api/2.0/sql/statements/s_chunk1/chunk/2",+ server.uri()+ )+ },+ "result": { "data_array": [+ ["default", "wide_orders", "id", "BIGINT", "主键"],+ ["default", "wide_orders", "region", "STRING", null]+ ] }+ })))+ .expect(1)+ .mount(&server)+ .await;+ Mock::given(method("GET"))+ .and(path("/api/2.0/sql/statements/s_chunk1/chunk/2"))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({+ "status": { "state": "SUCCEEDED" },+ "manifest": {+ "total_row_count": 3,+ "next_chunk_internal_link": format!(+ "{}/api/2.0/sql/statements/s_chunk1/chunk/3",+ server.uri()+ )+ },+ "result": { "data_array": [+ ["default", "wide_orders", "amount", "DECIMAL(12,2)", null]+ ] }+ })))+ .expect(1)+ .mount(&server)+ .await;+ Mock::given(method("GET"))+ .and(path("/api/2.0/sql/statements/s_chunk1/chunk/3"))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({+ "status": { "state": "SUCCEEDED" },+ "manifest": { "total_row_count": 3 }+ })))+ .expect(1)+ .mount(&server)+ .await;++ let cols = DatabricksEngine::new(conn(&server))+ .fetch_schema()+ .await+ .unwrap();+ assert_eq!(cols.len(), 3, "want 3 columns across chunks, got {cols:?}");+ assert_eq!(cols[0].column, "id");+ assert_eq!(cols[1].column, "region");+ assert_eq!(cols[2].column, "amount");+ }++ /// 跑 chat 查询时(带 row_limit),服务端真的把行砍了:manifest.truncated=true+ /// 应当直接失败,而不是带着 < ROW_CAP 行假装「查询成功」+ #[tokio::test]+ async fn a_chat_query_bails_loudly_when_the_service_says_truncated() {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(path("/api/2.0/sql/statements"))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({+ "statement_id": "s_trunc",+ "status": { "state": "SUCCEEDED" },+ "manifest": {+ "schema": { "columns": [+ { "name": "n", "type_text": "BIGINT", "position": 0 }+ ] },+ "total_row_count": 1000,+ "truncated": true+ },+ "result": { "data_array": [ ["1"], ["2"] ] }+ })))+ .expect(1)+ .mount(&server)+ .await;++ let err = DatabricksEngine::new(conn(&server))+ .execute("SELECT n FROM big")+ .await+ .unwrap_err()+ .to_string();+ assert!(err.contains("truncated"), "{err}");+ assert!(err.contains("row_limit"), "{err}");+ }++ /// schema 读 manifest 报的 total_row_count 跟实际收到的行数对不上——服务端+ /// 偷偷改 protocol,或中途丢包:fail-fast,免得把残缺的 schema 灌进结构文档+ #[tokio::test]+ async fn a_schema_read_fails_when_total_row_count_does_not_match() {+ let server = MockServer::start().await;+ Mock::given(method("POST"))+ .and(path("/api/2.0/sql/statements"))+ .and(body_string_contains("`main`.information_schema.columns"))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({+ "statement_id": "s_count_mismatch",+ "status": { "state": "SUCCEEDED" },+ "manifest": {+ "schema": { "columns": [+ { "name": "table_schema", "type_text": "STRING", "position": 0 },+ { "name": "table_name", "type_text": "STRING", "position": 1 },+ { "name": "column_name", "type_text": "STRING", "position": 2 },+ { "name": "data_type", "type_text": "STRING", "position": 3 },+ { "name": "comment", "type_text": "STRING", "position": 4 }+ ] },+ "total_row_count": 99+ },+ "result": { "data_array": [+ ["default", "orders", "id", "BIGINT", null]+ ] }+ })))+ .expect(1)+ .mount(&server)+ .await;++ let err = DatabricksEngine::new(conn(&server))+ .fetch_schema()+ .await+ .unwrap_err()+ .to_string();+ assert!(err.contains("schema read returned 1 rows"), "{err}");+ assert!(err.contains("99"), "{err}");+ }}diff --git a/crates/utopia-server/src/query_engine/snowflake.rs b/crates/utopia-server/src/query_engine/snowflake.rsindex ff810611..f8475fe4 100644--- a/crates/utopia-server/src/query_engine/snowflake.rs+++ b/crates/utopia-server/src/query_engine/snowflake.rs@@ -29,19 +29,35 @@ struct StatementResponse {handle: Option<String>,}-#[derive(Deserialize)]+#[derive(Deserialize, Clone)]struct Meta {#[serde(rename = "rowType")]row_type: Vec<RowType>,+ #[serde(rename = "numRows")]+ num_rows: Option<u64>,+ /// 多分块时给的 chunk URL 列表;每条带 rowCount 和 url。按 docs 的语义 GET+ /// 每个 url 拼回完整结果集。schema 读到这里的多分块表示 information_schema.columns+ /// 一次返回装不下,需要跟 #703 Databricks 的解法同样的「跟 link 拉齐」路径+ #[serde(rename = "partitionInfo")]+ partition_info: Option<Vec<PartitionInfo>>,}-#[derive(Deserialize)]+#[derive(Deserialize, Clone)]struct RowType {name: String,#[serde(rename = "type")]ty: String,}+/// Snowflake 的 partition 信息:每个 chunk 一个 URL,按 url GET 拿剩下的 data。+/// `row_count` 是这个 chunk 自己的行数,`url` 是相对路径+#[derive(Deserialize, Clone)]+struct PartitionInfo {+ #[serde(rename = "rowCount")]+ row_count: u64,+ url: Option<String>,+}+impl SnowflakeEngine {pub fn new(conn: SnowflakeConn) -> Self {Self { conn }@@ -110,8 +126,19 @@ impl SnowflakeEngine {anyhow::bail!("Snowflake {code}: {message}");}}- let types: Vec<RowType> = resp.meta.map(|m| m.row_type).unwrap_or_default();- let rows = resp+ // 把 meta 拷出一份再消费;下面既要从 partition_info 拉分块,+ // 又要从 row_type 还原类型,两个都要用+ let meta = resp.meta.clone();+ let types: Vec<RowType> = meta+ .as_ref()+ .map(|m| m.row_type.clone())+ .unwrap_or_default();+ let partitions: Vec<PartitionInfo> = meta+ .as_ref()+ .and_then(|m| m.partition_info.clone())+ .unwrap_or_default();+ let expected_total_rows: Option<u64> = meta.as_ref().and_then(|m| m.num_rows);+ let mut rows: Vec<Vec<serde_json::Value>> = resp.data.unwrap_or_default().into_iter()@@ -122,6 +149,49 @@ impl SnowflakeEngine {.collect()}).collect();+ // Snowflake 的 schema 读需要拉齐所有 partition:一次返回装不下时按 partitionInfo+ // 里每条 URL GET 拿剩下的 data。partitionInfo 的第一条对应的就是这次响应本身+ // (按 docs「first partition is returned inline」),所以从第二条开始拉+ // (#703:之前 partitionInfo 直接被丢,宽 catalog 上 schema 文档被截断)+ let mut seen_partition_rows: u64 = rows.len() as u64;+ if partitions.len() > 1 {+ for chunk in partitions.iter().skip(1) {+ let Some(url) = chunk.url.as_deref() else {+ // docs 说每个 partition 必须有 url;没有就当服务端偷偷少给了+ anyhow::bail!("Snowflake partition without a url; rows may be incomplete");+ };+ let chunk_resp: StatementResponse = self+ .request(client.get(url))+ .send()+ .await?+ .error_for_status()?+ .json()+ .await?;+ let chunk_rows = chunk_resp.data.unwrap_or_default();+ seen_partition_rows = seen_partition_rows.saturating_add(chunk_rows.len() as u64);+ rows.extend(chunk_rows.into_iter().map(|row| {+ row.iter()+ .enumerate()+ .map(|(i, v)| coerce(types.get(i).map(|t| t.ty.as_str()).unwrap_or(""), v))+ .collect()+ }));+ }+ }+ // numRows 是整个结果集的总数(不是当前 chunk 的);分区 rowCount 之和也应该是它+ // ——任一对不上都说明服务端偷偷改 protocol 或中途丢包,残缺数据不要灌进结构文档+ if let Some(expected) = expected_total_rows {+ if seen_partition_rows != expected {+ anyhow::bail!(+ "Snowflake schema read returned {seen_partition_rows} rows across partitions but numRows reports {expected}"+ );+ }+ }+ let partition_total: u64 = partitions.iter().map(|p| p.row_count).sum();+ if !partitions.is_empty() && partition_total != seen_partition_rows {+ anyhow::bail!(+ "Snowflake partition row counts sum to {partition_total} but received {seen_partition_rows} rows"+ );+ }Ok((types.into_iter().map(|t| t.name).collect(), rows))}}@@ -254,4 +324,74 @@ mod tests {.to_string();assert!(err.contains("does not exist"), "{err}");}++ /// schema 读按 partitionInfo 走多分块:第一条 partition 对应这次响应本身,+ /// 从第二条开始按 url GET 拿剩下的 data。docs 说 num_rows 是整个结果集的总数,+ /// 不是当前 chunk 的——可以拿来对账(这里不强校验,但证明分块走通)+ /// (#703:之前 partitionInfo 直接被丢,宽 catalog 上 schema 文档被截断)+ #[tokio::test]+ async fn a_schema_read_follows_partitions_until_exhausted() {+ let server = MockServer::start().await;+ let chunk2_url = format!("{}/api/v2/statements/h3/chunk/2", server.uri());+ let chunk3_url = format!("{}/api/v2/statements/h3/chunk/3", server.uri());+ Mock::given(method("POST"))+ .and(path("/api/v2/statements"))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({+ "resultSetMetaData": {+ "numRows": 3,+ "rowType": [+ { "name": "TABLE_SCHEMA", "type": "text" },+ { "name": "TABLE_NAME", "type": "text" },+ { "name": "COLUMN_NAME", "type": "text" },+ { "name": "DATA_TYPE", "type": "text" },+ { "name": "COMMENT", "type": "text" }+ ],+ "partitionInfo": [+ { "rowCount": 1, "url": format!("{}/api/v2/statements/h3/chunk/1", server.uri()) },+ { "rowCount": 1, "url": chunk2_url },+ { "rowCount": 1, "url": chunk3_url }+ ]+ },+ "data": [+ ["default", "wide_orders", "id", "BIGINT", "主键"]+ ],+ "code": "090001",+ "statementHandle": "h3",+ "message": "Statement executed successfully."+ })))+ .expect(1)+ .mount(&server)+ .await;+ Mock::given(method("GET"))+ .and(path("/api/v2/statements/h3/chunk/2"))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({+ "data": [ ["default", "wide_orders", "region", "STRING", null] ],+ "code": "090001"+ })))+ .expect(1)+ .mount(&server)+ .await;+ Mock::given(method("GET"))+ .and(path("/api/v2/statements/h3/chunk/3"))+ .respond_with(ResponseTemplate::new(200).set_body_json(json!({+ "data": [ ["default", "wide_orders", "amount", "DECIMAL(12,2)", null] ],+ "code": "090001"+ })))+ .expect(1)+ .mount(&server)+ .await;++ let cols = SnowflakeEngine::new(conn(&server))+ .fetch_schema()+ .await+ .unwrap();+ assert_eq!(+ cols.len(),+ 3,+ "want 3 columns across partitions, got {cols:?}"+ );+ assert_eq!(cols[0].column, "id");+ assert_eq!(cols[1].column, "region");+ assert_eq!(cols[2].column, "amount");+ }}